967 files changed, 41484 insertions, 30424 deletions

diff --git a/compiler/rustc/Cargo.toml b/compiler/rustc/Cargo.toml
index 27ee3dd2a..41003ad83 100644
--- a/compiler/rustc/Cargo.toml
+++ b/compiler/rustc/Cargo.toml
@@ -5,6 +5,7 @@ edition = "2021"
 
 [dependencies]
 rustc_driver = { path = "../rustc_driver" }
+rustc_driver_impl = { path = "../rustc_driver_impl" }
 
 # Make sure rustc_codegen_ssa ends up in the sysroot, because this
 # crate is intended to be used by codegen backends, which may not be in-tree.
@@ -20,6 +21,6 @@ features = ['unprefixed_malloc_on_supported_platforms']
 
 [features]
 jemalloc = ['jemalloc-sys']
-llvm = ['rustc_driver/llvm']
-max_level_info = ['rustc_driver/max_level_info']
-rustc_use_parallel_compiler = ['rustc_driver/rustc_use_parallel_compiler']
+llvm = ['rustc_driver_impl/llvm']
+max_level_info = ['rustc_driver_impl/max_level_info']
+rustc_use_parallel_compiler = ['rustc_driver_impl/rustc_use_parallel_compiler']
diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs
index 9c2cf58ef..54858b520 100644
--- a/compiler/rustc_abi/src/layout.rs
+++ b/compiler/rustc_abi/src/layout.rs
@@ -1,11 +1,5 @@
 use super::*;
-use std::{
-    borrow::Borrow,
-    cmp,
-    fmt::Debug,
-    iter,
-    ops::{Bound, Deref},
-};
+use std::{borrow::Borrow, cmp, iter, ops::Bound};
 
 #[cfg(feature = "randomize")]
 use rand::{seq::SliceRandom, SeedableRng};
@@ -33,7 +27,7 @@ pub trait LayoutCalculator {
     fn delay_bug(&self, txt: &str);
     fn current_data_layout(&self) -> Self::TargetDataLayoutRef;
 
-    fn scalar_pair<V: Idx>(&self, a: Scalar, b: Scalar) -> LayoutS<V> {
+    fn scalar_pair(&self, a: Scalar, b: Scalar) -> LayoutS {
         let dl = self.current_data_layout();
         let dl = dl.borrow();
         let b_align = b.align(dl);
@@ -49,7 +43,7 @@ pub trait LayoutCalculator {
             .max_by_key(|niche| niche.available(dl));
 
         LayoutS {
-            variants: Variants::Single { index: V::new(0) },
+            variants: Variants::Single { index: VariantIdx::new(0) },
             fields: FieldsShape::Arbitrary {
                 offsets: vec![Size::ZERO, b_offset],
                 memory_index: vec![0, 1],
@@ -61,13 +55,13 @@ pub trait LayoutCalculator {
         }
     }
 
-    fn univariant<'a, V: Idx, F: Deref<Target = &'a LayoutS<V>> + Debug>(
+    fn univariant(
         &self,
         dl: &TargetDataLayout,
-        fields: &[F],
+        fields: &[Layout<'_>],
         repr: &ReprOptions,
         kind: StructKind,
-    ) -> Option<LayoutS<V>> {
+    ) -> Option<LayoutS> {
         let pack = repr.pack;
         let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align };
         let mut inverse_memory_index: Vec<u32> = (0..fields.len() as u32).collect();
@@ -76,17 +70,17 @@ pub trait LayoutCalculator {
             let end =
                 if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() };
             let optimizing = &mut inverse_memory_index[..end];
-            let effective_field_align = |f: &F| {
+            let effective_field_align = |layout: Layout<'_>| {
                 if let Some(pack) = pack {
                     // return the packed alignment in bytes
-                    f.align.abi.min(pack).bytes()
+                    layout.align().abi.min(pack).bytes()
                 } else {
                     // returns log2(effective-align).
                     // This is ok since `pack` applies to all fields equally.
                     // The calculation assumes that size is an integer multiple of align, except for ZSTs.
                     //
                     // group [u8; 4] with align-4 or [u8; 6] with align-2 fields
-                    f.align.abi.bytes().max(f.size.bytes()).trailing_zeros() as u64
+                    layout.align().abi.bytes().max(layout.size().bytes()).trailing_zeros() as u64
                 }
             };
 
@@ -111,9 +105,9 @@ pub trait LayoutCalculator {
                             // Place ZSTs first to avoid "interesting offsets",
                             // especially with only one or two non-ZST fields.
                             // Then place largest alignments first, largest niches within an alignment group last
-                            let f = &fields[x as usize];
-                            let niche_size = f.largest_niche.map_or(0, |n| n.available(dl));
-                            (!f.is_zst(), cmp::Reverse(effective_field_align(f)), niche_size)
+                            let f = fields[x as usize];
+                            let niche_size = f.largest_niche().map_or(0, |n| n.available(dl));
+                            (!f.0.is_zst(), cmp::Reverse(effective_field_align(f)), niche_size)
                         });
                     }
 
@@ -123,8 +117,8 @@ pub trait LayoutCalculator {
                         // And put the largest niche in an alignment group at the end
                         // so it can be used as discriminant in jagged enums
                         optimizing.sort_by_key(|&x| {
-                            let f = &fields[x as usize];
-                            let niche_size = f.largest_niche.map_or(0, |n| n.available(dl));
+                            let f = fields[x as usize];
+                            let niche_size = f.largest_niche().map_or(0, |n| n.available(dl));
                             (effective_field_align(f), niche_size)
                         });
                     }
@@ -160,15 +154,15 @@ pub trait LayoutCalculator {
                 ));
             }
 
-            if field.is_unsized() {
+            if field.0.is_unsized() {
                 sized = false;
             }
 
             // Invariant: offset < dl.obj_size_bound() <= 1<<61
             let field_align = if let Some(pack) = pack {
-                field.align.min(AbiAndPrefAlign::new(pack))
+                field.align().min(AbiAndPrefAlign::new(pack))
             } else {
-                field.align
+                field.align()
             };
             offset = offset.align_to(field_align.abi);
             align = align.max(field_align);
@@ -176,7 +170,7 @@ pub trait LayoutCalculator {
             debug!("univariant offset: {:?} field: {:#?}", offset, field);
             offsets[i as usize] = offset;
 
-            if let Some(mut niche) = field.largest_niche {
+            if let Some(mut niche) = field.largest_niche() {
                 let available = niche.available(dl);
                 if available > largest_niche_available {
                     largest_niche_available = available;
@@ -185,7 +179,7 @@ pub trait LayoutCalculator {
                 }
             }
 
-            offset = offset.checked_add(field.size, dl)?;
+            offset = offset.checked_add(field.size(), dl)?;
         }
         if let Some(repr_align) = repr.align {
             align = align.max(AbiAndPrefAlign::new(repr_align));
@@ -205,24 +199,26 @@ pub trait LayoutCalculator {
         // Unpack newtype ABIs and find scalar pairs.
         if sized && size.bytes() > 0 {
             // All other fields must be ZSTs.
-            let mut non_zst_fields = fields.iter().enumerate().filter(|&(_, f)| !f.is_zst());
+            let mut non_zst_fields = fields.iter().enumerate().filter(|&(_, f)| !f.0.is_zst());
 
             match (non_zst_fields.next(), non_zst_fields.next(), non_zst_fields.next()) {
                 // We have exactly one non-ZST field.
                 (Some((i, field)), None, None) => {
                     // Field fills the struct and it has a scalar or scalar pair ABI.
-                    if offsets[i].bytes() == 0 && align.abi == field.align.abi && size == field.size
+                    if offsets[i].bytes() == 0
+                        && align.abi == field.align().abi
+                        && size == field.size()
                     {
-                        match field.abi {
+                        match field.abi() {
                             // For plain scalars, or vectors of them, we can't unpack
                             // newtypes for `#[repr(C)]`, as that affects C ABIs.
                             Abi::Scalar(_) | Abi::Vector { .. } if optimize => {
-                                abi = field.abi;
+                                abi = field.abi();
                             }
                             // But scalar pairs are Rust-specific and get
                             // treated as aggregates by C ABIs anyway.
                             Abi::ScalarPair(..) => {
-                                abi = field.abi;
+                                abi = field.abi();
                             }
                             _ => {}
                         }
@@ -231,7 +227,7 @@ pub trait LayoutCalculator {
 
                 // Two non-ZST fields, and they're both scalars.
                 (Some((i, a)), Some((j, b)), None) => {
-                    match (a.abi, b.abi) {
+                    match (a.abi(), b.abi()) {
                         (Abi::Scalar(a), Abi::Scalar(b)) => {
                             // Order by the memory placement, not source order.
                             let ((i, a), (j, b)) = if offsets[i] < offsets[j] {
@@ -239,7 +235,7 @@ pub trait LayoutCalculator {
                             } else {
                                 ((j, b), (i, a))
                             };
-                            let pair = self.scalar_pair::<V>(a, b);
+                            let pair = self.scalar_pair(a, b);
                             let pair_offsets = match pair.fields {
                                 FieldsShape::Arbitrary { ref offsets, ref memory_index } => {
                                     assert_eq!(memory_index, &[0, 1]);
@@ -264,11 +260,11 @@ pub trait LayoutCalculator {
                 _ => {}
             }
         }
-        if fields.iter().any(|f| f.abi.is_uninhabited()) {
+        if fields.iter().any(|f| f.abi().is_uninhabited()) {
             abi = Abi::Uninhabited;
         }
         Some(LayoutS {
-            variants: Variants::Single { index: V::new(0) },
+            variants: Variants::Single { index: VariantIdx::new(0) },
             fields: FieldsShape::Arbitrary { offsets, memory_index },
             abi,
             largest_niche,
@@ -277,11 +273,11 @@ pub trait LayoutCalculator {
         })
     }
 
-    fn layout_of_never_type<V: Idx>(&self) -> LayoutS<V> {
+    fn layout_of_never_type(&self) -> LayoutS {
         let dl = self.current_data_layout();
         let dl = dl.borrow();
         LayoutS {
-            variants: Variants::Single { index: V::new(0) },
+            variants: Variants::Single { index: VariantIdx::new(0) },
             fields: FieldsShape::Primitive,
             abi: Abi::Uninhabited,
             largest_niche: None,
@@ -290,18 +286,18 @@ pub trait LayoutCalculator {
         }
     }
 
-    fn layout_of_struct_or_enum<'a, V: Idx, F: Deref<Target = &'a LayoutS<V>> + Debug>(
+    fn layout_of_struct_or_enum(
         &self,
         repr: &ReprOptions,
-        variants: &IndexVec<V, Vec<F>>,
+        variants: &IndexVec<VariantIdx, Vec<Layout<'_>>>,
         is_enum: bool,
         is_unsafe_cell: bool,
         scalar_valid_range: (Bound<u128>, Bound<u128>),
         discr_range_of_repr: impl Fn(i128, i128) -> (Integer, bool),
-        discriminants: impl Iterator<Item = (V, i128)>,
+        discriminants: impl Iterator<Item = (VariantIdx, i128)>,
         niche_optimize_enum: bool,
         always_sized: bool,
-    ) -> Option<LayoutS<V>> {
+    ) -> Option<LayoutS> {
         let dl = self.current_data_layout();
         let dl = dl.borrow();
 
@@ -316,9 +312,9 @@ pub trait LayoutCalculator {
         // but *not* an encoding of the discriminant (e.g., a tag value).
         // See issue #49298 for more details on the need to leave space
         // for non-ZST uninhabited data (mostly partial initialization).
-        let absent = |fields: &[F]| {
-            let uninhabited = fields.iter().any(|f| f.abi.is_uninhabited());
-            let is_zst = fields.iter().all(|f| f.is_zst());
+        let absent = |fields: &[Layout<'_>]| {
+            let uninhabited = fields.iter().any(|f| f.abi().is_uninhabited());
+            let is_zst = fields.iter().all(|f| f.0.is_zst());
             uninhabited && is_zst
         };
         let (present_first, present_second) = {
@@ -335,7 +331,7 @@ pub trait LayoutCalculator {
             }
             // If it's a struct, still compute a layout so that we can still compute the
             // field offsets.
-            None => V::new(0),
+            None => VariantIdx::new(0),
         };
 
         let is_struct = !is_enum ||
@@ -439,12 +435,12 @@ pub trait LayoutCalculator {
         // variant layouts, so we can't store them in the
         // overall LayoutS. Store the overall LayoutS
         // and the variant LayoutSs here until then.
-        struct TmpLayout<V: Idx> {
-            layout: LayoutS<V>,
-            variants: IndexVec<V, LayoutS<V>>,
+        struct TmpLayout {
+            layout: LayoutS,
+            variants: IndexVec<VariantIdx, LayoutS>,
         }
 
-        let calculate_niche_filling_layout = || -> Option<TmpLayout<V>> {
+        let calculate_niche_filling_layout = || -> Option<TmpLayout> {
             if niche_optimize_enum {
                 return None;
             }
@@ -464,15 +460,16 @@ pub trait LayoutCalculator {
 
                     Some(st)
                 })
-                .collect::<Option<IndexVec<V, _>>>()?;
+                .collect::<Option<IndexVec<VariantIdx, _>>>()?;
 
             let largest_variant_index = variant_layouts
                 .iter_enumerated()
                 .max_by_key(|(_i, layout)| layout.size.bytes())
                 .map(|(i, _layout)| i)?;
 
-            let all_indices = (0..=variants.len() - 1).map(V::new);
-            let needs_disc = |index: V| index != largest_variant_index && !absent(&variants[index]);
+            let all_indices = (0..=variants.len() - 1).map(VariantIdx::new);
+            let needs_disc =
+                |index: VariantIdx| index != largest_variant_index && !absent(&variants[index]);
             let niche_variants = all_indices.clone().find(|v| needs_disc(*v)).unwrap().index()
                 ..=all_indices.rev().find(|v| needs_disc(*v)).unwrap().index();
 
@@ -482,7 +479,7 @@ pub trait LayoutCalculator {
             let (field_index, niche, (niche_start, niche_scalar)) = variants[largest_variant_index]
                 .iter()
                 .enumerate()
-                .filter_map(|(j, field)| Some((j, field.largest_niche?)))
+                .filter_map(|(j, field)| Some((j, field.largest_niche()?)))
                 .max_by_key(|(_, niche)| niche.available(dl))
                 .and_then(|(j, niche)| Some((j, niche, niche.reserve(dl, count)?)))?;
             let niche_offset =
@@ -514,7 +511,7 @@ pub trait LayoutCalculator {
                 match layout.fields {
                     FieldsShape::Arbitrary { ref mut offsets, .. } => {
                         for (j, offset) in offsets.iter_mut().enumerate() {
-                            if !variants[i][j].is_zst() {
+                            if !variants[i][j].0.is_zst() {
                                 *offset += this_offset;
                             }
                         }
@@ -572,8 +569,8 @@ pub trait LayoutCalculator {
                     tag: niche_scalar,
                     tag_encoding: TagEncoding::Niche {
                         untagged_variant: largest_variant_index,
-                        niche_variants: (V::new(*niche_variants.start())
-                            ..=V::new(*niche_variants.end())),
+                        niche_variants: (VariantIdx::new(*niche_variants.start())
+                            ..=VariantIdx::new(*niche_variants.end())),
                         niche_start,
                     },
                     tag_field: 0,
@@ -598,7 +595,7 @@ pub trait LayoutCalculator {
         let discr_type = repr.discr_type();
         let bits = Integer::from_attr(dl, discr_type).size().bits();
         for (i, mut val) in discriminants {
-            if variants[i].iter().any(|f| f.abi.is_uninhabited()) {
+            if variants[i].iter().any(|f| f.abi().is_uninhabited()) {
                 continue;
             }
             if discr_type.is_signed() {
@@ -636,7 +633,7 @@ pub trait LayoutCalculator {
         if repr.c() {
             for fields in variants {
                 for field in fields {
-                    prefix_align = prefix_align.max(field.align.abi);
+                    prefix_align = prefix_align.max(field.align().abi);
                 }
             }
         }
@@ -655,8 +652,8 @@ pub trait LayoutCalculator {
                 // Find the first field we can't move later
                 // to make room for a larger discriminant.
                 for field in st.fields.index_by_increasing_offset().map(|j| &field_layouts[j]) {
-                    if !field.is_zst() || field.align.abi.bytes() != 1 {
-                        start_align = start_align.min(field.align.abi);
+                    if !field.0.is_zst() || field.align().abi.bytes() != 1 {
+                        start_align = start_align.min(field.align().abi);
                         break;
                     }
                 }
@@ -664,7 +661,7 @@ pub trait LayoutCalculator {
                 align = align.max(st.align);
                 Some(st)
             })
-            .collect::<Option<IndexVec<V, _>>>()?;
+            .collect::<Option<IndexVec<VariantIdx, _>>>()?;
 
         // Align the maximum variant size to the largest alignment.
         size = size.align_to(align.abi);
@@ -759,7 +756,7 @@ pub trait LayoutCalculator {
                 let FieldsShape::Arbitrary { ref offsets, .. } = layout_variant.fields else {
                     panic!();
                 };
-                let mut fields = iter::zip(field_layouts, offsets).filter(|p| !p.0.is_zst());
+                let mut fields = iter::zip(field_layouts, offsets).filter(|p| !p.0.0.is_zst());
                 let (field, offset) = match (fields.next(), fields.next()) {
                     (None, None) => {
                         common_prim_initialized_in_all_variants = false;
@@ -771,7 +768,7 @@ pub trait LayoutCalculator {
                         break;
                     }
                 };
-                let prim = match field.abi {
+                let prim = match field.abi() {
                     Abi::Scalar(scalar) => {
                         common_prim_initialized_in_all_variants &=
                             matches!(scalar, Scalar::Initialized { .. });
@@ -802,7 +799,7 @@ pub trait LayoutCalculator {
                     // Common prim might be uninit.
                     Scalar::Union { value: prim }
                 };
-                let pair = self.scalar_pair::<V>(tag, prim_scalar);
+                let pair = self.scalar_pair(tag, prim_scalar);
                 let pair_offsets = match pair.fields {
                     FieldsShape::Arbitrary { ref offsets, ref memory_index } => {
                         assert_eq!(memory_index, &[0, 1]);
@@ -862,9 +859,8 @@ pub trait LayoutCalculator {
                 // pick the layout with the larger niche; otherwise,
                 // pick tagged as it has simpler codegen.
                 use cmp::Ordering::*;
-                let niche_size = |tmp_l: &TmpLayout<V>| {
-                    tmp_l.layout.largest_niche.map_or(0, |n| n.available(dl))
-                };
+                let niche_size =
+                    |tmp_l: &TmpLayout| tmp_l.layout.largest_niche.map_or(0, |n| n.available(dl));
                 match (tl.layout.size.cmp(&nl.layout.size), niche_size(&tl).cmp(&niche_size(&nl))) {
                     (Greater, _) => nl,
                     (Equal, Less) => nl,
@@ -884,11 +880,11 @@ pub trait LayoutCalculator {
         Some(best_layout.layout)
     }
 
-    fn layout_of_union<'a, V: Idx, F: Deref<Target = &'a LayoutS<V>> + Debug>(
+    fn layout_of_union(
         &self,
         repr: &ReprOptions,
-        variants: &IndexVec<V, Vec<F>>,
-    ) -> Option<LayoutS<V>> {
+        variants: &IndexVec<VariantIdx, Vec<Layout<'_>>>,
+    ) -> Option<LayoutS> {
         let dl = self.current_data_layout();
         let dl = dl.borrow();
         let mut align = if repr.pack.is_some() { dl.i8_align } else { dl.aggregate_align };
@@ -900,15 +896,15 @@ pub trait LayoutCalculator {
         let optimize = !repr.inhibit_union_abi_opt();
         let mut size = Size::ZERO;
         let mut abi = Abi::Aggregate { sized: true };
-        let index = V::new(0);
+        let index = VariantIdx::new(0);
         for field in &variants[index] {
-            assert!(field.is_sized());
-            align = align.max(field.align);
+            assert!(field.0.is_sized());
+            align = align.max(field.align());
 
             // If all non-ZST fields have the same ABI, forward this ABI
-            if optimize && !field.is_zst() {
+            if optimize && !field.0.is_zst() {
                 // Discard valid range information and allow undef
-                let field_abi = match field.abi {
+                let field_abi = match field.abi() {
                     Abi::Scalar(x) => Abi::Scalar(x.to_union()),
                     Abi::ScalarPair(x, y) => Abi::ScalarPair(x.to_union(), y.to_union()),
                     Abi::Vector { element: x, count } => {
@@ -926,7 +922,7 @@ pub trait LayoutCalculator {
                 }
             }
 
-            size = cmp::max(size, field.size);
+            size = cmp::max(size, field.size());
         }
 
         if let Some(pack) = repr.pack {
diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs
index f4cb459f3..39574ca55 100644
--- a/compiler/rustc_abi/src/lib.rs
+++ b/compiler/rustc_abi/src/lib.rs
@@ -8,6 +8,7 @@ use std::ops::{Add, AddAssign, Mul, RangeInclusive, Sub};
 use std::str::FromStr;
 
 use bitflags::bitflags;
+use rustc_data_structures::intern::Interned;
 #[cfg(feature = "nightly")]
 use rustc_data_structures::stable_hasher::StableOrd;
 use rustc_index::vec::{Idx, IndexVec};
@@ -170,7 +171,9 @@ pub struct TargetDataLayout {
 
     pub instruction_address_space: AddressSpace,
 
-    /// Minimum size of #[repr(C)] enums (default I32 bits)
+    /// Minimum size of #[repr(C)] enums (default c_int::BITS, usually 32)
+    /// Note: This isn't in LLVM's data layout string, it is `short_enum`
+    /// so the only valid spec for LLVM is c_int::BITS or 8
     pub c_enum_min_size: Integer,
 }
 
@@ -267,6 +270,9 @@ impl TargetDataLayout {
                 ["a", ref a @ ..] => dl.aggregate_align = align(a, "a")?,
                 ["f32", ref a @ ..] => dl.f32_align = align(a, "f32")?,
                 ["f64", ref a @ ..] => dl.f64_align = align(a, "f64")?,
+                // FIXME(erikdesjardins): we should be parsing nonzero address spaces
+                // this will require replacing TargetDataLayout::{pointer_size,pointer_align}
+                // with e.g. `fn pointer_size_in(AddressSpace)`
                 [p @ "p", s, ref a @ ..] | [p @ "p0", s, ref a @ ..] => {
                     dl.pointer_size = size(s, p)?;
                     dl.pointer_align = align(a, p)?;
@@ -861,7 +867,7 @@ pub enum Primitive {
     Int(Integer, bool),
     F32,
     F64,
-    Pointer,
+    Pointer(AddressSpace),
 }
 
 impl Primitive {
@@ -872,7 +878,10 @@ impl Primitive {
             Int(i, _) => i.size(),
             F32 => Size::from_bits(32),
             F64 => Size::from_bits(64),
-            Pointer => dl.pointer_size,
+            // FIXME(erikdesjardins): ignoring address space is technically wrong, pointers in
+            // different address spaces can have different sizes
+            // (but TargetDataLayout doesn't currently parse that part of the DL string)
+            Pointer(_) => dl.pointer_size,
         }
     }
 
@@ -883,26 +892,12 @@ impl Primitive {
             Int(i, _) => i.align(dl),
             F32 => dl.f32_align,
             F64 => dl.f64_align,
-            Pointer => dl.pointer_align,
+            // FIXME(erikdesjardins): ignoring address space is technically wrong, pointers in
+            // different address spaces can have different alignments
+            // (but TargetDataLayout doesn't currently parse that part of the DL string)
+            Pointer(_) => dl.pointer_align,
         }
     }
-
-    // FIXME(eddyb) remove, it's trivial thanks to `matches!`.
-    #[inline]
-    pub fn is_float(self) -> bool {
-        matches!(self, F32 | F64)
-    }
-
-    // FIXME(eddyb) remove, it's completely unused.
-    #[inline]
-    pub fn is_int(self) -> bool {
-        matches!(self, Int(..))
-    }
-
-    #[inline]
-    pub fn is_ptr(self) -> bool {
-        matches!(self, Pointer)
-    }
 }
 
 /// Inclusive wrap-around range of valid values, that is, if
@@ -1188,7 +1183,8 @@ impl FieldsShape {
 /// An identifier that specifies the address space that some operation
 /// should operate on. Special address spaces have an effect on code generation,
 /// depending on the target and the address spaces it implements.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[cfg_attr(feature = "nightly", derive(HashStable_Generic))]
 pub struct AddressSpace(pub u32);
 
 impl AddressSpace {
@@ -1257,9 +1253,9 @@ impl Abi {
 
 #[derive(PartialEq, Eq, Hash, Clone, Debug)]
 #[cfg_attr(feature = "nightly", derive(HashStable_Generic))]
-pub enum Variants<V: Idx> {
+pub enum Variants {
     /// Single enum variants, structs/tuples, unions, and all non-ADTs.
-    Single { index: V },
+    Single { index: VariantIdx },
 
     /// Enum-likes with more than one inhabited variant: each variant comes with
     /// a *discriminant* (usually the same as the variant index but the user can
@@ -1269,15 +1265,15 @@ pub enum Variants<V: Idx> {
     /// For enums, the tag is the sole field of the layout.
     Multiple {
         tag: Scalar,
-        tag_encoding: TagEncoding<V>,
+        tag_encoding: TagEncoding,
         tag_field: usize,
-        variants: IndexVec<V, LayoutS<V>>,
+        variants: IndexVec<VariantIdx, LayoutS>,
     },
 }
 
 #[derive(PartialEq, Eq, Hash, Clone, Debug)]
 #[cfg_attr(feature = "nightly", derive(HashStable_Generic))]
-pub enum TagEncoding<V: Idx> {
+pub enum TagEncoding {
     /// The tag directly stores the discriminant, but possibly with a smaller layout
     /// (so converting the tag to the discriminant can require sign extension).
     Direct,
@@ -1292,7 +1288,11 @@ pub enum TagEncoding<V: Idx> {
     /// For example, `Option<(usize, &T)>`  is represented such that
     /// `None` has a null pointer for the second tuple field, and
     /// `Some` is the identity function (with a non-null reference).
-    Niche { untagged_variant: V, niche_variants: RangeInclusive<V>, niche_start: u128 },
+    Niche {
+        untagged_variant: VariantIdx,
+        niche_variants: RangeInclusive<VariantIdx>,
+        niche_start: u128,
+    },
 }
 
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
@@ -1379,9 +1379,14 @@ impl Niche {
     }
 }
 
+rustc_index::newtype_index! {
+    #[derive(HashStable_Generic)]
+    pub struct VariantIdx {}
+}
+
 #[derive(PartialEq, Eq, Hash, Clone)]
 #[cfg_attr(feature = "nightly", derive(HashStable_Generic))]
-pub struct LayoutS<V: Idx> {
+pub struct LayoutS {
     /// Says where the fields are located within the layout.
     pub fields: FieldsShape,
 
@@ -1392,7 +1397,7 @@ pub struct LayoutS<V: Idx> {
     ///
     /// To access all fields of this layout, both `fields` and the fields of the active variant
     /// must be taken into account.
-    pub variants: Variants<V>,
+    pub variants: Variants,
 
     /// The `abi` defines how this data is passed between functions, and it defines
     /// value restrictions via `valid_range`.
@@ -1411,13 +1416,13 @@ pub struct LayoutS<V: Idx> {
     pub size: Size,
 }
 
-impl<V: Idx> LayoutS<V> {
+impl LayoutS {
     pub fn scalar<C: HasDataLayout>(cx: &C, scalar: Scalar) -> Self {
         let largest_niche = Niche::from_scalar(cx, Size::ZERO, scalar);
         let size = scalar.size(cx);
         let align = scalar.align(cx);
         LayoutS {
-            variants: Variants::Single { index: V::new(0) },
+            variants: Variants::Single { index: VariantIdx::new(0) },
             fields: FieldsShape::Primitive,
             abi: Abi::Scalar(scalar),
             largest_niche,
@@ -1427,7 +1432,7 @@ impl<V: Idx> LayoutS<V> {
     }
 }
 
-impl<V: Idx> fmt::Debug for LayoutS<V> {
+impl fmt::Debug for LayoutS {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         // This is how `Layout` used to print before it become
         // `Interned<LayoutS>`. We print it like this to avoid having to update
@@ -1444,42 +1449,63 @@ impl<V: Idx> fmt::Debug for LayoutS<V> {
     }
 }
 
-#[derive(Copy, Clone, PartialEq, Eq, Debug)]
-pub enum PointerKind {
-    /// Most general case, we know no restrictions to tell LLVM.
-    SharedMutable,
+#[derive(Copy, Clone, PartialEq, Eq, Hash, HashStable_Generic)]
+#[rustc_pass_by_value]
+pub struct Layout<'a>(pub Interned<'a, LayoutS>);
+
+impl<'a> fmt::Debug for Layout<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // See comment on `<LayoutS as Debug>::fmt` above.
+        self.0.0.fmt(f)
+    }
+}
 
-    /// `&T` where `T` contains no `UnsafeCell`, is `dereferenceable`, `noalias` and `readonly`.
-    Frozen,
+impl<'a> Layout<'a> {
+    pub fn fields(self) -> &'a FieldsShape {
+        &self.0.0.fields
+    }
 
-    /// `&mut T` which is `dereferenceable` and `noalias` but not `readonly`.
-    UniqueBorrowed,
+    pub fn variants(self) -> &'a Variants {
+        &self.0.0.variants
+    }
 
-    /// `&mut !Unpin`, which is `dereferenceable` but neither `noalias` nor `readonly`.
-    UniqueBorrowedPinned,
+    pub fn abi(self) -> Abi {
+        self.0.0.abi
+    }
+
+    pub fn largest_niche(self) -> Option<Niche> {
+        self.0.0.largest_niche
+    }
+
+    pub fn align(self) -> AbiAndPrefAlign {
+        self.0.0.align
+    }
 
-    /// `Box<T>`, which is `noalias` (even on return types, unlike the above) but neither `readonly`
-    /// nor `dereferenceable`.
-    UniqueOwned,
+    pub fn size(self) -> Size {
+        self.0.0.size
+    }
+}
+
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+pub enum PointerKind {
+    /// Shared reference. `frozen` indicates the absence of any `UnsafeCell`.
+    SharedRef { frozen: bool },
+    /// Mutable reference. `unpin` indicates the absence of any pinned data.
+    MutableRef { unpin: bool },
+    /// Box. `unpin` indicates the absence of any pinned data.
+    Box { unpin: bool },
 }
 
+/// Note that this information is advisory only, and backends are free to ignore it.
+/// It can only be used to encode potential optimizations, but no critical information.
 #[derive(Copy, Clone, Debug)]
 pub struct PointeeInfo {
     pub size: Size,
     pub align: Align,
     pub safe: Option<PointerKind>,
-    pub address_space: AddressSpace,
-}
-
-/// Used in `might_permit_raw_init` to indicate the kind of initialisation
-/// that is checked to be valid
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-pub enum InitKind {
-    Zero,
-    UninitMitigated0x01Fill,
 }
 
-impl<V: Idx> LayoutS<V> {
+impl LayoutS {
     /// Returns `true` if the layout corresponds to an unsized type.
     pub fn is_unsized(&self) -> bool {
         self.abi.is_unsized()
diff --git a/compiler/rustc_ast/Cargo.toml b/compiler/rustc_ast/Cargo.toml
index 9253b7e68..f0632ac92 100644
--- a/compiler/rustc_ast/Cargo.toml
+++ b/compiler/rustc_ast/Cargo.toml
@@ -7,6 +7,7 @@ edition = "2021"
 
 [dependencies]
 bitflags = "1.2.1"
+memchr = "2.5.0"
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_index = { path = "../rustc_index" }
 rustc_lexer = { path = "../rustc_lexer" }
@@ -14,5 +15,5 @@ rustc_macros = { path = "../rustc_macros" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_span = { path = "../rustc_span" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-thin-vec = "0.2.9"
+thin-vec = "0.2.12"
 tracing = "0.1"
diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
index 9317579f7..03c375c46 100644
--- a/compiler/rustc_ast/src/ast.rs
+++ b/compiler/rustc_ast/src/ast.rs
@@ -18,6 +18,7 @@
 //! - [`Attribute`]: Metadata associated with item.
 //! - [`UnOp`], [`BinOp`], and [`BinOpKind`]: Unary and binary operators.
 
+pub use crate::format::*;
 pub use crate::util::parser::ExprPrecedence;
 pub use GenericArgs::*;
 pub use UnsafeSource::*;
@@ -208,7 +209,7 @@ pub struct AngleBracketedArgs {
     /// The overall span.
     pub span: Span,
     /// The comma separated parts in the `<...>`.
-    pub args: Vec<AngleBracketedArg>,
+    pub args: ThinVec<AngleBracketedArg>,
 }
 
 /// Either an argument for a parameter e.g., `'a`, `Vec<u8>`, `0`,
@@ -252,7 +253,7 @@ pub struct ParenthesizedArgs {
     pub span: Span,
 
     /// `(A, B)`
-    pub inputs: Vec<P<Ty>>,
+    pub inputs: ThinVec<P<Ty>>,
 
     /// ```text
     /// Foo(A, B) -> C
@@ -383,7 +384,7 @@ impl GenericParam {
 /// a function, enum, trait, etc.
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub struct Generics {
-    pub params: Vec<GenericParam>,
+    pub params: ThinVec<GenericParam>,
     pub where_clause: WhereClause,
     pub span: Span,
 }
@@ -391,7 +392,7 @@ pub struct Generics {
 impl Default for Generics {
     /// Creates an instance of `Generics`.
     fn default() -> Generics {
-        Generics { params: Vec::new(), where_clause: Default::default(), span: DUMMY_SP }
+        Generics { params: ThinVec::new(), where_clause: Default::default(), span: DUMMY_SP }
     }
 }
 
@@ -402,13 +403,13 @@ pub struct WhereClause {
     /// if we parsed no predicates (e.g. `struct Foo where {}`).
     /// This allows us to pretty-print accurately.
     pub has_where_token: bool,
-    pub predicates: Vec<WherePredicate>,
+    pub predicates: ThinVec<WherePredicate>,
     pub span: Span,
 }
 
 impl Default for WhereClause {
     fn default() -> WhereClause {
-        WhereClause { has_where_token: false, predicates: Vec::new(), span: DUMMY_SP }
+        WhereClause { has_where_token: false, predicates: ThinVec::new(), span: DUMMY_SP }
     }
 }
 
@@ -440,7 +441,7 @@ impl WherePredicate {
 pub struct WhereBoundPredicate {
     pub span: Span,
     /// Any generics from a `for` binding.
-    pub bound_generic_params: Vec<GenericParam>,
+    pub bound_generic_params: ThinVec<GenericParam>,
     /// The type being bounded.
     pub bounded_ty: P<Ty>,
     /// Trait and lifetime bounds (`Clone + Send + 'static`).
@@ -470,7 +471,7 @@ pub struct WhereEqPredicate {
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub struct Crate {
     pub attrs: AttrVec,
-    pub items: Vec<P<Item>>,
+    pub items: ThinVec<P<Item>>,
     pub spans: ModSpans,
     /// Must be equal to `CRATE_NODE_ID` after the crate root is expanded, but may hold
     /// expansion placeholders or an unassigned value (`DUMMY_NODE_ID`) before that.
@@ -502,7 +503,7 @@ pub enum MetaItemKind {
     /// List meta item.
     ///
     /// E.g., `#[derive(..)]`, where the field represents the `..`.
-    List(Vec<NestedMetaItem>),
+    List(ThinVec<NestedMetaItem>),
 
     /// Name value meta item.
     ///
@@ -530,7 +531,7 @@ pub enum NestedMetaItem {
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub struct Block {
     /// The statements in the block.
-    pub stmts: Vec<Stmt>,
+    pub stmts: ThinVec<Stmt>,
     pub id: NodeId,
     /// Distinguishes between `unsafe { ... }` and `{ ... }`.
     pub rules: BlockCheckMode,
@@ -580,7 +581,7 @@ impl Pat {
             // A tuple pattern `(P0, .., Pn)` can be reparsed as `(T0, .., Tn)`
             // assuming `T0` to `Tn` are all syntactically valid as types.
             PatKind::Tuple(pats) => {
-                let mut tys = Vec::with_capacity(pats.len());
+                let mut tys = ThinVec::with_capacity(pats.len());
                 // FIXME(#48994) - could just be collected into an Option<Vec>
                 for pat in pats {
                     tys.push(pat.to_ty()?);
@@ -721,14 +722,14 @@ pub enum PatKind {
 
     /// A struct or struct variant pattern (e.g., `Variant {x, y, ..}`).
     /// The `bool` is `true` in the presence of a `..`.
-    Struct(Option<P<QSelf>>, Path, Vec<PatField>, /* recovered */ bool),
+    Struct(Option<P<QSelf>>, Path, ThinVec<PatField>, /* recovered */ bool),
 
     /// A tuple struct/variant pattern (`Variant(x, y, .., z)`).
-    TupleStruct(Option<P<QSelf>>, Path, Vec<P<Pat>>),
+    TupleStruct(Option<P<QSelf>>, Path, ThinVec<P<Pat>>),
 
     /// An or-pattern `A | B | C`.
     /// Invariant: `pats.len() >= 2`.
-    Or(Vec<P<Pat>>),
+    Or(ThinVec<P<Pat>>),
 
     /// A possibly qualified path pattern.
     /// Unqualified path patterns `A::B::C` can legally refer to variants, structs, constants
@@ -737,7 +738,7 @@ pub enum PatKind {
     Path(Option<P<QSelf>>, Path),
 
     /// A tuple pattern (`(a, b)`).
-    Tuple(Vec<P<Pat>>),
+    Tuple(ThinVec<P<Pat>>),
 
     /// A `box` pattern.
     Box(P<Pat>),
@@ -752,7 +753,7 @@ pub enum PatKind {
     Range(Option<P<Expr>>, Option<P<Expr>>, Spanned<RangeEnd>),
 
     /// A slice pattern `[a, b, c]`.
-    Slice(Vec<P<Pat>>),
+    Slice(ThinVec<P<Pat>>),
 
     /// A rest pattern `..`.
     ///
@@ -1168,7 +1169,7 @@ impl Expr {
     pub fn to_bound(&self) -> Option<GenericBound> {
         match &self.kind {
             ExprKind::Path(None, path) => Some(GenericBound::Trait(
-                PolyTraitRef::new(Vec::new(), path.clone(), self.span),
+                PolyTraitRef::new(ThinVec::new(), path.clone(), self.span),
                 TraitBoundModifier::None,
             )),
             _ => None,
@@ -1203,7 +1204,7 @@ impl Expr {
             ExprKind::Array(exprs) if exprs.len() == 1 => exprs[0].to_ty().map(TyKind::Slice)?,
 
             ExprKind::Tup(exprs) => {
-                let tys = exprs.iter().map(|expr| expr.to_ty()).collect::<Option<Vec<_>>>()?;
+                let tys = exprs.iter().map(|expr| expr.to_ty()).collect::<Option<ThinVec<_>>>()?;
                 TyKind::Tup(tys)
             }
 
@@ -1269,6 +1270,7 @@ impl Expr {
             ExprKind::Try(..) => ExprPrecedence::Try,
             ExprKind::Yield(..) => ExprPrecedence::Yield,
             ExprKind::Yeet(..) => ExprPrecedence::Yeet,
+            ExprKind::FormatArgs(..) => ExprPrecedence::FormatArgs,
             ExprKind::Err => ExprPrecedence::Err,
         }
     }
@@ -1335,7 +1337,7 @@ pub struct MethodCall {
     /// The receiver, e.g. `x`.
     pub receiver: P<Expr>,
     /// The arguments, e.g. `a, b, c`.
-    pub args: Vec<P<Expr>>,
+    pub args: ThinVec<P<Expr>>,
     /// The span of the function, without the dot and receiver e.g. `foo::<Bar,
     /// Baz>(a, b, c)`.
     pub span: Span,
@@ -1355,7 +1357,7 @@ pub enum StructRest {
 pub struct StructExpr {
     pub qself: Option<P<QSelf>>,
     pub path: Path,
-    pub fields: Vec<ExprField>,
+    pub fields: ThinVec<ExprField>,
     pub rest: StructRest,
 }
 
@@ -1364,7 +1366,7 @@ pub enum ExprKind {
     /// A `box x` expression.
     Box(P<Expr>),
     /// An array (`[a, b, c, d]`)
-    Array(Vec<P<Expr>>),
+    Array(ThinVec<P<Expr>>),
     /// Allow anonymous constants from an inline `const` block
     ConstBlock(AnonConst),
     /// A function call
@@ -1373,11 +1375,11 @@ pub enum ExprKind {
     /// and the second field is the list of arguments.
     /// This also represents calling the constructor of
     /// tuple-like ADTs such as tuple structs and enum variants.
-    Call(P<Expr>, Vec<P<Expr>>),
+    Call(P<Expr>, ThinVec<P<Expr>>),
     /// A method call (e.g. `x.foo::<Bar, Baz>(a, b, c)`).
     MethodCall(Box<MethodCall>),
     /// A tuple (e.g., `(a, b, c, d)`).
-    Tup(Vec<P<Expr>>),
+    Tup(ThinVec<P<Expr>>),
     /// A binary operation (e.g., `a + b`, `a * b`).
     Binary(BinOp, P<Expr>, P<Expr>),
     /// A unary operation (e.g., `!x`, `*x`).
@@ -1412,7 +1414,7 @@ pub enum ExprKind {
     /// `'label: loop { block }`
     Loop(P<Block>, Option<Label>, Span),
     /// A `match` block.
-    Match(P<Expr>, Vec<Arm>),
+    Match(P<Expr>, ThinVec<Arm>),
     /// A closure (e.g., `move |a, b, c| a + b + c`).
     Closure(Box<Closure>),
     /// A block (`'label: { ... }`).
@@ -1499,6 +1501,9 @@ pub enum ExprKind {
     /// with a `ByteStr` literal.
     IncludedBytes(Lrc<[u8]>),
 
+    /// A `format_args!()` expression.
+    FormatArgs(P<FormatArgs>),
+
     /// Placeholder for an expression that wasn't syntactically well formed in some way.
     Err,
 }
@@ -1569,7 +1574,7 @@ pub enum ClosureBinder {
         /// for<'a, 'b> |_: &'a (), _: &'b ()| { ... }
         ///     ^^^^^^ -- this
         /// ```
-        generic_params: P<[GenericParam]>,
+        generic_params: ThinVec<GenericParam>,
     },
 }
 
@@ -1821,6 +1826,13 @@ pub enum LitKind {
 }
 
 impl LitKind {
+    pub fn str(&self) -> Option<Symbol> {
+        match *self {
+            LitKind::Str(s, _) => Some(s),
+            _ => None,
+        }
+    }
+
     /// Returns `true` if this literal is a string.
     pub fn is_str(&self) -> bool {
         matches!(self, LitKind::Str(..))
@@ -2044,7 +2056,7 @@ impl Ty {
 pub struct BareFnTy {
     pub unsafety: Unsafe,
     pub ext: Extern,
-    pub generic_params: Vec<GenericParam>,
+    pub generic_params: ThinVec<GenericParam>,
     pub decl: P<FnDecl>,
     /// Span of the `fn(...) -> ...` part.
     pub decl_span: Span,
@@ -2066,7 +2078,7 @@ pub enum TyKind {
     /// The never type (`!`).
     Never,
     /// A tuple (`(A, B, C, D,...)`).
-    Tup(Vec<P<Ty>>),
+    Tup(ThinVec<P<Ty>>),
     /// A path (`module::module::...::Type`), optionally
     /// "qualified", e.g., `<Vec<T> as SomeTrait>::SomeType`.
     ///
@@ -2351,7 +2363,7 @@ impl Param {
 /// which contains metadata about function safety, asyncness, constness and ABI.
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub struct FnDecl {
-    pub inputs: Vec<Param>,
+    pub inputs: ThinVec<Param>,
     pub output: FnRetTy,
 }
 
@@ -2463,7 +2475,7 @@ pub enum ModKind {
     /// or with definition outlined to a separate file `mod foo;` and already loaded from it.
     /// The inner span is from the first token past `{` to the last token until `}`,
     /// or from the first to the last token in the loaded file.
-    Loaded(Vec<P<Item>>, Inline, ModSpans),
+    Loaded(ThinVec<P<Item>>, Inline, ModSpans),
     /// Module with definition outlined to a separate file `mod foo;` but not yet loaded from it.
     Unloaded,
 }
@@ -2485,12 +2497,12 @@ pub struct ForeignMod {
     /// semantically by Rust.
     pub unsafety: Unsafe,
     pub abi: Option<StrLit>,
-    pub items: Vec<P<ForeignItem>>,
+    pub items: ThinVec<P<ForeignItem>>,
 }
 
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub struct EnumDef {
-    pub variants: Vec<Variant>,
+    pub variants: ThinVec<Variant>,
 }
 /// Enum variant.
 #[derive(Clone, Encodable, Decodable, Debug)]
@@ -2520,7 +2532,7 @@ pub enum UseTreeKind {
     /// `use prefix` or `use prefix as rename`
     Simple(Option<Ident>),
     /// `use prefix::{...}`
-    Nested(Vec<(UseTree, NodeId)>),
+    Nested(ThinVec<(UseTree, NodeId)>),
     /// `use prefix::*`
     Glob,
 }
@@ -2624,7 +2636,7 @@ pub struct TraitRef {
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub struct PolyTraitRef {
     /// The `'a` in `for<'a> Foo<&'a T>`.
-    pub bound_generic_params: Vec<GenericParam>,
+    pub bound_generic_params: ThinVec<GenericParam>,
 
     /// The `Foo<&'a T>` in `<'a> Foo<&'a T>`.
     pub trait_ref: TraitRef,
@@ -2633,7 +2645,7 @@ pub struct PolyTraitRef {
 }
 
 impl PolyTraitRef {
-    pub fn new(generic_params: Vec<GenericParam>, path: Path, span: Span) -> Self {
+    pub fn new(generic_params: ThinVec<GenericParam>, path: Path, span: Span) -> Self {
         PolyTraitRef {
             bound_generic_params: generic_params,
             trait_ref: TraitRef { path, ref_id: DUMMY_NODE_ID },
@@ -2683,11 +2695,11 @@ pub enum VariantData {
     /// Struct variant.
     ///
     /// E.g., `Bar { .. }` as in `enum Foo { Bar { .. } }`.
-    Struct(Vec<FieldDef>, bool),
+    Struct(ThinVec<FieldDef>, bool),
     /// Tuple variant.
     ///
     /// E.g., `Bar(..)` as in `enum Foo { Bar(..) }`.
-    Tuple(Vec<FieldDef>, NodeId),
+    Tuple(ThinVec<FieldDef>, NodeId),
     /// Unit variant.
     ///
     /// E.g., `Bar = ..` as in `enum Foo { Bar = .. }`.
@@ -2814,7 +2826,7 @@ pub struct Trait {
     pub is_auto: IsAuto,
     pub generics: Generics,
     pub bounds: GenericBounds,
-    pub items: Vec<P<AssocItem>>,
+    pub items: ThinVec<P<AssocItem>>,
 }
 
 /// The location of a where clause on a `TyAlias` (`Span`) and whether there was
@@ -2862,7 +2874,7 @@ pub struct Impl {
     /// The trait being implemented, if any.
     pub of_trait: Option<TraitRef>,
     pub self_ty: P<Ty>,
-    pub items: Vec<P<AssocItem>>,
+    pub items: ThinVec<P<AssocItem>>,
 }
 
 #[derive(Clone, Encodable, Decodable, Debug)]
@@ -3100,26 +3112,26 @@ mod size_asserts {
     static_assert_size!(AssocItem, 104);
     static_assert_size!(AssocItemKind, 32);
     static_assert_size!(Attribute, 32);
-    static_assert_size!(Block, 48);
+    static_assert_size!(Block, 32);
     static_assert_size!(Expr, 72);
     static_assert_size!(ExprKind, 40);
-    static_assert_size!(Fn, 184);
+    static_assert_size!(Fn, 152);
     static_assert_size!(ForeignItem, 96);
     static_assert_size!(ForeignItemKind, 24);
     static_assert_size!(GenericArg, 24);
-    static_assert_size!(GenericBound, 72);
-    static_assert_size!(Generics, 72);
-    static_assert_size!(Impl, 184);
-    static_assert_size!(Item, 184);
-    static_assert_size!(ItemKind, 112);
+    static_assert_size!(GenericBound, 56);
+    static_assert_size!(Generics, 40);
+    static_assert_size!(Impl, 136);
+    static_assert_size!(Item, 136);
+    static_assert_size!(ItemKind, 64);
     static_assert_size!(LitKind, 24);
     static_assert_size!(Local, 72);
     static_assert_size!(MetaItemLit, 40);
     static_assert_size!(Param, 40);
-    static_assert_size!(Pat, 88);
+    static_assert_size!(Pat, 72);
     static_assert_size!(Path, 24);
     static_assert_size!(PathSegment, 24);
-    static_assert_size!(PatKind, 64);
+    static_assert_size!(PatKind, 48);
     static_assert_size!(Stmt, 32);
     static_assert_size!(StmtKind, 16);
     static_assert_size!(Ty, 64);
diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs
index c6b6207b3..2e83b3e62 100644
--- a/compiler/rustc_ast/src/attr/mod.rs
+++ b/compiler/rustc_ast/src/attr/mod.rs
@@ -20,7 +20,7 @@ use std::iter;
 use std::ops::BitXor;
 #[cfg(debug_assertions)]
 use std::sync::atomic::{AtomicU32, Ordering};
-use thin_vec::thin_vec;
+use thin_vec::{thin_vec, ThinVec};
 
 pub struct MarkedAttrs(GrowableBitSet<AttrId>);
 
@@ -40,84 +40,65 @@ impl MarkedAttrs {
     }
 }
 
-impl NestedMetaItem {
-    /// Returns the `MetaItem` if `self` is a `NestedMetaItem::MetaItem`.
-    pub fn meta_item(&self) -> Option<&MetaItem> {
-        match self {
-            NestedMetaItem::MetaItem(item) => Some(item),
-            _ => None,
-        }
-    }
+pub struct AttrIdGenerator(WorkerLocal<Cell<u32>>);
 
-    /// Returns the `MetaItemLit` if `self` is a `NestedMetaItem::Literal`s.
-    pub fn lit(&self) -> Option<&MetaItemLit> {
-        match self {
-            NestedMetaItem::Lit(lit) => Some(lit),
-            _ => None,
-        }
-    }
+#[cfg(debug_assertions)]
+static MAX_ATTR_ID: AtomicU32 = AtomicU32::new(u32::MAX);
 
-    /// Returns `true` if this list item is a MetaItem with a name of `name`.
-    pub fn has_name(&self, name: Symbol) -> bool {
-        self.meta_item().map_or(false, |meta_item| meta_item.has_name(name))
-    }
+impl AttrIdGenerator {
+    pub fn new() -> Self {
+        // We use `(index as u32).reverse_bits()` to initialize the
+        // starting value of AttrId in each worker thread.
+        // The `index` is the index of the worker thread.
+        // This ensures that the AttrId generated in each thread is unique.
+        AttrIdGenerator(WorkerLocal::new(|index| {
+            let index: u32 = index.try_into().unwrap();
 
-    /// For a single-segment meta item, returns its name; otherwise, returns `None`.
-    pub fn ident(&self) -> Option<Ident> {
-        self.meta_item().and_then(|meta_item| meta_item.ident())
-    }
-    pub fn name_or_empty(&self) -> Symbol {
-        self.ident().unwrap_or_else(Ident::empty).name
-    }
+            #[cfg(debug_assertions)]
+            {
+                let max_id = ((index + 1).next_power_of_two() - 1).bitxor(u32::MAX).reverse_bits();
+                MAX_ATTR_ID.fetch_min(max_id, Ordering::Release);
+            }
 
-    /// Gets the string value if `self` is a `MetaItem` and the `MetaItem` is a
-    /// `MetaItemKind::NameValue` variant containing a string, otherwise `None`.
-    pub fn value_str(&self) -> Option<Symbol> {
-        self.meta_item().and_then(|meta_item| meta_item.value_str())
+            Cell::new(index.reverse_bits())
+        }))
     }
 
-    /// Returns a name and single literal value tuple of the `MetaItem`.
-    pub fn name_value_literal(&self) -> Option<(Symbol, &MetaItemLit)> {
-        self.meta_item().and_then(|meta_item| {
-            meta_item.meta_item_list().and_then(|meta_item_list| {
-                if meta_item_list.len() == 1
-                    && let Some(ident) = meta_item.ident()
-                    && let Some(lit) = meta_item_list[0].lit()
-                {
-                    return Some((ident.name, lit));
-                }
-                None
-            })
-        })
-    }
+    pub fn mk_attr_id(&self) -> AttrId {
+        let id = self.0.get();
 
-    /// Gets a list of inner meta items from a list `MetaItem` type.
-    pub fn meta_item_list(&self) -> Option<&[NestedMetaItem]> {
-        self.meta_item().and_then(|meta_item| meta_item.meta_item_list())
-    }
+        // Ensure the assigned attr_id does not overlap the bits
+        // representing the number of threads.
+        #[cfg(debug_assertions)]
+        assert!(id <= MAX_ATTR_ID.load(Ordering::Acquire));
 
-    /// Returns `true` if the variant is `MetaItem`.
-    pub fn is_meta_item(&self) -> bool {
-        self.meta_item().is_some()
+        self.0.set(id + 1);
+        AttrId::from_u32(id)
     }
+}
 
-    /// Returns `true` if `self` is a `MetaItem` and the meta item is a word.
-    pub fn is_word(&self) -> bool {
-        self.meta_item().map_or(false, |meta_item| meta_item.is_word())
+impl Attribute {
+    pub fn get_normal_item(&self) -> &AttrItem {
+        match &self.kind {
+            AttrKind::Normal(normal) => &normal.item,
+            AttrKind::DocComment(..) => panic!("unexpected doc comment"),
+        }
     }
 
-    /// See [`MetaItem::name_value_literal_span`].
-    pub fn name_value_literal_span(&self) -> Option<Span> {
-        self.meta_item()?.name_value_literal_span()
+    pub fn unwrap_normal_item(self) -> AttrItem {
+        match self.kind {
+            AttrKind::Normal(normal) => normal.into_inner().item,
+            AttrKind::DocComment(..) => panic!("unexpected doc comment"),
+        }
     }
-}
 
-impl Attribute {
-    #[inline]
-    pub fn has_name(&self, name: Symbol) -> bool {
-        match &self.kind {
-            AttrKind::Normal(normal) => normal.item.path == name,
-            AttrKind::DocComment(..) => false,
+    /// Returns `true` if it is a sugared doc comment (`///` or `//!` for example).
+    /// So `#[doc = "doc"]` (which is a doc comment) and `#[doc(...)]` (which is not
+    /// a doc comment) will return `false`.
+    pub fn is_doc_comment(&self) -> bool {
+        match self.kind {
+            AttrKind::Normal(..) => false,
+            AttrKind::DocComment(..) => true,
         }
     }
 
@@ -138,20 +119,11 @@ impl Attribute {
         self.ident().unwrap_or_else(Ident::empty).name
     }
 
-    pub fn value_str(&self) -> Option<Symbol> {
-        match &self.kind {
-            AttrKind::Normal(normal) => normal.item.meta_kind().and_then(|kind| kind.value_str()),
-            AttrKind::DocComment(..) => None,
-        }
-    }
-
-    pub fn meta_item_list(&self) -> Option<Vec<NestedMetaItem>> {
+    #[inline]
+    pub fn has_name(&self, name: Symbol) -> bool {
         match &self.kind {
-            AttrKind::Normal(normal) => match normal.item.meta_kind() {
-                Some(MetaItemKind::List(list)) => Some(list),
-                _ => None,
-            },
-            AttrKind::DocComment(..) => None,
+            AttrKind::Normal(normal) => normal.item.path == name,
+            AttrKind::DocComment(..) => false,
         }
     }
 
@@ -162,82 +134,18 @@ impl Attribute {
             false
         }
     }
-}
-
-impl MetaItem {
-    /// For a single-segment meta item, returns its name; otherwise, returns `None`.
-    pub fn ident(&self) -> Option<Ident> {
-        if self.path.segments.len() == 1 { Some(self.path.segments[0].ident) } else { None }
-    }
-    pub fn name_or_empty(&self) -> Symbol {
-        self.ident().unwrap_or_else(Ident::empty).name
-    }
 
-    /// ```text
-    /// Example:
-    ///     #[attribute(name = "value")]
-    ///                 ^^^^^^^^^^^^^^
-    /// ```
-    pub fn name_value_literal(&self) -> Option<&MetaItemLit> {
+    pub fn meta_item_list(&self) -> Option<ThinVec<NestedMetaItem>> {
         match &self.kind {
-            MetaItemKind::NameValue(v) => Some(v),
-            _ => None,
+            AttrKind::Normal(normal) => normal.item.meta_item_list(),
+            AttrKind::DocComment(..) => None,
         }
     }
 
     pub fn value_str(&self) -> Option<Symbol> {
-        self.kind.value_str()
-    }
-
-    pub fn meta_item_list(&self) -> Option<&[NestedMetaItem]> {
         match &self.kind {
-            MetaItemKind::List(l) => Some(&**l),
-            _ => None,
-        }
-    }
-
-    pub fn is_word(&self) -> bool {
-        matches!(self.kind, MetaItemKind::Word)
-    }
-
-    pub fn has_name(&self, name: Symbol) -> bool {
-        self.path == name
-    }
-
-    /// This is used in case you want the value span instead of the whole attribute. Example:
-    ///
-    /// ```text
-    /// #[doc(alias = "foo")]
-    /// ```
-    ///
-    /// In here, it'll return a span for `"foo"`.
-    pub fn name_value_literal_span(&self) -> Option<Span> {
-        Some(self.name_value_literal()?.span)
-    }
-}
-
-impl AttrItem {
-    pub fn span(&self) -> Span {
-        self.args.span().map_or(self.path.span, |args_span| self.path.span.to(args_span))
-    }
-
-    pub fn meta(&self, span: Span) -> Option<MetaItem> {
-        Some(MetaItem { path: self.path.clone(), kind: self.meta_kind()?, span })
-    }
-
-    pub fn meta_kind(&self) -> Option<MetaItemKind> {
-        MetaItemKind::from_attr_args(&self.args)
-    }
-}
-
-impl Attribute {
-    /// Returns `true` if it is a sugared doc comment (`///` or `//!` for example).
-    /// So `#[doc = "doc"]` (which is a doc comment) and `#[doc(...)]` (which is not
-    /// a doc comment) will return `false`.
-    pub fn is_doc_comment(&self) -> bool {
-        match self.kind {
-            AttrKind::Normal(..) => false,
-            AttrKind::DocComment(..) => true,
+            AttrKind::Normal(normal) => normal.item.value_str(),
+            AttrKind::DocComment(..) => None,
         }
     }
 
@@ -247,13 +155,11 @@ impl Attribute {
     /// * `#[doc = "doc"]` returns `Some(("doc", CommentKind::Line))`.
     /// * `#[doc(...)]` returns `None`.
     pub fn doc_str_and_comment_kind(&self) -> Option<(Symbol, CommentKind)> {
-        match self.kind {
-            AttrKind::DocComment(kind, data) => Some((data, kind)),
-            AttrKind::Normal(ref normal) if normal.item.path == sym::doc => normal
-                .item
-                .meta_kind()
-                .and_then(|kind| kind.value_str())
-                .map(|data| (data, CommentKind::Line)),
+        match &self.kind {
+            AttrKind::DocComment(kind, data) => Some((*data, *kind)),
+            AttrKind::Normal(normal) if normal.item.path == sym::doc => {
+                normal.item.value_str().map(|s| (s, CommentKind::Line))
+            }
             _ => None,
         }
     }
@@ -265,9 +171,7 @@ impl Attribute {
     pub fn doc_str(&self) -> Option<Symbol> {
         match &self.kind {
             AttrKind::DocComment(.., data) => Some(*data),
-            AttrKind::Normal(normal) if normal.item.path == sym::doc => {
-                normal.item.meta_kind().and_then(|kind| kind.value_str())
-            }
+            AttrKind::Normal(normal) if normal.item.path == sym::doc => normal.item.value_str(),
             _ => None,
         }
     }
@@ -276,20 +180,6 @@ impl Attribute {
         self.doc_str().map_or(false, |s| comments::may_have_doc_links(s.as_str()))
     }
 
-    pub fn get_normal_item(&self) -> &AttrItem {
-        match &self.kind {
-            AttrKind::Normal(normal) => &normal.item,
-            AttrKind::DocComment(..) => panic!("unexpected doc comment"),
-        }
-    }
-
-    pub fn unwrap_normal_item(self) -> AttrItem {
-        match self.kind {
-            AttrKind::Normal(normal) => normal.into_inner().item,
-            AttrKind::DocComment(..) => panic!("unexpected doc comment"),
-        }
-    }
-
     /// Extracts the MetaItem from inside this Attribute.
     pub fn meta(&self) -> Option<MetaItem> {
         match &self.kind {
@@ -321,130 +211,102 @@ impl Attribute {
     }
 }
 
-pub struct AttrIdGenerator(WorkerLocal<Cell<u32>>);
-
-#[cfg(debug_assertions)]
-static MAX_ATTR_ID: AtomicU32 = AtomicU32::new(u32::MAX);
-
-impl AttrIdGenerator {
-    pub fn new() -> Self {
-        // We use `(index as u32).reverse_bits()` to initialize the
-        // starting value of AttrId in each worker thread.
-        // The `index` is the index of the worker thread.
-        // This ensures that the AttrId generated in each thread is unique.
-        AttrIdGenerator(WorkerLocal::new(|index| {
-            let index: u32 = index.try_into().unwrap();
+impl AttrItem {
+    pub fn span(&self) -> Span {
+        self.args.span().map_or(self.path.span, |args_span| self.path.span.to(args_span))
+    }
 
-            #[cfg(debug_assertions)]
-            {
-                let max_id = ((index + 1).next_power_of_two() - 1).bitxor(u32::MAX).reverse_bits();
-                MAX_ATTR_ID.fetch_min(max_id, Ordering::Release);
+    fn meta_item_list(&self) -> Option<ThinVec<NestedMetaItem>> {
+        match &self.args {
+            AttrArgs::Delimited(args) if args.delim == MacDelimiter::Parenthesis => {
+                MetaItemKind::list_from_tokens(args.tokens.clone())
             }
-
-            Cell::new(index.reverse_bits())
-        }))
+            AttrArgs::Delimited(_) | AttrArgs::Eq(..) | AttrArgs::Empty => None,
+        }
     }
 
-    pub fn mk_attr_id(&self) -> AttrId {
-        let id = self.0.get();
+    fn value_str(&self) -> Option<Symbol> {
+        match &self.args {
+            AttrArgs::Eq(_, args) => args.value_str(),
+            AttrArgs::Delimited(_) | AttrArgs::Empty => None,
+        }
+    }
 
-        // Ensure the assigned attr_id does not overlap the bits
-        // representing the number of threads.
-        #[cfg(debug_assertions)]
-        assert!(id <= MAX_ATTR_ID.load(Ordering::Acquire));
+    pub fn meta(&self, span: Span) -> Option<MetaItem> {
+        Some(MetaItem { path: self.path.clone(), kind: self.meta_kind()?, span })
+    }
 
-        self.0.set(id + 1);
-        AttrId::from_u32(id)
+    pub fn meta_kind(&self) -> Option<MetaItemKind> {
+        MetaItemKind::from_attr_args(&self.args)
     }
 }
 
-pub fn mk_attr(
-    g: &AttrIdGenerator,
-    style: AttrStyle,
-    path: Path,
-    args: AttrArgs,
-    span: Span,
-) -> Attribute {
-    mk_attr_from_item(g, AttrItem { path, args, tokens: None }, None, style, span)
+impl AttrArgsEq {
+    fn value_str(&self) -> Option<Symbol> {
+        match self {
+            AttrArgsEq::Ast(expr) => match expr.kind {
+                ExprKind::Lit(token_lit) => {
+                    LitKind::from_token_lit(token_lit).ok().and_then(|lit| lit.str())
+                }
+                _ => None,
+            },
+            AttrArgsEq::Hir(lit) => lit.kind.str(),
+        }
+    }
 }
 
-pub fn mk_attr_from_item(
-    g: &AttrIdGenerator,
-    item: AttrItem,
-    tokens: Option<LazyAttrTokenStream>,
-    style: AttrStyle,
-    span: Span,
-) -> Attribute {
-    Attribute {
-        kind: AttrKind::Normal(P(NormalAttr { item, tokens })),
-        id: g.mk_attr_id(),
-        style,
-        span,
+impl MetaItem {
+    /// For a single-segment meta item, returns its name; otherwise, returns `None`.
+    pub fn ident(&self) -> Option<Ident> {
+        if self.path.segments.len() == 1 { Some(self.path.segments[0].ident) } else { None }
     }
-}
 
-pub fn mk_attr_word(g: &AttrIdGenerator, style: AttrStyle, name: Symbol, span: Span) -> Attribute {
-    let path = Path::from_ident(Ident::new(name, span));
-    let args = AttrArgs::Empty;
-    mk_attr(g, style, path, args, span)
-}
+    pub fn name_or_empty(&self) -> Symbol {
+        self.ident().unwrap_or_else(Ident::empty).name
+    }
 
-pub fn mk_attr_name_value_str(
-    g: &AttrIdGenerator,
-    style: AttrStyle,
-    name: Symbol,
-    val: Symbol,
-    span: Span,
-) -> Attribute {
-    let lit = token::Lit::new(token::Str, escape_string_symbol(val), None);
-    let expr = P(Expr {
-        id: DUMMY_NODE_ID,
-        kind: ExprKind::Lit(lit),
-        span,
-        attrs: AttrVec::new(),
-        tokens: None,
-    });
-    let path = Path::from_ident(Ident::new(name, span));
-    let args = AttrArgs::Eq(span, AttrArgsEq::Ast(expr));
-    mk_attr(g, style, path, args, span)
-}
+    pub fn has_name(&self, name: Symbol) -> bool {
+        self.path == name
+    }
 
-pub fn mk_attr_nested_word(
-    g: &AttrIdGenerator,
-    style: AttrStyle,
-    outer: Symbol,
-    inner: Symbol,
-    span: Span,
-) -> Attribute {
-    let inner_tokens = TokenStream::new(vec![TokenTree::Token(
-        Token::from_ast_ident(Ident::new(inner, span)),
-        Spacing::Alone,
-    )]);
-    let outer_ident = Ident::new(outer, span);
-    let path = Path::from_ident(outer_ident);
-    let attr_args = AttrArgs::Delimited(DelimArgs {
-        dspan: DelimSpan::from_single(span),
-        delim: MacDelimiter::Parenthesis,
-        tokens: inner_tokens,
-    });
-    mk_attr(g, style, path, attr_args, span)
-}
+    pub fn is_word(&self) -> bool {
+        matches!(self.kind, MetaItemKind::Word)
+    }
 
-pub fn mk_doc_comment(
-    g: &AttrIdGenerator,
-    comment_kind: CommentKind,
-    style: AttrStyle,
-    data: Symbol,
-    span: Span,
-) -> Attribute {
-    Attribute { kind: AttrKind::DocComment(comment_kind, data), id: g.mk_attr_id(), style, span }
-}
+    pub fn meta_item_list(&self) -> Option<&[NestedMetaItem]> {
+        match &self.kind {
+            MetaItemKind::List(l) => Some(&**l),
+            _ => None,
+        }
+    }
 
-pub fn list_contains_name(items: &[NestedMetaItem], name: Symbol) -> bool {
-    items.iter().any(|item| item.has_name(name))
-}
+    /// ```text
+    /// Example:
+    ///     #[attribute(name = "value")]
+    ///                 ^^^^^^^^^^^^^^
+    /// ```
+    pub fn name_value_literal(&self) -> Option<&MetaItemLit> {
+        match &self.kind {
+            MetaItemKind::NameValue(v) => Some(v),
+            _ => None,
+        }
+    }
+
+    /// This is used in case you want the value span instead of the whole attribute. Example:
+    ///
+    /// ```text
+    /// #[doc(alias = "foo")]
+    /// ```
+    ///
+    /// In here, it'll return a span for `"foo"`.
+    pub fn name_value_literal_span(&self) -> Option<Span> {
+        Some(self.name_value_literal()?.span)
+    }
+
+    pub fn value_str(&self) -> Option<Symbol> {
+        self.kind.value_str()
+    }
 
-impl MetaItem {
     fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
     where
         I: Iterator<Item = TokenTree>,
@@ -508,17 +370,14 @@ impl MetaItem {
 impl MetaItemKind {
     pub fn value_str(&self) -> Option<Symbol> {
         match self {
-            MetaItemKind::NameValue(v) => match v.kind {
-                LitKind::Str(s, _) => Some(s),
-                _ => None,
-            },
+            MetaItemKind::NameValue(v) => v.kind.str(),
             _ => None,
         }
     }
 
-    fn list_from_tokens(tokens: TokenStream) -> Option<MetaItemKind> {
+    fn list_from_tokens(tokens: TokenStream) -> Option<ThinVec<NestedMetaItem>> {
         let mut tokens = tokens.into_trees().peekable();
-        let mut result = Vec::new();
+        let mut result = ThinVec::new();
         while tokens.peek().is_some() {
             let item = NestedMetaItem::from_tokens(&mut tokens)?;
             result.push(item);
@@ -527,7 +386,7 @@ impl MetaItemKind {
                 _ => return None,
             }
         }
-        Some(MetaItemKind::List(result))
+        Some(result)
     }
 
     fn name_value_from_tokens(
@@ -544,6 +403,24 @@ impl MetaItemKind {
         }
     }
 
+    fn from_tokens(
+        tokens: &mut iter::Peekable<impl Iterator<Item = TokenTree>>,
+    ) -> Option<MetaItemKind> {
+        match tokens.peek() {
+            Some(TokenTree::Delimited(_, Delimiter::Parenthesis, inner_tokens)) => {
+                let inner_tokens = inner_tokens.clone();
+                tokens.next();
+                MetaItemKind::list_from_tokens(inner_tokens).map(MetaItemKind::List)
+            }
+            Some(TokenTree::Delimited(..)) => None,
+            Some(TokenTree::Token(Token { kind: token::Eq, .. }, _)) => {
+                tokens.next();
+                MetaItemKind::name_value_from_tokens(tokens)
+            }
+            _ => Some(MetaItemKind::Word),
+        }
+    }
+
     fn from_attr_args(args: &AttrArgs) -> Option<MetaItemKind> {
         match args {
             AttrArgs::Empty => Some(MetaItemKind::Word),
@@ -551,7 +428,7 @@ impl MetaItemKind {
                 dspan: _,
                 delim: MacDelimiter::Parenthesis,
                 tokens,
-            }) => MetaItemKind::list_from_tokens(tokens.clone()),
+            }) => MetaItemKind::list_from_tokens(tokens.clone()).map(MetaItemKind::List),
             AttrArgs::Delimited(..) => None,
             AttrArgs::Eq(_, AttrArgsEq::Ast(expr)) => match expr.kind {
                 ExprKind::Lit(token_lit) => {
@@ -565,24 +442,6 @@ impl MetaItemKind {
             AttrArgs::Eq(_, AttrArgsEq::Hir(lit)) => Some(MetaItemKind::NameValue(lit.clone())),
         }
     }
-
-    fn from_tokens(
-        tokens: &mut iter::Peekable<impl Iterator<Item = TokenTree>>,
-    ) -> Option<MetaItemKind> {
-        match tokens.peek() {
-            Some(TokenTree::Delimited(_, Delimiter::Parenthesis, inner_tokens)) => {
-                let inner_tokens = inner_tokens.clone();
-                tokens.next();
-                MetaItemKind::list_from_tokens(inner_tokens)
-            }
-            Some(TokenTree::Delimited(..)) => None,
-            Some(TokenTree::Token(Token { kind: token::Eq, .. }, _)) => {
-                tokens.next();
-                MetaItemKind::name_value_from_tokens(tokens)
-            }
-            _ => Some(MetaItemKind::Word),
-        }
-    }
 }
 
 impl NestedMetaItem {
@@ -593,6 +452,77 @@ impl NestedMetaItem {
         }
     }
 
+    /// For a single-segment meta item, returns its name; otherwise, returns `None`.
+    pub fn ident(&self) -> Option<Ident> {
+        self.meta_item().and_then(|meta_item| meta_item.ident())
+    }
+
+    pub fn name_or_empty(&self) -> Symbol {
+        self.ident().unwrap_or_else(Ident::empty).name
+    }
+
+    /// Returns `true` if this list item is a MetaItem with a name of `name`.
+    pub fn has_name(&self, name: Symbol) -> bool {
+        self.meta_item().map_or(false, |meta_item| meta_item.has_name(name))
+    }
+
+    /// Returns `true` if `self` is a `MetaItem` and the meta item is a word.
+    pub fn is_word(&self) -> bool {
+        self.meta_item().map_or(false, |meta_item| meta_item.is_word())
+    }
+
+    /// Gets a list of inner meta items from a list `MetaItem` type.
+    pub fn meta_item_list(&self) -> Option<&[NestedMetaItem]> {
+        self.meta_item().and_then(|meta_item| meta_item.meta_item_list())
+    }
+
+    /// Returns a name and single literal value tuple of the `MetaItem`.
+    pub fn name_value_literal(&self) -> Option<(Symbol, &MetaItemLit)> {
+        self.meta_item().and_then(|meta_item| {
+            meta_item.meta_item_list().and_then(|meta_item_list| {
+                if meta_item_list.len() == 1
+                    && let Some(ident) = meta_item.ident()
+                    && let Some(lit) = meta_item_list[0].lit()
+                {
+                    return Some((ident.name, lit));
+                }
+                None
+            })
+        })
+    }
+
+    /// See [`MetaItem::name_value_literal_span`].
+    pub fn name_value_literal_span(&self) -> Option<Span> {
+        self.meta_item()?.name_value_literal_span()
+    }
+
+    /// Gets the string value if `self` is a `MetaItem` and the `MetaItem` is a
+    /// `MetaItemKind::NameValue` variant containing a string, otherwise `None`.
+    pub fn value_str(&self) -> Option<Symbol> {
+        self.meta_item().and_then(|meta_item| meta_item.value_str())
+    }
+
+    /// Returns the `MetaItemLit` if `self` is a `NestedMetaItem::Literal`s.
+    pub fn lit(&self) -> Option<&MetaItemLit> {
+        match self {
+            NestedMetaItem::Lit(lit) => Some(lit),
+            _ => None,
+        }
+    }
+
+    /// Returns the `MetaItem` if `self` is a `NestedMetaItem::MetaItem`.
+    pub fn meta_item(&self) -> Option<&MetaItem> {
+        match self {
+            NestedMetaItem::MetaItem(item) => Some(item),
+            _ => None,
+        }
+    }
+
+    /// Returns `true` if the variant is `MetaItem`.
+    pub fn is_meta_item(&self) -> bool {
+        self.meta_item().is_some()
+    }
+
     fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<NestedMetaItem>
     where
         I: Iterator<Item = TokenTree>,
@@ -614,3 +544,89 @@ impl NestedMetaItem {
         MetaItem::from_tokens(tokens).map(NestedMetaItem::MetaItem)
     }
 }
+
+pub fn mk_doc_comment(
+    g: &AttrIdGenerator,
+    comment_kind: CommentKind,
+    style: AttrStyle,
+    data: Symbol,
+    span: Span,
+) -> Attribute {
+    Attribute { kind: AttrKind::DocComment(comment_kind, data), id: g.mk_attr_id(), style, span }
+}
+
+pub fn mk_attr(
+    g: &AttrIdGenerator,
+    style: AttrStyle,
+    path: Path,
+    args: AttrArgs,
+    span: Span,
+) -> Attribute {
+    mk_attr_from_item(g, AttrItem { path, args, tokens: None }, None, style, span)
+}
+
+pub fn mk_attr_from_item(
+    g: &AttrIdGenerator,
+    item: AttrItem,
+    tokens: Option<LazyAttrTokenStream>,
+    style: AttrStyle,
+    span: Span,
+) -> Attribute {
+    Attribute {
+        kind: AttrKind::Normal(P(NormalAttr { item, tokens })),
+        id: g.mk_attr_id(),
+        style,
+        span,
+    }
+}
+
+pub fn mk_attr_word(g: &AttrIdGenerator, style: AttrStyle, name: Symbol, span: Span) -> Attribute {
+    let path = Path::from_ident(Ident::new(name, span));
+    let args = AttrArgs::Empty;
+    mk_attr(g, style, path, args, span)
+}
+
+pub fn mk_attr_nested_word(
+    g: &AttrIdGenerator,
+    style: AttrStyle,
+    outer: Symbol,
+    inner: Symbol,
+    span: Span,
+) -> Attribute {
+    let inner_tokens = TokenStream::new(vec![TokenTree::Token(
+        Token::from_ast_ident(Ident::new(inner, span)),
+        Spacing::Alone,
+    )]);
+    let outer_ident = Ident::new(outer, span);
+    let path = Path::from_ident(outer_ident);
+    let attr_args = AttrArgs::Delimited(DelimArgs {
+        dspan: DelimSpan::from_single(span),
+        delim: MacDelimiter::Parenthesis,
+        tokens: inner_tokens,
+    });
+    mk_attr(g, style, path, attr_args, span)
+}
+
+pub fn mk_attr_name_value_str(
+    g: &AttrIdGenerator,
+    style: AttrStyle,
+    name: Symbol,
+    val: Symbol,
+    span: Span,
+) -> Attribute {
+    let lit = token::Lit::new(token::Str, escape_string_symbol(val), None);
+    let expr = P(Expr {
+        id: DUMMY_NODE_ID,
+        kind: ExprKind::Lit(lit),
+        span,
+        attrs: AttrVec::new(),
+        tokens: None,
+    });
+    let path = Path::from_ident(Ident::new(name, span));
+    let args = AttrArgs::Eq(span, AttrArgsEq::Ast(expr));
+    mk_attr(g, style, path, args, span)
+}
+
+pub fn list_contains_name(items: &[NestedMetaItem], name: Symbol) -> bool {
+    items.iter().any(|item| item.has_name(name))
+}
diff --git a/compiler/rustc_builtin_macros/src/format/ast.rs b/compiler/rustc_ast/src/format.rs
index 01dbffa21..d021bea5e 100644
--- a/compiler/rustc_builtin_macros/src/format/ast.rs
+++ b/compiler/rustc_ast/src/format.rs
@@ -1,5 +1,5 @@
-use rustc_ast::ptr::P;
-use rustc_ast::Expr;
+use crate::ptr::P;
+use crate::Expr;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_span::symbol::{Ident, Symbol};
 use rustc_span::Span;
@@ -39,7 +39,7 @@ use rustc_span::Span;
 /// Basically the "AST" for a complete `format_args!()`.
 ///
 /// E.g., `format_args!("hello {name}");`.
-#[derive(Clone, Debug)]
+#[derive(Clone, Encodable, Decodable, Debug)]
 pub struct FormatArgs {
     pub span: Span,
     pub template: Vec<FormatArgsPiece>,
@@ -49,7 +49,7 @@ pub struct FormatArgs {
 /// A piece of a format template string.
 ///
 /// E.g. "hello" or "{name}".
-#[derive(Clone, Debug)]
+#[derive(Clone, Encodable, Decodable, Debug)]
 pub enum FormatArgsPiece {
     Literal(Symbol),
     Placeholder(FormatPlaceholder),
@@ -59,7 +59,7 @@ pub enum FormatArgsPiece {
 ///
 /// E.g. `1, 2, name="ferris", n=3`,
 /// but also implicit captured arguments like `x` in `format_args!("{x}")`.
-#[derive(Clone, Debug)]
+#[derive(Clone, Encodable, Decodable, Debug)]
 pub struct FormatArguments {
     arguments: Vec<FormatArgument>,
     num_unnamed_args: usize,
@@ -67,6 +67,12 @@ pub struct FormatArguments {
     names: FxHashMap<Symbol, usize>,
 }
 
+// FIXME: Rustdoc has trouble proving Send/Sync for this. See #106930.
+#[cfg(parallel_compiler)]
+unsafe impl Sync for FormatArguments {}
+#[cfg(parallel_compiler)]
+unsafe impl Send for FormatArguments {}
+
 impl FormatArguments {
     pub fn new() -> Self {
         Self {
@@ -121,18 +127,22 @@ impl FormatArguments {
         &self.arguments[..self.num_explicit_args]
     }
 
-    pub fn into_vec(self) -> Vec<FormatArgument> {
-        self.arguments
+    pub fn all_args(&self) -> &[FormatArgument] {
+        &self.arguments[..]
+    }
+
+    pub fn all_args_mut(&mut self) -> &mut [FormatArgument] {
+        &mut self.arguments[..]
     }
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Encodable, Decodable, Debug)]
 pub struct FormatArgument {
     pub kind: FormatArgumentKind,
     pub expr: P<Expr>,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Encodable, Decodable, Debug)]
 pub enum FormatArgumentKind {
     /// `format_args(…, arg)`
     Normal,
@@ -152,7 +162,7 @@ impl FormatArgumentKind {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq)]
 pub struct FormatPlaceholder {
     /// Index into [`FormatArgs::arguments`].
     pub argument: FormatArgPosition,
@@ -164,7 +174,7 @@ pub struct FormatPlaceholder {
     pub format_options: FormatOptions,
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq)]
 pub struct FormatArgPosition {
     /// Which argument this position refers to (Ok),
     /// or would've referred to if it existed (Err).
@@ -175,7 +185,7 @@ pub struct FormatArgPosition {
     pub span: Option<Span>,
 }
 
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+#[derive(Copy, Clone, Encodable, Decodable, Debug, PartialEq, Eq)]
 pub enum FormatArgPositionKind {
     /// `{}` or `{:.*}`
     Implicit,
@@ -185,7 +195,7 @@ pub enum FormatArgPositionKind {
     Named,
 }
 
-#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)]
+#[derive(Copy, Clone, Encodable, Decodable, Debug, PartialEq, Eq, Hash)]
 pub enum FormatTrait {
     /// `{}`
     Display,
@@ -207,7 +217,7 @@ pub enum FormatTrait {
     UpperHex,
 }
 
-#[derive(Clone, Debug, Default, PartialEq, Eq)]
+#[derive(Clone, Encodable, Decodable, Default, Debug, PartialEq, Eq)]
 pub struct FormatOptions {
     /// The width. E.g. `{:5}` or `{:width$}`.
     pub width: Option<FormatCount>,
@@ -217,11 +227,33 @@ pub struct FormatOptions {
     pub alignment: Option<FormatAlignment>,
     /// The fill character. E.g. the `.` in `{:.>10}`.
     pub fill: Option<char>,
-    /// The `+`, `-`, `0`, `#`, `x?` and `X?` flags.
-    pub flags: u32,
+    /// The `+` or `-` flag.
+    pub sign: Option<FormatSign>,
+    /// The `#` flag.
+    pub alternate: bool,
+    /// The `0` flag. E.g. the `0` in `{:02x}`.
+    pub zero_pad: bool,
+    /// The `x` or `X` flag (for `Debug` only). E.g. the `x` in `{:x?}`.
+    pub debug_hex: Option<FormatDebugHex>,
+}
+
+#[derive(Copy, Clone, Encodable, Decodable, Debug, PartialEq, Eq)]
+pub enum FormatSign {
+    /// The `+` flag.
+    Plus,
+    /// The `-` flag.
+    Minus,
+}
+
+#[derive(Copy, Clone, Encodable, Decodable, Debug, PartialEq, Eq)]
+pub enum FormatDebugHex {
+    /// The `x` flag in `{:x?}`.
+    Lower,
+    /// The `X` flag in `{:X?}`.
+    Upper,
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Copy, Clone, Encodable, Decodable, Debug, PartialEq, Eq)]
 pub enum FormatAlignment {
     /// `{:<}`
     Left,
@@ -231,7 +263,7 @@ pub enum FormatAlignment {
     Center,
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq)]
 pub enum FormatCount {
     /// `{:5}` or `{:.5}`
     Literal(usize),
diff --git a/compiler/rustc_ast/src/lib.rs b/compiler/rustc_ast/src/lib.rs
index 9c1dfeb1a..23c32fa96 100644
--- a/compiler/rustc_ast/src/lib.rs
+++ b/compiler/rustc_ast/src/lib.rs
@@ -16,7 +16,6 @@
 #![feature(let_chains)]
 #![feature(min_specialization)]
 #![feature(negative_impls)]
-#![feature(slice_internals)]
 #![feature(stmt_expr_attributes)]
 #![recursion_limit = "256"]
 #![deny(rustc::untranslatable_diagnostic)]
@@ -42,6 +41,7 @@ pub mod ast_traits;
 pub mod attr;
 pub mod entry;
 pub mod expand;
+pub mod format;
 pub mod mut_visit;
 pub mod node_id;
 pub mod ptr;
@@ -51,6 +51,7 @@ pub mod visit;
 
 pub use self::ast::*;
 pub use self::ast_traits::{AstDeref, AstNodeWrapper, HasAttrs, HasNodeId, HasSpan, HasTokens};
+pub use self::format::*;
 
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 
diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs
index 77f342d1e..7dcb03b4c 100644
--- a/compiler/rustc_ast/src/mut_visit.rs
+++ b/compiler/rustc_ast/src/mut_visit.rs
@@ -17,10 +17,10 @@ use rustc_data_structures::sync::Lrc;
 use rustc_span::source_map::Spanned;
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
-
 use smallvec::{smallvec, Array, SmallVec};
 use std::ops::DerefMut;
 use std::{panic, ptr};
+use thin_vec::ThinVec;
 
 pub trait ExpectOne<A: Array> {
     fn expect_one(self, err: &'static str) -> A::Item;
@@ -297,6 +297,10 @@ pub trait MutVisitor: Sized {
     fn visit_inline_asm_sym(&mut self, sym: &mut InlineAsmSym) {
         noop_visit_inline_asm_sym(sym, self)
     }
+
+    fn visit_format_args(&mut self, fmt: &mut FormatArgs) {
+        noop_visit_format_args(fmt, self)
+    }
 }
 
 /// Use a map-style function (`FnOnce(T) -> T`) to overwrite a `&mut T`. Useful
@@ -333,6 +337,17 @@ where
 
 // No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
 #[inline]
+pub fn visit_thin_vec<T, F>(elems: &mut ThinVec<T>, mut visit_elem: F)
+where
+    F: FnMut(&mut T),
+{
+    for elem in elems {
+        visit_elem(elem);
+    }
+}
+
+// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
+#[inline]
 pub fn visit_opt<T, F>(opt: &mut Option<T>, mut visit_elem: F)
 where
     F: FnMut(&mut T),
@@ -355,6 +370,11 @@ pub fn visit_exprs<T: MutVisitor>(exprs: &mut Vec<P<Expr>>, vis: &mut T) {
 }
 
 // No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
+pub fn visit_thin_exprs<T: MutVisitor>(exprs: &mut ThinVec<P<Expr>>, vis: &mut T) {
+    exprs.flat_map_in_place(|expr| vis.filter_map_expr(expr))
+}
+
+// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
 pub fn visit_bounds<T: MutVisitor>(bounds: &mut GenericBounds, vis: &mut T) {
     visit_vec(bounds, |bound| vis.visit_param_bound(bound));
 }
@@ -470,7 +490,7 @@ pub fn noop_visit_ty<T: MutVisitor>(ty: &mut P<Ty>, vis: &mut T) {
             vis.visit_fn_decl(decl);
             vis.visit_span(decl_span);
         }
-        TyKind::Tup(tys) => visit_vec(tys, |ty| vis.visit_ty(ty)),
+        TyKind::Tup(tys) => visit_thin_vec(tys, |ty| vis.visit_ty(ty)),
         TyKind::Paren(ty) => vis.visit_ty(ty),
         TyKind::Path(qself, path) => {
             vis.visit_qself(qself);
@@ -557,7 +577,7 @@ pub fn noop_visit_angle_bracketed_parameter_data<T: MutVisitor>(
     vis: &mut T,
 ) {
     let AngleBracketedArgs { args, span } = data;
-    visit_vec(args, |arg| match arg {
+    visit_thin_vec(args, |arg| match arg {
         AngleBracketedArg::Arg(arg) => vis.visit_generic_arg(arg),
         AngleBracketedArg::Constraint(constraint) => vis.visit_constraint(constraint),
     });
@@ -569,7 +589,7 @@ pub fn noop_visit_parenthesized_parameter_data<T: MutVisitor>(
     vis: &mut T,
 ) {
     let ParenthesizedArgs { inputs, output, span, .. } = args;
-    visit_vec(inputs, |input| vis.visit_ty(input));
+    visit_thin_vec(inputs, |input| vis.visit_ty(input));
     noop_visit_fn_ret_ty(output, vis);
     vis.visit_span(span);
 }
@@ -632,7 +652,7 @@ pub fn noop_visit_meta_item<T: MutVisitor>(mi: &mut MetaItem, vis: &mut T) {
     let MetaItem { path: _, kind, span } = mi;
     match kind {
         MetaItemKind::Word => {}
-        MetaItemKind::List(mis) => visit_vec(mis, |mi| vis.visit_meta_list_item(mi)),
+        MetaItemKind::List(mis) => visit_thin_vec(mis, |mi| vis.visit_meta_list_item(mi)),
         MetaItemKind::NameValue(_s) => {}
     }
     vis.visit_span(span);
@@ -835,9 +855,7 @@ pub fn noop_visit_closure_binder<T: MutVisitor>(binder: &mut ClosureBinder, vis:
     match binder {
         ClosureBinder::NotPresent => {}
         ClosureBinder::For { span: _, generic_params } => {
-            let mut vec = std::mem::take(generic_params).into_vec();
-            vec.flat_map_in_place(|param| vis.flat_map_generic_param(param));
-            *generic_params = P::from_vec(vec);
+            generic_params.flat_map_in_place(|param| vis.flat_map_generic_param(param));
         }
     }
 }
@@ -915,7 +933,7 @@ pub fn noop_visit_generics<T: MutVisitor>(generics: &mut Generics, vis: &mut T)
 
 pub fn noop_visit_where_clause<T: MutVisitor>(wc: &mut WhereClause, vis: &mut T) {
     let WhereClause { has_where_token: _, predicates, span } = wc;
-    visit_vec(predicates, |predicate| vis.visit_where_predicate(predicate));
+    visit_thin_vec(predicates, |predicate| vis.visit_where_predicate(predicate));
     vis.visit_span(span);
 }
 
@@ -1223,7 +1241,7 @@ pub fn noop_visit_pat<T: MutVisitor>(pat: &mut P<Pat>, vis: &mut T) {
         PatKind::TupleStruct(qself, path, elems) => {
             vis.visit_qself(qself);
             vis.visit_path(path);
-            visit_vec(elems, |elem| vis.visit_pat(elem));
+            visit_thin_vec(elems, |elem| vis.visit_pat(elem));
         }
         PatKind::Path(qself, path) => {
             vis.visit_qself(qself);
@@ -1242,7 +1260,7 @@ pub fn noop_visit_pat<T: MutVisitor>(pat: &mut P<Pat>, vis: &mut T) {
             vis.visit_span(span);
         }
         PatKind::Tuple(elems) | PatKind::Slice(elems) | PatKind::Or(elems) => {
-            visit_vec(elems, |elem| vis.visit_pat(elem))
+            visit_thin_vec(elems, |elem| vis.visit_pat(elem))
         }
         PatKind::Paren(inner) => vis.visit_pat(inner),
         PatKind::MacCall(mac) => vis.visit_mac_call(mac),
@@ -1284,13 +1302,22 @@ pub fn noop_visit_inline_asm_sym<T: MutVisitor>(
     vis.visit_path(path);
 }
 
+pub fn noop_visit_format_args<T: MutVisitor>(fmt: &mut FormatArgs, vis: &mut T) {
+    for arg in fmt.arguments.all_args_mut() {
+        if let FormatArgumentKind::Named(name) = &mut arg.kind {
+            vis.visit_ident(name);
+        }
+        vis.visit_expr(&mut arg.expr);
+    }
+}
+
 pub fn noop_visit_expr<T: MutVisitor>(
     Expr { kind, id, span, attrs, tokens }: &mut Expr,
     vis: &mut T,
 ) {
     match kind {
         ExprKind::Box(expr) => vis.visit_expr(expr),
-        ExprKind::Array(exprs) => visit_exprs(exprs, vis),
+        ExprKind::Array(exprs) => visit_thin_exprs(exprs, vis),
         ExprKind::ConstBlock(anon_const) => {
             vis.visit_anon_const(anon_const);
         }
@@ -1298,10 +1325,10 @@ pub fn noop_visit_expr<T: MutVisitor>(
             vis.visit_expr(expr);
             vis.visit_anon_const(count);
         }
-        ExprKind::Tup(exprs) => visit_exprs(exprs, vis),
+        ExprKind::Tup(exprs) => visit_thin_exprs(exprs, vis),
         ExprKind::Call(f, args) => {
             vis.visit_expr(f);
-            visit_exprs(args, vis);
+            visit_thin_exprs(args, vis);
         }
         ExprKind::MethodCall(box MethodCall {
             seg: PathSegment { ident, id, args: seg_args },
@@ -1313,7 +1340,7 @@ pub fn noop_visit_expr<T: MutVisitor>(
             vis.visit_id(id);
             visit_opt(seg_args, |args| vis.visit_generic_args(args));
             vis.visit_method_receiver_expr(receiver);
-            visit_exprs(call_args, vis);
+            visit_thin_exprs(call_args, vis);
             vis.visit_span(span);
         }
         ExprKind::Binary(_binop, lhs, rhs) => {
@@ -1425,6 +1452,7 @@ pub fn noop_visit_expr<T: MutVisitor>(
             visit_opt(expr, |expr| vis.visit_expr(expr));
         }
         ExprKind::InlineAsm(asm) => vis.visit_inline_asm(asm),
+        ExprKind::FormatArgs(fmt) => vis.visit_format_args(fmt),
         ExprKind::MacCall(mac) => vis.visit_mac_call(mac),
         ExprKind::Struct(se) => {
             let StructExpr { qself, path, fields, rest } = se.deref_mut();
diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs
index fabd43a16..f0a6a5e07 100644
--- a/compiler/rustc_ast/src/tokenstream.rs
+++ b/compiler/rustc_ast/src/tokenstream.rs
@@ -41,7 +41,8 @@ use std::{fmt, iter};
 /// Nothing special happens to misnamed or misplaced `SubstNt`s.
 #[derive(Debug, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
 pub enum TokenTree {
-    /// A single token.
+    /// A single token. Should never be `OpenDelim` or `CloseDelim`, because
+    /// delimiters are implicitly represented by `Delimited`.
     Token(Token, Spacing),
     /// A delimited sequence of token trees.
     Delimited(DelimSpan, Delimiter, TokenStream),
@@ -388,12 +389,12 @@ impl TokenStream {
         self.0.len()
     }
 
-    pub fn trees(&self) -> CursorRef<'_> {
-        CursorRef::new(self)
+    pub fn trees(&self) -> RefTokenTreeCursor<'_> {
+        RefTokenTreeCursor::new(self)
     }
 
-    pub fn into_trees(self) -> Cursor {
-        Cursor::new(self)
+    pub fn into_trees(self) -> TokenTreeCursor {
+        TokenTreeCursor::new(self)
     }
 
     /// Compares two `TokenStream`s, checking equality without regarding span information.
@@ -551,16 +552,17 @@ impl TokenStream {
     }
 }
 
-/// By-reference iterator over a [`TokenStream`].
+/// By-reference iterator over a [`TokenStream`], that produces `&TokenTree`
+/// items.
 #[derive(Clone)]
-pub struct CursorRef<'t> {
+pub struct RefTokenTreeCursor<'t> {
     stream: &'t TokenStream,
     index: usize,
 }
 
-impl<'t> CursorRef<'t> {
+impl<'t> RefTokenTreeCursor<'t> {
     fn new(stream: &'t TokenStream) -> Self {
-        CursorRef { stream, index: 0 }
+        RefTokenTreeCursor { stream, index: 0 }
     }
 
     pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
@@ -568,7 +570,7 @@ impl<'t> CursorRef<'t> {
     }
 }
 
-impl<'t> Iterator for CursorRef<'t> {
+impl<'t> Iterator for RefTokenTreeCursor<'t> {
     type Item = &'t TokenTree;
 
     fn next(&mut self) -> Option<&'t TokenTree> {
@@ -579,15 +581,16 @@ impl<'t> Iterator for CursorRef<'t> {
     }
 }
 
-/// Owning by-value iterator over a [`TokenStream`].
+/// Owning by-value iterator over a [`TokenStream`], that produces `TokenTree`
+/// items.
 // FIXME: Many uses of this can be replaced with by-reference iterator to avoid clones.
 #[derive(Clone)]
-pub struct Cursor {
+pub struct TokenTreeCursor {
     pub stream: TokenStream,
     index: usize,
 }
 
-impl Iterator for Cursor {
+impl Iterator for TokenTreeCursor {
     type Item = TokenTree;
 
     fn next(&mut self) -> Option<TokenTree> {
@@ -598,9 +601,9 @@ impl Iterator for Cursor {
     }
 }
 
-impl Cursor {
+impl TokenTreeCursor {
     fn new(stream: TokenStream) -> Self {
-        Cursor { stream, index: 0 }
+        TokenTreeCursor { stream, index: 0 }
     }
 
     #[inline]
@@ -614,6 +617,15 @@ impl Cursor {
     pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
         self.stream.0.get(self.index + n)
     }
+
+    // Replace the previously obtained token tree with `tts`, and rewind to
+    // just before them.
+    pub fn replace_prev_and_rewind(&mut self, tts: Vec<TokenTree>) {
+        assert!(self.index > 0);
+        self.index -= 1;
+        let stream = Lrc::make_mut(&mut self.stream.0);
+        stream.splice(self.index..self.index + 1, tts);
+    }
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
diff --git a/compiler/rustc_ast/src/util/comments.rs b/compiler/rustc_ast/src/util/comments.rs
index 275ed02c2..eece99a3e 100644
--- a/compiler/rustc_ast/src/util/comments.rs
+++ b/compiler/rustc_ast/src/util/comments.rs
@@ -58,23 +58,24 @@ pub fn beautify_doc_string(data: Symbol, kind: CommentKind) -> Symbol {
         // In case we have doc comments like `/**` or `/*!`, we want to remove stars if they are
         // present. However, we first need to strip the empty lines so they don't get in the middle
         // when we try to compute the "horizontal trim".
-        let lines = if kind == CommentKind::Block {
-            // Whatever happens, we skip the first line.
-            let mut i = lines
-                .get(0)
-                .map(|l| if l.trim_start().starts_with('*') { 0 } else { 1 })
-                .unwrap_or(0);
-            let mut j = lines.len();
-
-            while i < j && lines[i].trim().is_empty() {
-                i += 1;
-            }
-            while j > i && lines[j - 1].trim().is_empty() {
-                j -= 1;
+        let lines = match kind {
+            CommentKind::Block => {
+                // Whatever happens, we skip the first line.
+                let mut i = lines
+                    .get(0)
+                    .map(|l| if l.trim_start().starts_with('*') { 0 } else { 1 })
+                    .unwrap_or(0);
+                let mut j = lines.len();
+
+                while i < j && lines[i].trim().is_empty() {
+                    i += 1;
+                }
+                while j > i && lines[j - 1].trim().is_empty() {
+                    j -= 1;
+                }
+                &lines[i..j]
             }
-            &lines[i..j]
-        } else {
-            lines
+            CommentKind::Line => lines,
         };
 
         for line in lines {
diff --git a/compiler/rustc_ast/src/util/parser.rs b/compiler/rustc_ast/src/util/parser.rs
index 4f7099c7b..3a0af04f9 100644
--- a/compiler/rustc_ast/src/util/parser.rs
+++ b/compiler/rustc_ast/src/util/parser.rs
@@ -225,7 +225,7 @@ impl AssocOp {
             AssignOp(_) | // `{ 42 } +=`
             As | // `{ 42 } as usize`
             // Equal | // `{ 42 } == { 42 }`    Accepting these here would regress incorrect
-            // NotEqual | // `{ 42 } != { 42 }  struct literals parser recovery.
+            // NotEqual | // `{ 42 } != { 42 }` struct literals parser recovery.
             Colon, // `{ 42 }: usize`
         )
     }
@@ -271,6 +271,7 @@ pub enum ExprPrecedence {
     Try,
     InlineAsm,
     Mac,
+    FormatArgs,
 
     Array,
     Repeat,
@@ -335,7 +336,8 @@ impl ExprPrecedence {
             | ExprPrecedence::Index
             | ExprPrecedence::Try
             | ExprPrecedence::InlineAsm
-            | ExprPrecedence::Mac => PREC_POSTFIX,
+            | ExprPrecedence::Mac
+            | ExprPrecedence::FormatArgs => PREC_POSTFIX,
 
             // Never need parens
             ExprPrecedence::Array
diff --git a/compiler/rustc_ast/src/util/unicode.rs b/compiler/rustc_ast/src/util/unicode.rs
index 0eae791b2..6f57d66b2 100644
--- a/compiler/rustc_ast/src/util/unicode.rs
+++ b/compiler/rustc_ast/src/util/unicode.rs
@@ -17,7 +17,7 @@ pub fn contains_text_flow_control_chars(s: &str) -> bool {
     // U+2069 - E2 81 A9
     let mut bytes = s.as_bytes();
     loop {
-        match core::slice::memchr::memchr(0xE2, bytes) {
+        match memchr::memchr(0xE2, bytes) {
             Some(idx) => {
                 // bytes are valid UTF-8 -> E2 must be followed by two bytes
                 let ch = &bytes[idx..idx + 3];
diff --git a/compiler/rustc_ast/src/visit.rs b/compiler/rustc_ast/src/visit.rs
index e8823eff8..bdb1879ec 100644
--- a/compiler/rustc_ast/src/visit.rs
+++ b/compiler/rustc_ast/src/visit.rs
@@ -242,6 +242,9 @@ pub trait Visitor<'ast>: Sized {
     fn visit_inline_asm(&mut self, asm: &'ast InlineAsm) {
         walk_inline_asm(self, asm)
     }
+    fn visit_format_args(&mut self, fmt: &'ast FormatArgs) {
+        walk_format_args(self, fmt)
+    }
     fn visit_inline_asm_sym(&mut self, sym: &'ast InlineAsmSym) {
         walk_inline_asm_sym(self, sym)
     }
@@ -756,6 +759,15 @@ pub fn walk_inline_asm_sym<'a, V: Visitor<'a>>(visitor: &mut V, sym: &'a InlineA
     visitor.visit_path(&sym.path, sym.id);
 }
 
+pub fn walk_format_args<'a, V: Visitor<'a>>(visitor: &mut V, fmt: &'a FormatArgs) {
+    for arg in fmt.arguments.all_args() {
+        if let FormatArgumentKind::Named(name) = arg.kind {
+            visitor.visit_ident(name);
+        }
+        visitor.visit_expr(&arg.expr);
+    }
+}
+
 pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) {
     walk_list!(visitor, visit_attribute, expression.attrs.iter());
 
@@ -896,6 +908,7 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) {
         ExprKind::MacCall(mac) => visitor.visit_mac_call(mac),
         ExprKind::Paren(subexpression) => visitor.visit_expr(subexpression),
         ExprKind::InlineAsm(asm) => visitor.visit_inline_asm(asm),
+        ExprKind::FormatArgs(f) => visitor.visit_format_args(f),
         ExprKind::Yield(optional_expression) => {
             walk_list!(visitor, visit_expr, optional_expression);
         }
diff --git a/compiler/rustc_ast_lowering/Cargo.toml b/compiler/rustc_ast_lowering/Cargo.toml
index 6a59b9e61..eb2e82d79 100644
--- a/compiler/rustc_ast_lowering/Cargo.toml
+++ b/compiler/rustc_ast_lowering/Cargo.toml
@@ -7,7 +7,6 @@ edition = "2021"
 doctest = false
 
 [dependencies]
-rustc_arena = { path = "../rustc_arena" }
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_data_structures = { path = "../rustc_data_structures" }
@@ -16,10 +15,9 @@ rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_macros = { path = "../rustc_macros" }
-rustc_query_system = { path = "../rustc_query_system" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-thin-vec = "0.2.9"
+thin-vec = "0.2.12"
 tracing = "0.1"
diff --git a/compiler/rustc_error_messages/locales/en-US/ast_lowering.ftl b/compiler/rustc_ast_lowering/locales/en-US.ftl
index 03c88c6c0..3ccd84398 100644
--- a/compiler/rustc_error_messages/locales/en-US/ast_lowering.ftl
+++ b/compiler/rustc_ast_lowering/locales/en-US.ftl
@@ -19,8 +19,8 @@ ast_lowering_remove_parentheses = remove these parentheses
 ast_lowering_misplaced_impl_trait =
     `impl Trait` only allowed in function and inherent method return types, not in {$position}
 
-ast_lowering_rustc_box_attribute_error =
-    #[rustc_box] requires precisely one argument and no other attributes are allowed
+ast_lowering_misplaced_assoc_ty_binding =
+    associated type bounds are only allowed in where clauses and function signatures, not in {$position}
 
 ast_lowering_underscore_expr_lhs_assign =
     in expressions, `_` can only be used on the left-hand side of an assignment
diff --git a/compiler/rustc_ast_lowering/src/errors.rs b/compiler/rustc_ast_lowering/src/errors.rs
index 21c6a2d26..5e6b6050b 100644
--- a/compiler/rustc_ast_lowering/src/errors.rs
+++ b/compiler/rustc_ast_lowering/src/errors.rs
@@ -79,11 +79,12 @@ pub struct MisplacedImplTrait<'a> {
     pub position: DiagnosticArgFromDisplay<'a>,
 }
 
-#[derive(Diagnostic, Clone, Copy)]
-#[diag(ast_lowering_rustc_box_attribute_error)]
-pub struct RustcBoxAttributeError {
+#[derive(Diagnostic)]
+#[diag(ast_lowering_misplaced_assoc_ty_binding)]
+pub struct MisplacedAssocTyBinding<'a> {
     #[primary_span]
     pub span: Span,
+    pub position: DiagnosticArgFromDisplay<'a>,
 }
 
 #[derive(Diagnostic, Clone, Copy)]
@@ -339,7 +340,7 @@ pub struct InclusiveRangeWithNoEnd {
 #[derive(Diagnostic, Clone, Copy)]
 #[diag(ast_lowering_trait_fn_async, code = "E0706")]
 #[note]
-#[note(note2)]
+#[note(ast_lowering_note2)]
 pub struct TraitFnAsync {
     #[primary_span]
     pub fn_span: Span,
diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index c3611b2f5..ffb30b1b3 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -2,7 +2,7 @@ use super::errors::{
     AsyncGeneratorsNotSupported, AsyncNonMoveClosureNotSupported, AwaitOnlyInAsyncFnAndBlocks,
     BaseExpressionDoubleDot, ClosureCannotBeStatic, FunctionalRecordUpdateDestructuringAssignemnt,
     GeneratorTooManyParameters, InclusiveRangeWithNoEnd, NotSupportedForLifetimeBinderAsyncClosure,
-    RustcBoxAttributeError, UnderscoreExprLhsAssign,
+    UnderscoreExprLhsAssign,
 };
 use super::ResolverAstLoweringExt;
 use super::{ImplTraitContext, LoweringContext, ParamMode, ParenthesizedGenericArgs};
@@ -16,9 +16,9 @@ use rustc_hir::def::Res;
 use rustc_hir::definitions::DefPathData;
 use rustc_session::errors::report_lit_error;
 use rustc_span::source_map::{respan, DesugaringKind, Span, Spanned};
-use rustc_span::symbol::{sym, Ident};
+use rustc_span::symbol::{sym, Ident, Symbol};
 use rustc_span::DUMMY_SP;
-use thin_vec::thin_vec;
+use thin_vec::{thin_vec, ThinVec};
 
 impl<'hir> LoweringContext<'_, 'hir> {
     fn lower_exprs(&mut self, exprs: &[AstP<Expr>]) -> &'hir [hir::Expr<'hir>] {
@@ -83,15 +83,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 }
                 ExprKind::Tup(elts) => hir::ExprKind::Tup(self.lower_exprs(elts)),
                 ExprKind::Call(f, args) => {
-                    if e.attrs.get(0).map_or(false, |a| a.has_name(sym::rustc_box)) {
-                        if let [inner] = &args[..] && e.attrs.len() == 1 {
-                            let kind = hir::ExprKind::Box(self.lower_expr(&inner));
-                            return hir::Expr { hir_id, kind, span: self.lower_span(e.span) };
-                        } else {
-                            self.tcx.sess.emit_err(RustcBoxAttributeError { span: e.span });
-                            hir::ExprKind::Err
-                        }
-                    } else if let Some(legacy_args) = self.resolver.legacy_const_generic_args(f) {
+                    if let Some(legacy_args) = self.resolver.legacy_const_generic_args(f) {
                         self.lower_legacy_const_generics((**f).clone(), args.clone(), &legacy_args)
                     } else {
                         let f = self.lower_expr(f);
@@ -139,13 +131,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 ExprKind::Cast(expr, ty) => {
                     let expr = self.lower_expr(expr);
                     let ty =
-                        self.lower_ty(ty, &ImplTraitContext::Disallowed(ImplTraitPosition::Type));
+                        self.lower_ty(ty, &ImplTraitContext::Disallowed(ImplTraitPosition::Cast));
                     hir::ExprKind::Cast(expr, ty)
                 }
                 ExprKind::Type(expr, ty) => {
                     let expr = self.lower_expr(expr);
                     let ty =
-                        self.lower_ty(ty, &ImplTraitContext::Disallowed(ImplTraitPosition::Type));
+                        self.lower_ty(ty, &ImplTraitContext::Disallowed(ImplTraitPosition::Cast));
                     hir::ExprKind::Type(expr, ty)
                 }
                 ExprKind::AddrOf(k, m, ohs) => {
@@ -266,8 +258,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     self.lower_expr_range(e.span, e1.as_deref(), e2.as_deref(), *lims)
                 }
                 ExprKind::Underscore => {
-                    self.tcx.sess.emit_err(UnderscoreExprLhsAssign { span: e.span });
-                    hir::ExprKind::Err
+                    let guar = self.tcx.sess.emit_err(UnderscoreExprLhsAssign { span: e.span });
+                    hir::ExprKind::Err(guar)
                 }
                 ExprKind::Path(qself, path) => {
                     let qpath = self.lower_qpath(
@@ -294,12 +286,14 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 ExprKind::InlineAsm(asm) => {
                     hir::ExprKind::InlineAsm(self.lower_inline_asm(e.span, asm))
                 }
+                ExprKind::FormatArgs(fmt) => self.lower_format_args(e.span, fmt),
                 ExprKind::Struct(se) => {
                     let rest = match &se.rest {
                         StructRest::Base(e) => Some(self.lower_expr(e)),
                         StructRest::Rest(sp) => {
-                            self.tcx.sess.emit_err(BaseExpressionDoubleDot { span: *sp });
-                            Some(&*self.arena.alloc(self.expr_err(*sp)))
+                            let guar =
+                                self.tcx.sess.emit_err(BaseExpressionDoubleDot { span: *sp });
+                            Some(&*self.arena.alloc(self.expr_err(*sp, guar)))
                         }
                         StructRest::None => None,
                     };
@@ -317,7 +311,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     )
                 }
                 ExprKind::Yield(opt_expr) => self.lower_expr_yield(e.span, opt_expr.as_deref()),
-                ExprKind::Err => hir::ExprKind::Err,
+                ExprKind::Err => hir::ExprKind::Err(
+                    self.tcx.sess.delay_span_bug(e.span, "lowered ExprKind::Err"),
+                ),
                 ExprKind::Try(sub_expr) => self.lower_expr_try(e.span, sub_expr),
 
                 ExprKind::Paren(_) | ExprKind::ForLoop(..) => unreachable!("already handled"),
@@ -366,7 +362,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
     fn lower_legacy_const_generics(
         &mut self,
         mut f: Expr,
-        args: Vec<AstP<Expr>>,
+        args: ThinVec<AstP<Expr>>,
         legacy_args_idx: &[usize],
     ) -> hir::ExprKind<'hir> {
         let ExprKind::Path(None, path) = &mut f.kind else {
@@ -375,7 +371,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
         // Split the arguments into const generics and normal arguments
         let mut real_args = vec![];
-        let mut generic_args = vec![];
+        let mut generic_args = ThinVec::new();
         for (idx, arg) in args.into_iter().enumerate() {
             if legacy_args_idx.contains(&idx) {
                 let parent_def_id = self.current_hir_id_owner;
@@ -760,7 +756,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 self.expr_ident_mut(span, task_context_ident, task_context_hid)
             } else {
                 // Use of `await` outside of an async context, we cannot use `task_context` here.
-                self.expr_err(span)
+                self.expr_err(span, self.tcx.sess.delay_span_bug(span, "no task_context hir id"))
             };
             let new_unchecked = self.expr_call_lang_item_fn_mut(
                 span,
@@ -1735,7 +1731,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         self.expr(span, hir::ExprKind::DropTemps(expr))
     }
 
-    fn expr_match(
+    pub(super) fn expr_match(
         &mut self,
         span: Span,
         arg: &'hir hir::Expr<'hir>,
@@ -1763,7 +1759,44 @@ impl<'hir> LoweringContext<'_, 'hir> {
         self.arena.alloc(self.expr(sp, hir::ExprKind::Tup(&[])))
     }
 
-    fn expr_call_mut(
+    pub(super) fn expr_usize(&mut self, sp: Span, value: usize) -> hir::Expr<'hir> {
+        self.expr(
+            sp,
+            hir::ExprKind::Lit(hir::Lit {
+                span: sp,
+                node: ast::LitKind::Int(
+                    value as u128,
+                    ast::LitIntType::Unsigned(ast::UintTy::Usize),
+                ),
+            }),
+        )
+    }
+
+    pub(super) fn expr_u32(&mut self, sp: Span, value: u32) -> hir::Expr<'hir> {
+        self.expr(
+            sp,
+            hir::ExprKind::Lit(hir::Lit {
+                span: sp,
+                node: ast::LitKind::Int(value.into(), ast::LitIntType::Unsigned(ast::UintTy::U32)),
+            }),
+        )
+    }
+
+    pub(super) fn expr_char(&mut self, sp: Span, value: char) -> hir::Expr<'hir> {
+        self.expr(sp, hir::ExprKind::Lit(hir::Lit { span: sp, node: ast::LitKind::Char(value) }))
+    }
+
+    pub(super) fn expr_str(&mut self, sp: Span, value: Symbol) -> hir::Expr<'hir> {
+        self.expr(
+            sp,
+            hir::ExprKind::Lit(hir::Lit {
+                span: sp,
+                node: ast::LitKind::Str(value, ast::StrStyle::Cooked),
+            }),
+        )
+    }
+
+    pub(super) fn expr_call_mut(
         &mut self,
         span: Span,
         e: &'hir hir::Expr<'hir>,
@@ -1772,7 +1805,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         self.expr(span, hir::ExprKind::Call(e, args))
     }
 
-    fn expr_call(
+    pub(super) fn expr_call(
         &mut self,
         span: Span,
         e: &'hir hir::Expr<'hir>,
@@ -1814,6 +1847,27 @@ impl<'hir> LoweringContext<'_, 'hir> {
         )
     }
 
+    /// `<LangItem>::name`
+    pub(super) fn expr_lang_item_type_relative(
+        &mut self,
+        span: Span,
+        lang_item: hir::LangItem,
+        name: Symbol,
+    ) -> hir::Expr<'hir> {
+        let path = hir::ExprKind::Path(hir::QPath::TypeRelative(
+            self.arena.alloc(self.ty(
+                span,
+                hir::TyKind::Path(hir::QPath::LangItem(lang_item, self.lower_span(span), None)),
+            )),
+            self.arena.alloc(hir::PathSegment::new(
+                Ident::new(name, span),
+                self.next_id(),
+                Res::Err,
+            )),
+        ));
+        self.expr(span, path)
+    }
+
     pub(super) fn expr_ident(
         &mut self,
         sp: Span,
@@ -1872,12 +1926,25 @@ impl<'hir> LoweringContext<'_, 'hir> {
         self.expr(b.span, hir::ExprKind::Block(b, None))
     }
 
+    pub(super) fn expr_array_ref(
+        &mut self,
+        span: Span,
+        elements: &'hir [hir::Expr<'hir>],
+    ) -> hir::Expr<'hir> {
+        let addrof = hir::ExprKind::AddrOf(
+            hir::BorrowKind::Ref,
+            hir::Mutability::Not,
+            self.arena.alloc(self.expr(span, hir::ExprKind::Array(elements))),
+        );
+        self.expr(span, addrof)
+    }
+
     pub(super) fn expr(&mut self, span: Span, kind: hir::ExprKind<'hir>) -> hir::Expr<'hir> {
         let hir_id = self.next_id();
         hir::Expr { hir_id, kind, span: self.lower_span(span) }
     }
 
-    fn expr_field(
+    pub(super) fn expr_field(
         &mut self,
         ident: Ident,
         expr: &'hir hir::Expr<'hir>,
@@ -1892,7 +1959,11 @@ impl<'hir> LoweringContext<'_, 'hir> {
         }
     }
 
-    fn arm(&mut self, pat: &'hir hir::Pat<'hir>, expr: &'hir hir::Expr<'hir>) -> hir::Arm<'hir> {
+    pub(super) fn arm(
+        &mut self,
+        pat: &'hir hir::Pat<'hir>,
+        expr: &'hir hir::Expr<'hir>,
+    ) -> hir::Arm<'hir> {
         hir::Arm {
             hir_id: self.next_id(),
             pat,
diff --git a/compiler/rustc_ast_lowering/src/format.rs b/compiler/rustc_ast_lowering/src/format.rs
new file mode 100644
index 000000000..4095e225a
--- /dev/null
+++ b/compiler/rustc_ast_lowering/src/format.rs
@@ -0,0 +1,411 @@
+use super::LoweringContext;
+use rustc_ast as ast;
+use rustc_ast::visit::{self, Visitor};
+use rustc_ast::*;
+use rustc_data_structures::fx::FxIndexSet;
+use rustc_hir as hir;
+use rustc_span::{
+    sym,
+    symbol::{kw, Ident},
+    Span,
+};
+
+impl<'hir> LoweringContext<'_, 'hir> {
+    pub(crate) fn lower_format_args(&mut self, sp: Span, fmt: &FormatArgs) -> hir::ExprKind<'hir> {
+        expand_format_args(self, sp, fmt)
+    }
+}
+
+#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)]
+enum ArgumentType {
+    Format(FormatTrait),
+    Usize,
+}
+
+/// Generate a hir expression representing an argument to a format_args invocation.
+///
+/// Generates:
+///
+/// ```text
+///     <core::fmt::ArgumentV1>::new_…(arg)
+/// ```
+fn make_argument<'hir>(
+    ctx: &mut LoweringContext<'_, 'hir>,
+    sp: Span,
+    arg: &'hir hir::Expr<'hir>,
+    ty: ArgumentType,
+) -> hir::Expr<'hir> {
+    use ArgumentType::*;
+    use FormatTrait::*;
+    let new_fn = ctx.arena.alloc(ctx.expr_lang_item_type_relative(
+        sp,
+        hir::LangItem::FormatArgument,
+        match ty {
+            Format(Display) => sym::new_display,
+            Format(Debug) => sym::new_debug,
+            Format(LowerExp) => sym::new_lower_exp,
+            Format(UpperExp) => sym::new_upper_exp,
+            Format(Octal) => sym::new_octal,
+            Format(Pointer) => sym::new_pointer,
+            Format(Binary) => sym::new_binary,
+            Format(LowerHex) => sym::new_lower_hex,
+            Format(UpperHex) => sym::new_upper_hex,
+            Usize => sym::from_usize,
+        },
+    ));
+    ctx.expr_call_mut(sp, new_fn, std::slice::from_ref(arg))
+}
+
+/// Generate a hir expression for a format_args Count.
+///
+/// Generates:
+///
+/// ```text
+///     <core::fmt::rt::v1::Count>::Is(…)
+/// ```
+///
+/// or
+///
+/// ```text
+///     <core::fmt::rt::v1::Count>::Param(…)
+/// ```
+///
+/// or
+///
+/// ```text
+///     <core::fmt::rt::v1::Count>::Implied
+/// ```
+fn make_count<'hir>(
+    ctx: &mut LoweringContext<'_, 'hir>,
+    sp: Span,
+    count: &Option<FormatCount>,
+    argmap: &mut FxIndexSet<(usize, ArgumentType)>,
+) -> hir::Expr<'hir> {
+    match count {
+        Some(FormatCount::Literal(n)) => {
+            let count_is = ctx.arena.alloc(ctx.expr_lang_item_type_relative(
+                sp,
+                hir::LangItem::FormatCount,
+                sym::Is,
+            ));
+            let value = ctx.arena.alloc_from_iter([ctx.expr_usize(sp, *n)]);
+            ctx.expr_call_mut(sp, count_is, value)
+        }
+        Some(FormatCount::Argument(arg)) => {
+            if let Ok(arg_index) = arg.index {
+                let (i, _) = argmap.insert_full((arg_index, ArgumentType::Usize));
+                let count_param = ctx.arena.alloc(ctx.expr_lang_item_type_relative(
+                    sp,
+                    hir::LangItem::FormatCount,
+                    sym::Param,
+                ));
+                let value = ctx.arena.alloc_from_iter([ctx.expr_usize(sp, i)]);
+                ctx.expr_call_mut(sp, count_param, value)
+            } else {
+                ctx.expr(
+                    sp,
+                    hir::ExprKind::Err(
+                        ctx.tcx.sess.delay_span_bug(sp, "lowered bad format_args count"),
+                    ),
+                )
+            }
+        }
+        None => ctx.expr_lang_item_type_relative(sp, hir::LangItem::FormatCount, sym::Implied),
+    }
+}
+
+/// Generate a hir expression for a format_args placeholder specification.
+///
+/// Generates
+///
+/// ```text
+///     <core::fmt::rt::v1::Argument::new(
+///         …usize, // position
+///         '…', // fill
+///         <core::fmt::rt::v1::Alignment>::…, // alignment
+///         …u32, // flags
+///         <core::fmt::rt::v1::Count::…>, // width
+///         <core::fmt::rt::v1::Count::…>, // precision
+///     )
+/// ```
+fn make_format_spec<'hir>(
+    ctx: &mut LoweringContext<'_, 'hir>,
+    sp: Span,
+    placeholder: &FormatPlaceholder,
+    argmap: &mut FxIndexSet<(usize, ArgumentType)>,
+) -> hir::Expr<'hir> {
+    let position = match placeholder.argument.index {
+        Ok(arg_index) => {
+            let (i, _) =
+                argmap.insert_full((arg_index, ArgumentType::Format(placeholder.format_trait)));
+            ctx.expr_usize(sp, i)
+        }
+        Err(_) => ctx.expr(
+            sp,
+            hir::ExprKind::Err(ctx.tcx.sess.delay_span_bug(sp, "lowered bad format_args count")),
+        ),
+    };
+    let &FormatOptions {
+        ref width,
+        ref precision,
+        alignment,
+        fill,
+        sign,
+        alternate,
+        zero_pad,
+        debug_hex,
+    } = &placeholder.format_options;
+    let fill = ctx.expr_char(sp, fill.unwrap_or(' '));
+    let align = ctx.expr_lang_item_type_relative(
+        sp,
+        hir::LangItem::FormatAlignment,
+        match alignment {
+            Some(FormatAlignment::Left) => sym::Left,
+            Some(FormatAlignment::Right) => sym::Right,
+            Some(FormatAlignment::Center) => sym::Center,
+            None => sym::Unknown,
+        },
+    );
+    // This needs to match `FlagV1` in library/core/src/fmt/mod.rs.
+    let flags: u32 = ((sign == Some(FormatSign::Plus)) as u32)
+        | ((sign == Some(FormatSign::Minus)) as u32) << 1
+        | (alternate as u32) << 2
+        | (zero_pad as u32) << 3
+        | ((debug_hex == Some(FormatDebugHex::Lower)) as u32) << 4
+        | ((debug_hex == Some(FormatDebugHex::Upper)) as u32) << 5;
+    let flags = ctx.expr_u32(sp, flags);
+    let precision = make_count(ctx, sp, &precision, argmap);
+    let width = make_count(ctx, sp, &width, argmap);
+    let format_placeholder_new = ctx.arena.alloc(ctx.expr_lang_item_type_relative(
+        sp,
+        hir::LangItem::FormatPlaceholder,
+        sym::new,
+    ));
+    let args = ctx.arena.alloc_from_iter([position, fill, align, flags, precision, width]);
+    ctx.expr_call_mut(sp, format_placeholder_new, args)
+}
+
+fn expand_format_args<'hir>(
+    ctx: &mut LoweringContext<'_, 'hir>,
+    macsp: Span,
+    fmt: &FormatArgs,
+) -> hir::ExprKind<'hir> {
+    let lit_pieces =
+        ctx.arena.alloc_from_iter(fmt.template.iter().enumerate().filter_map(|(i, piece)| {
+            match piece {
+                &FormatArgsPiece::Literal(s) => Some(ctx.expr_str(fmt.span, s)),
+                &FormatArgsPiece::Placeholder(_) => {
+                    // Inject empty string before placeholders when not already preceded by a literal piece.
+                    if i == 0 || matches!(fmt.template[i - 1], FormatArgsPiece::Placeholder(_)) {
+                        Some(ctx.expr_str(fmt.span, kw::Empty))
+                    } else {
+                        None
+                    }
+                }
+            }
+        }));
+    let lit_pieces = ctx.expr_array_ref(fmt.span, lit_pieces);
+
+    // Whether we'll use the `Arguments::new_v1_formatted` form (true),
+    // or the `Arguments::new_v1` form (false).
+    let mut use_format_options = false;
+
+    // Create a list of all _unique_ (argument, format trait) combinations.
+    // E.g. "{0} {0:x} {0} {1}" -> [(0, Display), (0, LowerHex), (1, Display)]
+    let mut argmap = FxIndexSet::default();
+    for piece in &fmt.template {
+        let FormatArgsPiece::Placeholder(placeholder) = piece else { continue };
+        if placeholder.format_options != Default::default() {
+            // Can't use basic form if there's any formatting options.
+            use_format_options = true;
+        }
+        if let Ok(index) = placeholder.argument.index {
+            if !argmap.insert((index, ArgumentType::Format(placeholder.format_trait))) {
+                // Duplicate (argument, format trait) combination,
+                // which we'll only put once in the args array.
+                use_format_options = true;
+            }
+        }
+    }
+
+    let format_options = use_format_options.then(|| {
+        // Generate:
+        //     &[format_spec_0, format_spec_1, format_spec_2]
+        let elements: Vec<_> = fmt
+            .template
+            .iter()
+            .filter_map(|piece| {
+                let FormatArgsPiece::Placeholder(placeholder) = piece else { return None };
+                Some(make_format_spec(ctx, macsp, placeholder, &mut argmap))
+            })
+            .collect();
+        ctx.expr_array_ref(macsp, ctx.arena.alloc_from_iter(elements))
+    });
+
+    let arguments = fmt.arguments.all_args();
+
+    // If the args array contains exactly all the original arguments once,
+    // in order, we can use a simple array instead of a `match` construction.
+    // However, if there's a yield point in any argument except the first one,
+    // we don't do this, because an ArgumentV1 cannot be kept across yield points.
+    //
+    // This is an optimization, speeding up compilation about 1-2% in some cases.
+    // See https://github.com/rust-lang/rust/pull/106770#issuecomment-1380790609
+    let use_simple_array = argmap.len() == arguments.len()
+        && argmap.iter().enumerate().all(|(i, &(j, _))| i == j)
+        && arguments.iter().skip(1).all(|arg| !may_contain_yield_point(&arg.expr));
+
+    let args = if use_simple_array {
+        // Generate:
+        //     &[
+        //         <core::fmt::ArgumentV1>::new_display(&arg0),
+        //         <core::fmt::ArgumentV1>::new_lower_hex(&arg1),
+        //         <core::fmt::ArgumentV1>::new_debug(&arg2),
+        //         …
+        //     ]
+        let elements: Vec<_> = arguments
+            .iter()
+            .zip(argmap)
+            .map(|(arg, (_, ty))| {
+                let sp = arg.expr.span.with_ctxt(macsp.ctxt());
+                let arg = ctx.lower_expr(&arg.expr);
+                let ref_arg = ctx.arena.alloc(ctx.expr(
+                    sp,
+                    hir::ExprKind::AddrOf(hir::BorrowKind::Ref, hir::Mutability::Not, arg),
+                ));
+                make_argument(ctx, sp, ref_arg, ty)
+            })
+            .collect();
+        ctx.expr_array_ref(macsp, ctx.arena.alloc_from_iter(elements))
+    } else {
+        // Generate:
+        //     &match (&arg0, &arg1, &…) {
+        //         args => [
+        //             <core::fmt::ArgumentV1>::new_display(args.0),
+        //             <core::fmt::ArgumentV1>::new_lower_hex(args.1),
+        //             <core::fmt::ArgumentV1>::new_debug(args.0),
+        //             …
+        //         ]
+        //     }
+        let args_ident = Ident::new(sym::args, macsp);
+        let (args_pat, args_hir_id) = ctx.pat_ident(macsp, args_ident);
+        let args = ctx.arena.alloc_from_iter(argmap.iter().map(|&(arg_index, ty)| {
+            if let Some(arg) = arguments.get(arg_index) {
+                let sp = arg.expr.span.with_ctxt(macsp.ctxt());
+                let args_ident_expr = ctx.expr_ident(macsp, args_ident, args_hir_id);
+                let arg = ctx.arena.alloc(ctx.expr(
+                    sp,
+                    hir::ExprKind::Field(
+                        args_ident_expr,
+                        Ident::new(sym::integer(arg_index), macsp),
+                    ),
+                ));
+                make_argument(ctx, sp, arg, ty)
+            } else {
+                ctx.expr(
+                    macsp,
+                    hir::ExprKind::Err(
+                        ctx.tcx.sess.delay_span_bug(macsp, format!("no arg at {arg_index}")),
+                    ),
+                )
+            }
+        }));
+        let elements: Vec<_> = arguments
+            .iter()
+            .map(|arg| {
+                let arg_expr = ctx.lower_expr(&arg.expr);
+                ctx.expr(
+                    arg.expr.span.with_ctxt(macsp.ctxt()),
+                    hir::ExprKind::AddrOf(hir::BorrowKind::Ref, hir::Mutability::Not, arg_expr),
+                )
+            })
+            .collect();
+        let args_tuple = ctx
+            .arena
+            .alloc(ctx.expr(macsp, hir::ExprKind::Tup(ctx.arena.alloc_from_iter(elements))));
+        let array = ctx.arena.alloc(ctx.expr(macsp, hir::ExprKind::Array(args)));
+        let match_arms = ctx.arena.alloc_from_iter([ctx.arm(args_pat, array)]);
+        let match_expr = ctx.arena.alloc(ctx.expr_match(
+            macsp,
+            args_tuple,
+            match_arms,
+            hir::MatchSource::FormatArgs,
+        ));
+        ctx.expr(
+            macsp,
+            hir::ExprKind::AddrOf(hir::BorrowKind::Ref, hir::Mutability::Not, match_expr),
+        )
+    };
+
+    if let Some(format_options) = format_options {
+        // Generate:
+        //     <core::fmt::Arguments>::new_v1_formatted(
+        //         lit_pieces,
+        //         args,
+        //         format_options,
+        //         unsafe { ::core::fmt::UnsafeArg::new() }
+        //     )
+        let new_v1_formatted = ctx.arena.alloc(ctx.expr_lang_item_type_relative(
+            macsp,
+            hir::LangItem::FormatArguments,
+            sym::new_v1_formatted,
+        ));
+        let unsafe_arg_new = ctx.arena.alloc(ctx.expr_lang_item_type_relative(
+            macsp,
+            hir::LangItem::FormatUnsafeArg,
+            sym::new,
+        ));
+        let unsafe_arg_new_call = ctx.expr_call(macsp, unsafe_arg_new, &[]);
+        let hir_id = ctx.next_id();
+        let unsafe_arg = ctx.expr_block(ctx.arena.alloc(hir::Block {
+            stmts: &[],
+            expr: Some(unsafe_arg_new_call),
+            hir_id,
+            rules: hir::BlockCheckMode::UnsafeBlock(hir::UnsafeSource::CompilerGenerated),
+            span: macsp,
+            targeted_by_break: false,
+        }));
+        let args = ctx.arena.alloc_from_iter([lit_pieces, args, format_options, unsafe_arg]);
+        hir::ExprKind::Call(new_v1_formatted, args)
+    } else {
+        // Generate:
+        //     <core::fmt::Arguments>::new_v1(
+        //         lit_pieces,
+        //         args,
+        //     )
+        let new_v1 = ctx.arena.alloc(ctx.expr_lang_item_type_relative(
+            macsp,
+            hir::LangItem::FormatArguments,
+            sym::new_v1,
+        ));
+        let new_args = ctx.arena.alloc_from_iter([lit_pieces, args]);
+        hir::ExprKind::Call(new_v1, new_args)
+    }
+}
+
+fn may_contain_yield_point(e: &ast::Expr) -> bool {
+    struct MayContainYieldPoint(bool);
+
+    impl Visitor<'_> for MayContainYieldPoint {
+        fn visit_expr(&mut self, e: &ast::Expr) {
+            if let ast::ExprKind::Await(_) | ast::ExprKind::Yield(_) = e.kind {
+                self.0 = true;
+            } else {
+                visit::walk_expr(self, e);
+            }
+        }
+
+        fn visit_mac_call(&mut self, _: &ast::MacCall) {
+            // Macros should be expanded at this point.
+            unreachable!("unexpanded macro in ast lowering");
+        }
+
+        fn visit_item(&mut self, _: &ast::Item) {
+            // Do not recurse into nested items.
+        }
+    }
+
+    let mut visitor = MayContainYieldPoint(false);
+    visitor.visit_expr(e);
+    visitor.0
+}
diff --git a/compiler/rustc_ast_lowering/src/index.rs b/compiler/rustc_ast_lowering/src/index.rs
index 63033085b..f7fe0d771 100644
--- a/compiler/rustc_ast_lowering/src/index.rs
+++ b/compiler/rustc_ast_lowering/src/index.rs
@@ -275,19 +275,6 @@ impl<'a, 'hir> Visitor<'hir> for NodeCollector<'a, 'hir> {
         });
     }
 
-    fn visit_fn(
-        &mut self,
-        fk: intravisit::FnKind<'hir>,
-        fd: &'hir FnDecl<'hir>,
-        b: BodyId,
-        _: Span,
-        id: HirId,
-    ) {
-        assert_eq!(self.owner, id.owner);
-        assert_eq!(self.parent_node, id.local_id);
-        intravisit::walk_fn(self, fk, fd, b, id);
-    }
-
     fn visit_block(&mut self, block: &'hir Block<'hir>) {
         self.insert(block.span, block.hir_id, Node::Block(block));
         self.with_parent(block.hir_id, |this| {
diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
index 5d2589cb2..41295f2b7 100644
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@@ -7,13 +7,14 @@ use rustc_ast::ptr::P;
 use rustc_ast::visit::AssocCtxt;
 use rustc_ast::*;
 use rustc_data_structures::sorted_map::SortedMap;
+use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::{LocalDefId, CRATE_DEF_ID};
 use rustc_hir::PredicateOrigin;
 use rustc_index::vec::{Idx, IndexVec};
 use rustc_middle::ty::{DefIdTree, ResolverAstLowering, TyCtxt};
-use rustc_span::lev_distance::find_best_match_for_name;
+use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::source_map::DesugaringKind;
 use rustc_span::symbol::{kw, sym, Ident};
 use rustc_span::{Span, Symbol};
@@ -67,7 +68,6 @@ impl<'a, 'hir> ItemLowerer<'a, 'hir> {
             current_hir_id_owner: hir::CRATE_OWNER_ID,
             item_local_id_counter: hir::ItemLocalId::new(0),
             node_id_to_local_id: Default::default(),
-            local_id_to_def_id: SortedMap::new(),
             trait_map: Default::default(),
 
             // Lowering state.
@@ -285,7 +285,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     .alloc_from_iter(fm.items.iter().map(|x| self.lower_foreign_item_ref(x))),
             },
             ItemKind::GlobalAsm(asm) => hir::ItemKind::GlobalAsm(self.lower_inline_asm(span, asm)),
-            ItemKind::TyAlias(box TyAlias { generics, where_clauses, ty: Some(ty), .. }) => {
+            ItemKind::TyAlias(box TyAlias { generics, where_clauses, ty, .. }) => {
                 // We lower
                 //
                 // type Foo = impl Trait
@@ -300,18 +300,16 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     &generics,
                     id,
                     &ImplTraitContext::Disallowed(ImplTraitPosition::Generic),
-                    |this| this.lower_ty(ty, &ImplTraitContext::TypeAliasesOpaqueTy),
-                );
-                hir::ItemKind::TyAlias(ty, generics)
-            }
-            ItemKind::TyAlias(box TyAlias { generics, where_clauses, ty: None, .. }) => {
-                let mut generics = generics.clone();
-                add_ty_alias_where_clause(&mut generics, *where_clauses, true);
-                let (generics, ty) = self.lower_generics(
-                    &generics,
-                    id,
-                    &ImplTraitContext::Disallowed(ImplTraitPosition::Generic),
-                    |this| this.arena.alloc(this.ty(span, hir::TyKind::Err)),
+                    |this| match ty {
+                        None => {
+                            let guar = this.tcx.sess.delay_span_bug(
+                                span,
+                                "expected to lower type alias type, but it was missing",
+                            );
+                            this.arena.alloc(this.ty(span, hir::TyKind::Err(guar)))
+                        }
+                        Some(ty) => this.lower_ty(ty, &ImplTraitContext::TypeAliasesOpaqueTy),
+                    },
                 );
                 hir::ItemKind::TyAlias(ty, generics)
             }
@@ -379,8 +377,10 @@ impl<'hir> LoweringContext<'_, 'hir> {
                             )
                         });
 
-                        let lowered_ty = this
-                            .lower_ty(ty, &ImplTraitContext::Disallowed(ImplTraitPosition::Type));
+                        let lowered_ty = this.lower_ty(
+                            ty,
+                            &ImplTraitContext::Disallowed(ImplTraitPosition::ImplSelf),
+                        );
 
                         (trait_ref, lowered_ty)
                     });
@@ -459,7 +459,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         span: Span,
         body: Option<&Expr>,
     ) -> (&'hir hir::Ty<'hir>, hir::BodyId) {
-        let ty = self.lower_ty(ty, &ImplTraitContext::Disallowed(ImplTraitPosition::Type));
+        let ty = self.lower_ty(ty, &ImplTraitContext::Disallowed(ImplTraitPosition::ConstTy));
         (ty, self.lower_const_body(span, body))
     }
 
@@ -609,8 +609,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     hir::ForeignItemKind::Fn(fn_dec, fn_args, generics)
                 }
                 ForeignItemKind::Static(t, m, _) => {
-                    let ty =
-                        self.lower_ty(t, &ImplTraitContext::Disallowed(ImplTraitPosition::Type));
+                    let ty = self
+                        .lower_ty(t, &ImplTraitContext::Disallowed(ImplTraitPosition::StaticTy));
                     hir::ForeignItemKind::Static(ty, *m)
                 }
                 ForeignItemKind::TyAlias(..) => hir::ForeignItemKind::Type,
@@ -680,11 +680,11 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 qself,
                 path,
                 ParamMode::ExplicitNamed, // no `'_` in declarations (Issue #61124)
-                &ImplTraitContext::Disallowed(ImplTraitPosition::Path),
+                &ImplTraitContext::Disallowed(ImplTraitPosition::FieldTy),
             );
             self.arena.alloc(t)
         } else {
-            self.lower_ty(&f.ty, &ImplTraitContext::Disallowed(ImplTraitPosition::Type))
+            self.lower_ty(&f.ty, &ImplTraitContext::Disallowed(ImplTraitPosition::FieldTy))
         };
         let hir_id = self.lower_node_id(f.id);
         self.lower_attrs(hir_id, &f.attrs);
@@ -709,7 +709,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
         let (generics, kind, has_default) = match &i.kind {
             AssocItemKind::Const(_, ty, default) => {
-                let ty = self.lower_ty(ty, &ImplTraitContext::Disallowed(ImplTraitPosition::Type));
+                let ty =
+                    self.lower_ty(ty, &ImplTraitContext::Disallowed(ImplTraitPosition::ConstTy));
                 let body = default.as_ref().map(|x| self.lower_const_body(i.span, Some(x)));
                 (hir::Generics::empty(), hir::TraitItemKind::Const(ty, body), body.is_some())
             }
@@ -747,7 +748,10 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     &ImplTraitContext::Disallowed(ImplTraitPosition::Generic),
                     |this| {
                         let ty = ty.as_ref().map(|x| {
-                            this.lower_ty(x, &ImplTraitContext::Disallowed(ImplTraitPosition::Type))
+                            this.lower_ty(
+                                x,
+                                &ImplTraitContext::Disallowed(ImplTraitPosition::AssocTy),
+                            )
                         });
                         hir::TraitItemKind::Type(
                             this.lower_param_bounds(
@@ -793,8 +797,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
     }
 
     /// Construct `ExprKind::Err` for the given `span`.
-    pub(crate) fn expr_err(&mut self, span: Span) -> hir::Expr<'hir> {
-        self.expr(span, hir::ExprKind::Err)
+    pub(crate) fn expr_err(&mut self, span: Span, guar: ErrorGuaranteed) -> hir::Expr<'hir> {
+        self.expr(span, hir::ExprKind::Err(guar))
     }
 
     fn lower_impl_item(&mut self, i: &AssocItem) -> &'hir hir::ImplItem<'hir> {
@@ -806,7 +810,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
         let (generics, kind) = match &i.kind {
             AssocItemKind::Const(_, ty, expr) => {
-                let ty = self.lower_ty(ty, &ImplTraitContext::Disallowed(ImplTraitPosition::Type));
+                let ty =
+                    self.lower_ty(ty, &ImplTraitContext::Disallowed(ImplTraitPosition::ConstTy));
                 (
                     hir::Generics::empty(),
                     hir::ImplItemKind::Const(ty, self.lower_const_body(i.span, expr.as_deref())),
@@ -841,7 +846,11 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     &ImplTraitContext::Disallowed(ImplTraitPosition::Generic),
                     |this| match ty {
                         None => {
-                            let ty = this.arena.alloc(this.ty(i.span, hir::TyKind::Err));
+                            let guar = this.tcx.sess.delay_span_bug(
+                                i.span,
+                                "expected to lower associated type, but it was missing",
+                            );
+                            let ty = this.arena.alloc(this.ty(i.span, hir::TyKind::Err(guar)));
                             hir::ImplItemKind::Type(ty)
                         }
                         Some(ty) => {
@@ -967,7 +976,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
     fn lower_block_expr_opt(&mut self, span: Span, block: Option<&Block>) -> hir::Expr<'hir> {
         match block {
             Some(block) => self.lower_block_expr(block),
-            None => self.expr_err(span),
+            None => self.expr_err(span, self.tcx.sess.delay_span_bug(span, "no block")),
         }
     }
 
@@ -977,7 +986,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 &[],
                 match expr {
                     Some(expr) => this.lower_expr_mut(expr),
-                    None => this.expr_err(span),
+                    None => this.expr_err(span, this.tcx.sess.delay_span_bug(span, "no block")),
                 },
             )
         })
@@ -1330,13 +1339,19 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 .map(|predicate| self.lower_where_predicate(predicate)),
         );
 
-        let mut params: SmallVec<[hir::GenericParam<'hir>; 4]> =
-            self.lower_generic_params_mut(&generics.params).collect();
+        let mut params: SmallVec<[hir::GenericParam<'hir>; 4]> = self
+            .lower_generic_params_mut(&generics.params, hir::GenericParamSource::Generics)
+            .collect();
 
         // Introduce extra lifetimes if late resolution tells us to.
         let extra_lifetimes = self.resolver.take_extra_lifetime_params(parent_node_id);
         params.extend(extra_lifetimes.into_iter().filter_map(|(ident, node_id, res)| {
-            self.lifetime_res_to_generic_param(ident, node_id, res)
+            self.lifetime_res_to_generic_param(
+                ident,
+                node_id,
+                res,
+                hir::GenericParamSource::Generics,
+            )
         }));
 
         let has_where_clause_predicates = !generics.where_clause.predicates.is_empty();
@@ -1440,9 +1455,10 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 span,
             }) => hir::WherePredicate::BoundPredicate(hir::WhereBoundPredicate {
                 hir_id: self.next_id(),
-                bound_generic_params: self.lower_generic_params(bound_generic_params),
+                bound_generic_params: self
+                    .lower_generic_params(bound_generic_params, hir::GenericParamSource::Binder),
                 bounded_ty: self
-                    .lower_ty(bounded_ty, &ImplTraitContext::Disallowed(ImplTraitPosition::Type)),
+                    .lower_ty(bounded_ty, &ImplTraitContext::Disallowed(ImplTraitPosition::Bound)),
                 bounds: self.arena.alloc_from_iter(bounds.iter().map(|bound| {
                     self.lower_param_bound(
                         bound,
@@ -1466,9 +1482,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
             WherePredicate::EqPredicate(WhereEqPredicate { lhs_ty, rhs_ty, span }) => {
                 hir::WherePredicate::EqPredicate(hir::WhereEqPredicate {
                     lhs_ty: self
-                        .lower_ty(lhs_ty, &ImplTraitContext::Disallowed(ImplTraitPosition::Type)),
+                        .lower_ty(lhs_ty, &ImplTraitContext::Disallowed(ImplTraitPosition::Bound)),
                     rhs_ty: self
-                        .lower_ty(rhs_ty, &ImplTraitContext::Disallowed(ImplTraitPosition::Type)),
+                        .lower_ty(rhs_ty, &ImplTraitContext::Disallowed(ImplTraitPosition::Bound)),
                     span: self.lower_span(*span),
                 })
             }
diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs
index bc6d2cf12..b20157f2c 100644
--- a/compiler/rustc_ast_lowering/src/lib.rs
+++ b/compiler/rustc_ast_lowering/src/lib.rs
@@ -52,23 +52,28 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sorted_map::SortedMap;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::Lrc;
-use rustc_errors::{DiagnosticArgFromDisplay, Handler, StashKey};
+use rustc_errors::{
+    DiagnosticArgFromDisplay, DiagnosticMessage, Handler, StashKey, SubdiagnosticMessage,
+};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, LifetimeRes, Namespace, PartialRes, PerNS, Res};
 use rustc_hir::def_id::{LocalDefId, CRATE_DEF_ID};
 use rustc_hir::definitions::DefPathData;
 use rustc_hir::{ConstArg, GenericArg, ItemLocalId, ParamName, TraitCandidate};
 use rustc_index::vec::{Idx, IndexVec};
-use rustc_middle::span_bug;
-use rustc_middle::ty::{ResolverAstLowering, TyCtxt};
+use rustc_macros::fluent_messages;
+use rustc_middle::{
+    span_bug,
+    ty::{ResolverAstLowering, TyCtxt},
+};
 use rustc_session::parse::feature_err;
 use rustc_span::hygiene::MacroKind;
 use rustc_span::source_map::DesugaringKind;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
-
 use smallvec::SmallVec;
 use std::collections::hash_map::Entry;
+use thin_vec::ThinVec;
 
 macro_rules! arena_vec {
     ($this:expr; $($x:expr),*) => (
@@ -80,12 +85,15 @@ mod asm;
 mod block;
 mod errors;
 mod expr;
+mod format;
 mod index;
 mod item;
 mod lifetime_collector;
 mod pat;
 mod path;
 
+fluent_messages! { "../locales/en-US.ftl" }
+
 struct LoweringContext<'a, 'hir> {
     tcx: TyCtxt<'hir>,
     resolver: &'a mut ResolverAstLowering,
@@ -118,7 +126,6 @@ struct LoweringContext<'a, 'hir> {
 
     current_hir_id_owner: hir::OwnerId,
     item_local_id_counter: hir::ItemLocalId,
-    local_id_to_def_id: SortedMap<ItemLocalId, LocalDefId>,
     trait_map: FxHashMap<ItemLocalId, Box<[TraitCandidate]>>,
 
     impl_trait_defs: Vec<hir::GenericParam<'hir>>,
@@ -253,7 +260,6 @@ enum ImplTraitContext {
 enum ImplTraitPosition {
     Path,
     Variable,
-    Type,
     Trait,
     AsyncBlock,
     Bound,
@@ -270,30 +276,43 @@ enum ImplTraitPosition {
     FnTraitReturn,
     TraitReturn,
     ImplReturn,
+    GenericDefault,
+    ConstTy,
+    StaticTy,
+    AssocTy,
+    FieldTy,
+    Cast,
+    ImplSelf,
 }
 
 impl std::fmt::Display for ImplTraitPosition {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let name = match self {
-            ImplTraitPosition::Path => "path",
-            ImplTraitPosition::Variable => "variable binding",
-            ImplTraitPosition::Type => "type",
-            ImplTraitPosition::Trait => "trait",
-            ImplTraitPosition::AsyncBlock => "async block",
-            ImplTraitPosition::Bound => "bound",
-            ImplTraitPosition::Generic => "generic",
-            ImplTraitPosition::ExternFnParam => "`extern fn` param",
-            ImplTraitPosition::ClosureParam => "closure param",
-            ImplTraitPosition::PointerParam => "`fn` pointer param",
-            ImplTraitPosition::FnTraitParam => "`Fn` trait param",
-            ImplTraitPosition::TraitParam => "trait method param",
-            ImplTraitPosition::ImplParam => "`impl` method param",
-            ImplTraitPosition::ExternFnReturn => "`extern fn` return",
-            ImplTraitPosition::ClosureReturn => "closure return",
-            ImplTraitPosition::PointerReturn => "`fn` pointer return",
-            ImplTraitPosition::FnTraitReturn => "`Fn` trait return",
-            ImplTraitPosition::TraitReturn => "trait method return",
-            ImplTraitPosition::ImplReturn => "`impl` method return",
+            ImplTraitPosition::Path => "paths",
+            ImplTraitPosition::Variable => "variable bindings",
+            ImplTraitPosition::Trait => "traits",
+            ImplTraitPosition::AsyncBlock => "async blocks",
+            ImplTraitPosition::Bound => "bounds",
+            ImplTraitPosition::Generic => "generics",
+            ImplTraitPosition::ExternFnParam => "`extern fn` params",
+            ImplTraitPosition::ClosureParam => "closure params",
+            ImplTraitPosition::PointerParam => "`fn` pointer params",
+            ImplTraitPosition::FnTraitParam => "`Fn` trait params",
+            ImplTraitPosition::TraitParam => "trait method params",
+            ImplTraitPosition::ImplParam => "`impl` method params",
+            ImplTraitPosition::ExternFnReturn => "`extern fn` return types",
+            ImplTraitPosition::ClosureReturn => "closure return types",
+            ImplTraitPosition::PointerReturn => "`fn` pointer return types",
+            ImplTraitPosition::FnTraitReturn => "`Fn` trait return types",
+            ImplTraitPosition::TraitReturn => "trait method return types",
+            ImplTraitPosition::ImplReturn => "`impl` method return types",
+            ImplTraitPosition::GenericDefault => "generic parameter defaults",
+            ImplTraitPosition::ConstTy => "const types",
+            ImplTraitPosition::StaticTy => "static types",
+            ImplTraitPosition::AssocTy => "associated types",
+            ImplTraitPosition::FieldTy => "field types",
+            ImplTraitPosition::Cast => "cast types",
+            ImplTraitPosition::ImplSelf => "impl headers",
         };
 
         write!(f, "{name}")
@@ -416,6 +435,7 @@ fn compute_hir_hash(
 
 pub fn lower_to_hir(tcx: TyCtxt<'_>, (): ()) -> hir::Crate<'_> {
     let sess = tcx.sess;
+    tcx.ensure().output_filenames(());
     let (mut resolver, krate) = tcx.resolver_for_lowering(()).steal();
 
     let ast_index = index_crate(&resolver.node_id_to_def_id, &krate);
@@ -565,7 +585,6 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         let current_attrs = std::mem::take(&mut self.attrs);
         let current_bodies = std::mem::take(&mut self.bodies);
         let current_node_ids = std::mem::take(&mut self.node_id_to_local_id);
-        let current_id_to_def_id = std::mem::take(&mut self.local_id_to_def_id);
         let current_trait_map = std::mem::take(&mut self.trait_map);
         let current_owner =
             std::mem::replace(&mut self.current_hir_id_owner, hir::OwnerId { def_id });
@@ -592,7 +611,6 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         self.attrs = current_attrs;
         self.bodies = current_bodies;
         self.node_id_to_local_id = current_node_ids;
-        self.local_id_to_def_id = current_id_to_def_id;
         self.trait_map = current_trait_map;
         self.current_hir_id_owner = current_owner;
         self.item_local_id_counter = current_local_counter;
@@ -627,7 +645,6 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
     fn make_owner_info(&mut self, node: hir::OwnerNode<'hir>) -> &'hir hir::OwnerInfo<'hir> {
         let attrs = std::mem::take(&mut self.attrs);
         let mut bodies = std::mem::take(&mut self.bodies);
-        let local_id_to_def_id = std::mem::take(&mut self.local_id_to_def_id);
         let trait_map = std::mem::take(&mut self.trait_map);
 
         #[cfg(debug_assertions)]
@@ -643,13 +660,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         let (hash_including_bodies, hash_without_bodies) = self.hash_owner(node, &bodies);
         let (nodes, parenting) =
             index::index_hir(self.tcx.sess, &*self.tcx.definitions_untracked(), node, &bodies);
-        let nodes = hir::OwnerNodes {
-            hash_including_bodies,
-            hash_without_bodies,
-            nodes,
-            bodies,
-            local_id_to_def_id,
-        };
+        let nodes = hir::OwnerNodes { hash_including_bodies, hash_without_bodies, nodes, bodies };
         let attrs = {
             let hash = self.tcx.with_stable_hashing_context(|mut hcx| {
                 let mut stable_hasher = StableHasher::new();
@@ -708,7 +719,6 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 assert_ne!(local_id, hir::ItemLocalId::new(0));
                 if let Some(def_id) = self.opt_local_def_id(ast_node_id) {
                     self.children.push((def_id, hir::MaybeOwner::NonOwner(hir_id)));
-                    self.local_id_to_def_id.insert(local_id, def_id);
                 }
 
                 if let Some(traits) = self.resolver.trait_map.remove(&ast_node_id) {
@@ -794,6 +804,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         ident: Ident,
         node_id: NodeId,
         res: LifetimeRes,
+        source: hir::GenericParamSource,
     ) -> Option<hir::GenericParam<'hir>> {
         let (name, kind) = match res {
             LifetimeRes::Param { .. } => {
@@ -827,6 +838,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             pure_wrt_drop: false,
             kind: hir::GenericParamKind::Lifetime { kind },
             colon_span: None,
+            source,
         })
     }
 
@@ -842,11 +854,13 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         binder: NodeId,
         generic_params: &[GenericParam],
     ) -> &'hir [hir::GenericParam<'hir>] {
-        let mut generic_params: Vec<_> = self.lower_generic_params_mut(generic_params).collect();
+        let mut generic_params: Vec<_> = self
+            .lower_generic_params_mut(generic_params, hir::GenericParamSource::Binder)
+            .collect();
         let extra_lifetimes = self.resolver.take_extra_lifetime_params(binder);
         debug!(?extra_lifetimes);
         generic_params.extend(extra_lifetimes.into_iter().filter_map(|(ident, node_id, res)| {
-            self.lifetime_res_to_generic_param(ident, node_id, res)
+            self.lifetime_res_to_generic_param(ident, node_id, res, hir::GenericParamSource::Binder)
         }));
         let generic_params = self.arena.alloc_from_iter(generic_params);
         debug!(?generic_params);
@@ -992,8 +1006,6 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         } else {
             self.arena.alloc(hir::GenericArgs::none())
         };
-        let itctx_tait = &ImplTraitContext::TypeAliasesOpaqueTy;
-
         let kind = match &constraint.kind {
             AssocConstraintKind::Equality { term } => {
                 let term = match term {
@@ -1003,8 +1015,14 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 hir::TypeBindingKind::Equality { term }
             }
             AssocConstraintKind::Bound { bounds } => {
+                enum DesugarKind<'a> {
+                    ImplTrait,
+                    Error(&'a ImplTraitPosition),
+                    Bound,
+                }
+
                 // Piggy-back on the `impl Trait` context to figure out the correct behavior.
-                let (desugar_to_impl_trait, itctx) = match itctx {
+                let desugar_kind = match itctx {
                     // We are in the return position:
                     //
                     //     fn foo() -> impl Iterator<Item: Debug>
@@ -1013,7 +1031,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     //
                     //     fn foo() -> impl Iterator<Item = impl Debug>
                     ImplTraitContext::ReturnPositionOpaqueTy { .. }
-                    | ImplTraitContext::TypeAliasesOpaqueTy { .. } => (true, itctx),
+                    | ImplTraitContext::TypeAliasesOpaqueTy { .. } => DesugarKind::ImplTrait,
 
                     // We are in the argument position, but within a dyn type:
                     //
@@ -1022,15 +1040,11 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     // so desugar to
                     //
                     //     fn foo(x: dyn Iterator<Item = impl Debug>)
-                    ImplTraitContext::Universal if self.is_in_dyn_type => (true, itctx),
+                    ImplTraitContext::Universal if self.is_in_dyn_type => DesugarKind::ImplTrait,
 
-                    // In `type Foo = dyn Iterator<Item: Debug>` we desugar to
-                    // `type Foo = dyn Iterator<Item = impl Debug>` but we have to override the
-                    // "impl trait context" to permit `impl Debug` in this position (it desugars
-                    // then to an opaque type).
-                    //
-                    // FIXME: this is only needed until `impl Trait` is allowed in type aliases.
-                    ImplTraitContext::Disallowed(_) if self.is_in_dyn_type => (true, itctx_tait),
+                    ImplTraitContext::Disallowed(position) if self.is_in_dyn_type => {
+                        DesugarKind::Error(position)
+                    }
 
                     // We are in the parameter position, but not within a dyn type:
                     //
@@ -1039,35 +1053,47 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     // so we leave it as is and this gets expanded in astconv to a bound like
                     // `<T as Iterator>::Item: Debug` where `T` is the type parameter for the
                     // `impl Iterator`.
-                    _ => (false, itctx),
+                    _ => DesugarKind::Bound,
                 };
 
-                if desugar_to_impl_trait {
-                    // Desugar `AssocTy: Bounds` into `AssocTy = impl Bounds`. We do this by
-                    // constructing the HIR for `impl bounds...` and then lowering that.
-
-                    let impl_trait_node_id = self.next_node_id();
-
-                    self.with_dyn_type_scope(false, |this| {
-                        let node_id = this.next_node_id();
-                        let ty = this.lower_ty(
-                            &Ty {
-                                id: node_id,
-                                kind: TyKind::ImplTrait(impl_trait_node_id, bounds.clone()),
-                                span: this.lower_span(constraint.span),
-                                tokens: None,
-                            },
-                            itctx,
-                        );
+                match desugar_kind {
+                    DesugarKind::ImplTrait => {
+                        // Desugar `AssocTy: Bounds` into `AssocTy = impl Bounds`. We do this by
+                        // constructing the HIR for `impl bounds...` and then lowering that.
 
-                        hir::TypeBindingKind::Equality { term: ty.into() }
-                    })
-                } else {
-                    // Desugar `AssocTy: Bounds` into a type binding where the
-                    // later desugars into a trait predicate.
-                    let bounds = self.lower_param_bounds(bounds, itctx);
+                        let impl_trait_node_id = self.next_node_id();
+
+                        self.with_dyn_type_scope(false, |this| {
+                            let node_id = this.next_node_id();
+                            let ty = this.lower_ty(
+                                &Ty {
+                                    id: node_id,
+                                    kind: TyKind::ImplTrait(impl_trait_node_id, bounds.clone()),
+                                    span: this.lower_span(constraint.span),
+                                    tokens: None,
+                                },
+                                itctx,
+                            );
+
+                            hir::TypeBindingKind::Equality { term: ty.into() }
+                        })
+                    }
+                    DesugarKind::Bound => {
+                        // Desugar `AssocTy: Bounds` into a type binding where the
+                        // later desugars into a trait predicate.
+                        let bounds = self.lower_param_bounds(bounds, itctx);
 
-                    hir::TypeBindingKind::Constraint { bounds }
+                        hir::TypeBindingKind::Constraint { bounds }
+                    }
+                    DesugarKind::Error(position) => {
+                        let guar = self.tcx.sess.emit_err(errors::MisplacedAssocTyBinding {
+                            span: constraint.span,
+                            position: DiagnosticArgFromDisplay(position),
+                        });
+                        let err_ty =
+                            &*self.arena.alloc(self.ty(constraint.span, hir::TyKind::Err(guar)));
+                        hir::TypeBindingKind::Equality { term: err_ty.into() }
+                    }
                 }
             }
         };
@@ -1204,7 +1230,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             let (bounds, lifetime_bound) = self.with_dyn_type_scope(true, |this| {
                 let bound = this.lower_poly_trait_ref(
                     &PolyTraitRef {
-                        bound_generic_params: vec![],
+                        bound_generic_params: ThinVec::new(),
                         trait_ref: TraitRef { path: path.clone(), ref_id: t.id },
                         span: t.span
                     },
@@ -1234,7 +1260,9 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
     fn lower_ty_direct(&mut self, t: &Ty, itctx: &ImplTraitContext) -> hir::Ty<'hir> {
         let kind = match &t.kind {
             TyKind::Infer => hir::TyKind::Infer,
-            TyKind::Err => hir::TyKind::Err,
+            TyKind::Err => {
+                hir::TyKind::Err(self.tcx.sess.delay_span_bug(t.span, "TyKind::Err lowered"))
+            }
             TyKind::Slice(ty) => hir::TyKind::Slice(self.lower_ty(ty, itctx)),
             TyKind::Ptr(mt) => hir::TyKind::Ptr(self.lower_mt(mt, itctx)),
             TyKind::Ref(region, mt) => {
@@ -1351,8 +1379,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                             span,
                         );
                         let ident = Ident::from_str_and_span(&pprust::ty_to_string(t), span);
-                        let (param, bounds, path) =
-                            self.lower_generic_and_bounds(*def_node_id, span, ident, bounds);
+                        let (param, bounds, path) = self.lower_universal_param_and_bounds(
+                            *def_node_id,
+                            span,
+                            ident,
+                            bounds,
+                        );
                         self.impl_trait_defs.push(param);
                         if let Some(bounds) = bounds {
                             self.impl_trait_bounds.push(bounds);
@@ -1360,7 +1392,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                         path
                     }
                     ImplTraitContext::FeatureGated(position, feature) => {
-                        self.tcx
+                        let guar = self
+                            .tcx
                             .sess
                             .create_feature_err(
                                 MisplacedImplTrait {
@@ -1370,24 +1403,24 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                                 *feature,
                             )
                             .emit();
-                        hir::TyKind::Err
+                        hir::TyKind::Err(guar)
                     }
                     ImplTraitContext::Disallowed(position) => {
-                        self.tcx.sess.emit_err(MisplacedImplTrait {
+                        let guar = self.tcx.sess.emit_err(MisplacedImplTrait {
                             span: t.span,
                             position: DiagnosticArgFromDisplay(position),
                         });
-                        hir::TyKind::Err
+                        hir::TyKind::Err(guar)
                     }
                 }
             }
             TyKind::MacCall(_) => panic!("`TyKind::MacCall` should have been expanded by now"),
             TyKind::CVarArgs => {
-                self.tcx.sess.delay_span_bug(
+                let guar = self.tcx.sess.delay_span_bug(
                     t.span,
                     "`TyKind::CVarArgs` should have been handled elsewhere",
                 );
-                hir::TyKind::Err
+                hir::TyKind::Err(guar)
             }
         };
 
@@ -1505,6 +1538,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                             pure_wrt_drop: false,
                             kind: hir::GenericParamKind::Lifetime { kind },
                             colon_span: None,
+                            source: hir::GenericParamSource::Generics,
                         }
                     },
                 ));
@@ -1573,7 +1607,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         hir::OwnerNode::Item(self.arena.alloc(opaque_ty_item))
     }
 
-    /// Given a `parent_def_id`, a list of `lifetimes_in_bounds and a `remapping` hash to be
+    /// Given a `parent_def_id`, a list of `lifetimes_in_bounds` and a `remapping` hash to be
     /// filled, this function creates new definitions for `Param` and `Fresh` lifetimes, inserts the
     /// new definition, adds it to the remapping with the definition of the given lifetime and
     /// returns a list of lifetimes to be lowered afterwards.
@@ -1962,6 +1996,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                             pure_wrt_drop: false,
                             kind: hir::GenericParamKind::Lifetime { kind },
                             colon_span: None,
+                            source: hir::GenericParamSource::Generics,
                         }
                     },
                 ));
@@ -2127,16 +2162,25 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
     fn lower_generic_params_mut<'s>(
         &'s mut self,
         params: &'s [GenericParam],
+        source: hir::GenericParamSource,
     ) -> impl Iterator<Item = hir::GenericParam<'hir>> + Captures<'a> + Captures<'s> {
-        params.iter().map(move |param| self.lower_generic_param(param))
+        params.iter().map(move |param| self.lower_generic_param(param, source))
     }
 
-    fn lower_generic_params(&mut self, params: &[GenericParam]) -> &'hir [hir::GenericParam<'hir>] {
-        self.arena.alloc_from_iter(self.lower_generic_params_mut(params))
+    fn lower_generic_params(
+        &mut self,
+        params: &[GenericParam],
+        source: hir::GenericParamSource,
+    ) -> &'hir [hir::GenericParam<'hir>] {
+        self.arena.alloc_from_iter(self.lower_generic_params_mut(params, source))
     }
 
     #[instrument(level = "trace", skip(self))]
-    fn lower_generic_param(&mut self, param: &GenericParam) -> hir::GenericParam<'hir> {
+    fn lower_generic_param(
+        &mut self,
+        param: &GenericParam,
+        source: hir::GenericParamSource,
+    ) -> hir::GenericParam<'hir> {
         let (name, kind) = self.lower_generic_param_kind(param);
 
         let hir_id = self.lower_node_id(param.id);
@@ -2149,6 +2193,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             pure_wrt_drop: self.tcx.sess.contains_name(&param.attrs, sym::may_dangle),
             kind,
             colon_span: param.colon_span.map(|s| self.lower_span(s)),
+            source,
         }
     }
 
@@ -2175,7 +2220,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             GenericParamKind::Type { default, .. } => {
                 let kind = hir::GenericParamKind::Type {
                     default: default.as_ref().map(|x| {
-                        self.lower_ty(x, &ImplTraitContext::Disallowed(ImplTraitPosition::Type))
+                        self.lower_ty(
+                            x,
+                            &ImplTraitContext::Disallowed(ImplTraitPosition::GenericDefault),
+                        )
                     }),
                     synthetic: false,
                 };
@@ -2183,7 +2231,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 (hir::ParamName::Plain(self.lower_ident(param.ident)), kind)
             }
             GenericParamKind::Const { ty, kw_span: _, default } => {
-                let ty = self.lower_ty(&ty, &ImplTraitContext::Disallowed(ImplTraitPosition::Type));
+                let ty = self.lower_ty(
+                    &ty,
+                    &ImplTraitContext::Disallowed(ImplTraitPosition::GenericDefault),
+                );
                 let default = default.as_ref().map(|def| self.lower_anon_const(def));
                 (
                     hir::ParamName::Plain(self.lower_ident(param.ident)),
@@ -2235,7 +2286,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
     }
 
     #[instrument(level = "debug", skip(self), ret)]
-    fn lower_generic_and_bounds(
+    fn lower_universal_param_and_bounds(
         &mut self,
         node_id: NodeId,
         span: Span,
@@ -2255,6 +2306,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             span,
             kind: hir::GenericParamKind::Type { default: None, synthetic: true },
             colon_span: None,
+            source: hir::GenericParamSource::Generics,
         };
 
         let preds = self.lower_generic_bound_predicate(
diff --git a/compiler/rustc_ast_lowering/src/pat.rs b/compiler/rustc_ast_lowering/src/pat.rs
index 06d885a45..2509b7056 100644
--- a/compiler/rustc_ast_lowering/src/pat.rs
+++ b/compiler/rustc_ast_lowering/src/pat.rs
@@ -330,8 +330,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             ExprKind::Path(..) if allow_paths => {}
             ExprKind::Unary(UnOp::Neg, inner) if matches!(inner.kind, ExprKind::Lit(_)) => {}
             _ => {
-                self.tcx.sess.emit_err(ArbitraryExpressionInPattern { span: expr.span });
-                return self.arena.alloc(self.expr_err(expr.span));
+                let guar = self.tcx.sess.emit_err(ArbitraryExpressionInPattern { span: expr.span });
+                return self.arena.alloc(self.expr_err(expr.span, guar));
             }
         }
         self.lower_expr(expr)
diff --git a/compiler/rustc_ast_passes/Cargo.toml b/compiler/rustc_ast_passes/Cargo.toml
index 37eff9207..8bd212073 100644
--- a/compiler/rustc_ast_passes/Cargo.toml
+++ b/compiler/rustc_ast_passes/Cargo.toml
@@ -5,7 +5,7 @@ edition = "2021"
 
 [dependencies]
 itertools = "0.10.1"
-tracing = "0.1"
+rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_attr = { path = "../rustc_attr" }
 rustc_data_structures = { path = "../rustc_data_structures" }
@@ -16,4 +16,5 @@ rustc_parse = { path = "../rustc_parse" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
-rustc_ast = { path = "../rustc_ast" }
+thin-vec = "0.2.12"
+tracing = "0.1"
diff --git a/compiler/rustc_ast_passes/locales/en-US.ftl b/compiler/rustc_ast_passes/locales/en-US.ftl
new file mode 100644
index 000000000..747bd52b2
--- /dev/null
+++ b/compiler/rustc_ast_passes/locales/en-US.ftl
@@ -0,0 +1,236 @@
+ast_passes_forbidden_let =
+    `let` expressions are not supported here
+    .note = only supported directly in conditions of `if` and `while` expressions
+    .not_supported_or = `||` operators are not supported in let chain expressions
+    .not_supported_parentheses = `let`s wrapped in parentheses are not supported in a context with let chains
+
+ast_passes_forbidden_let_stable =
+    expected expression, found statement (`let`)
+    .note = variable declaration using `let` is a statement
+
+ast_passes_deprecated_where_clause_location =
+    where clause not allowed here
+
+ast_passes_keyword_lifetime =
+    lifetimes cannot use keyword names
+
+ast_passes_invalid_label =
+    invalid label name `{$name}`
+
+ast_passes_invalid_visibility =
+    unnecessary visibility qualifier
+    .implied = `pub` not permitted here because it's implied
+    .individual_impl_items = place qualifiers on individual impl items instead
+    .individual_foreign_items = place qualifiers on individual foreign items instead
+
+ast_passes_trait_fn_const =
+    functions in traits cannot be declared const
+    .label = functions in traits cannot be const
+
+ast_passes_forbidden_lifetime_bound =
+    lifetime bounds cannot be used in this context
+
+ast_passes_forbidden_non_lifetime_param =
+    only lifetime parameters can be used in this context
+
+ast_passes_fn_param_too_many =
+    function can not have more than {$max_num_args} arguments
+
+ast_passes_fn_param_c_var_args_only =
+    C-variadic function must be declared with at least one named argument
+
+ast_passes_fn_param_c_var_args_not_last =
+    `...` must be the last argument of a C-variadic function
+
+ast_passes_fn_param_doc_comment =
+    documentation comments cannot be applied to function parameters
+    .label = doc comments are not allowed here
+
+ast_passes_fn_param_forbidden_attr =
+    allow, cfg, cfg_attr, deny, expect, forbid, and warn are the only allowed built-in attributes in function parameters
+
+ast_passes_fn_param_forbidden_self =
+    `self` parameter is only allowed in associated functions
+    .label = not semantically valid as function parameter
+    .note = associated functions are those in `impl` or `trait` definitions
+
+ast_passes_forbidden_default =
+    `default` is only allowed on items in trait impls
+    .label = `default` because of this
+
+ast_passes_assoc_const_without_body =
+    associated constant in `impl` without body
+    .suggestion = provide a definition for the constant
+
+ast_passes_assoc_fn_without_body =
+    associated function in `impl` without body
+    .suggestion = provide a definition for the function
+
+ast_passes_assoc_type_without_body =
+    associated type in `impl` without body
+    .suggestion = provide a definition for the type
+
+ast_passes_const_without_body =
+    free constant item without body
+    .suggestion = provide a definition for the constant
+
+ast_passes_static_without_body =
+    free static item without body
+    .suggestion = provide a definition for the static
+
+ast_passes_ty_alias_without_body =
+    free type alias without body
+    .suggestion = provide a definition for the type
+
+ast_passes_fn_without_body =
+    free function without a body
+    .suggestion = provide a definition for the function
+
+ast_passes_extern_block_suggestion = if you meant to declare an externally defined function, use an `extern` block
+
+ast_passes_bound_in_context = bounds on `type`s in {$ctx} have no effect
+
+ast_passes_extern_types_cannot = `type`s inside `extern` blocks cannot have {$descr}
+    .suggestion = remove the {$remove_descr}
+    .label = `extern` block begins here
+
+ast_passes_extern_keyword_link = for more information, visit https://doc.rust-lang.org/std/keyword.extern.html
+
+ast_passes_body_in_extern = incorrect `{$kind}` inside `extern` block
+    .cannot_have = cannot have a body
+    .invalid = the invalid body
+    .existing = `extern` blocks define existing foreign {$kind}s and {$kind}s inside of them cannot have a body
+
+ast_passes_fn_body_extern = incorrect function inside `extern` block
+    .cannot_have = cannot have a body
+    .suggestion = remove the invalid body
+    .help = you might have meant to write a function accessible through FFI, which can be done by writing `extern fn` outside of the `extern` block
+    .label = `extern` blocks define existing foreign functions and functions inside of them cannot have a body
+
+ast_passes_extern_fn_qualifiers = functions in `extern` blocks cannot have qualifiers
+    .label = in this `extern` block
+    .suggestion = remove the qualifiers
+
+ast_passes_extern_item_ascii = items in `extern` blocks cannot use non-ascii identifiers
+    .label = in this `extern` block
+    .note = this limitation may be lifted in the future; see issue #83942 <https://github.com/rust-lang/rust/issues/83942> for more information
+
+ast_passes_bad_c_variadic = only foreign or `unsafe extern "C"` functions may be C-variadic
+
+ast_passes_item_underscore = `{$kind}` items in this context need a name
+    .label = `_` is not a valid name for this `{$kind}` item
+
+ast_passes_nomangle_ascii = `#[no_mangle]` requires ASCII identifier
+
+ast_passes_module_nonascii = trying to load file for module `{$name}` with non-ascii identifier name
+    .help = consider using the `#[path]` attribute to specify filesystem path
+
+ast_passes_auto_generic = auto traits cannot have generic parameters
+    .label = auto trait cannot have generic parameters
+    .suggestion = remove the parameters
+
+ast_passes_auto_super_lifetime = auto traits cannot have super traits or lifetime bounds
+    .label = {ast_passes_auto_super_lifetime}
+    .suggestion = remove the super traits or lifetime bounds
+
+ast_passes_auto_items = auto traits cannot have associated items
+    .label = {ast_passes_auto_items}
+    .suggestion = remove these associated items
+
+ast_passes_generic_before_constraints = generic arguments must come before the first constraint
+    .constraints = {$constraint_len ->
+    [one] constraint
+    *[other] constraints
+    }
+    .args = generic {$args_len ->
+    [one] argument
+    *[other] arguments
+    }
+    .empty_string = {""},
+    .suggestion = move the {$constraint_len ->
+    [one] constraint
+    *[other] constraints
+    } after the generic {$args_len ->
+    [one] argument
+    *[other] arguments
+    }
+
+ast_passes_pattern_in_fn_pointer = patterns aren't allowed in function pointer types
+
+ast_passes_trait_object_single_bound = only a single explicit lifetime bound is permitted
+
+ast_passes_impl_trait_path = `impl Trait` is not allowed in path parameters
+
+ast_passes_nested_impl_trait = nested `impl Trait` is not allowed
+    .outer = outer `impl Trait`
+    .inner = nested `impl Trait` here
+
+ast_passes_at_least_one_trait = at least one trait must be specified
+
+ast_passes_extern_without_abi = extern declarations without an explicit ABI are deprecated
+
+ast_passes_out_of_order_params = {$param_ord} parameters must be declared prior to {$max_param} parameters
+    .suggestion = reorder the parameters: lifetimes, then consts and types
+
+ast_passes_obsolete_auto = `impl Trait for .. {"{}"}` is an obsolete syntax
+    .help = use `auto trait Trait {"{}"}` instead
+
+ast_passes_unsafe_negative_impl = negative impls cannot be unsafe
+    .negative = negative because of this
+    .unsafe = unsafe because of this
+
+ast_passes_inherent_cannot_be = inherent impls cannot be {$annotation}
+    .because = {$annotation} because of this
+    .type = inherent impl for this type
+    .only_trait = only trait implementations may be annotated with {$annotation}
+
+ast_passes_unsafe_item = {$kind} cannot be declared unsafe
+
+ast_passes_fieldless_union = unions cannot have zero fields
+
+ast_passes_where_after_type_alias = where clauses are not allowed after the type for type aliases
+    .note = see issue #89122 <https://github.com/rust-lang/rust/issues/89122> for more information
+
+ast_passes_generic_default_trailing = generic parameters with a default must be trailing
+
+ast_passes_nested_lifetimes = nested quantification of lifetimes
+
+ast_passes_optional_trait_supertrait = `?Trait` is not permitted in supertraits
+    .note = traits are `?{$path_str}` by default
+
+ast_passes_optional_trait_object = `?Trait` is not permitted in trait object types
+
+ast_passes_tilde_const_disallowed = `~const` is not allowed here
+    .trait = trait objects cannot have `~const` trait bounds
+    .closure = closures cannot have `~const` trait bounds
+    .function = this function is not `const`, so it cannot have `~const` trait bounds
+
+ast_passes_optional_const_exclusive = `~const` and `?` are mutually exclusive
+
+ast_passes_const_and_async = functions cannot be both `const` and `async`
+    .const = `const` because of this
+    .async = `async` because of this
+    .label = {""}
+
+ast_passes_pattern_in_foreign = patterns aren't allowed in foreign function declarations
+    .label = pattern not allowed in foreign function
+
+ast_passes_pattern_in_bodiless = patterns aren't allowed in functions without bodies
+    .label = pattern not allowed in function without body
+
+ast_passes_equality_in_where = equality constraints are not yet supported in `where` clauses
+    .label = not supported
+    .suggestion = if `{$ident}` is an associated type you're trying to set, use the associated type binding syntax
+    .suggestion_path = if `{$trait_segment}::{$potential_assoc}` is an associated type you're trying to set, use the associated type binding syntax
+    .note = see issue #20041 <https://github.com/rust-lang/rust/issues/20041> for more information
+
+ast_passes_stability_outside_std = stability attributes may not be used outside of the standard library
+
+ast_passes_feature_on_non_nightly = `#![feature]` may not be used on the {$channel} release channel
+    .suggestion = remove the attribute
+    .stable_since = the feature `{$name}` has been stable since `{$since}` and no longer requires an attribute to enable
+
+ast_passes_incompatbile_features = `{$f1}` and `{$f2}` are incompatible, using them at the same time is not allowed
+    .help = remove one of these features
+
+ast_passes_show_span = {$msg}
diff --git a/compiler/rustc_ast_passes/src/ast_validation.rs b/compiler/rustc_ast_passes/src/ast_validation.rs
index 902b4b1a1..2cc009410 100644
--- a/compiler/rustc_ast_passes/src/ast_validation.rs
+++ b/compiler/rustc_ast_passes/src/ast_validation.rs
@@ -13,7 +13,6 @@ use rustc_ast::walk_list;
 use rustc_ast::*;
 use rustc_ast_pretty::pprust::{self, State};
 use rustc_data_structures::fx::FxHashMap;
-use rustc_errors::{error_code, fluent, pluralize, struct_span_err, Applicability};
 use rustc_macros::Subdiagnostic;
 use rustc_parse::validate_attr;
 use rustc_session::lint::builtin::{
@@ -27,11 +26,10 @@ use rustc_span::Span;
 use rustc_target::spec::abi;
 use std::mem;
 use std::ops::{Deref, DerefMut};
+use thin_vec::thin_vec;
 
-use crate::errors::*;
-
-const MORE_EXTERN: &str =
-    "for more information, visit https://doc.rust-lang.org/std/keyword.extern.html";
+use crate::errors;
+use crate::fluent_generated as fluent;
 
 /// Is `self` allowed semantically as the first parameter in an `FnDecl`?
 enum SelfSemantic {
@@ -69,10 +67,6 @@ struct AstValidator<'a> {
     /// or `Foo::Bar<impl Trait>`
     is_impl_trait_banned: bool,
 
-    /// Used to ban associated type bounds (i.e., `Type<AssocType: Bounds>`) in
-    /// certain positions.
-    is_assoc_ty_bound_banned: bool,
-
     /// See [ForbiddenLetReason]
     forbidden_let_reason: Option<ForbiddenLetReason>,
 
@@ -136,9 +130,9 @@ impl<'a> AstValidator<'a> {
     fn ban_let_expr(&self, expr: &'a Expr, forbidden_let_reason: ForbiddenLetReason) {
         let sess = &self.session;
         if sess.opts.unstable_features.is_nightly_build() {
-            sess.emit_err(ForbiddenLet { span: expr.span, reason: forbidden_let_reason });
+            sess.emit_err(errors::ForbiddenLet { span: expr.span, reason: forbidden_let_reason });
         } else {
-            sess.emit_err(ForbiddenLetStable { span: expr.span });
+            sess.emit_err(errors::ForbiddenLetStable { span: expr.span });
         }
     }
 
@@ -178,30 +172,12 @@ impl<'a> AstValidator<'a> {
         }
     }
 
-    fn with_banned_assoc_ty_bound(&mut self, f: impl FnOnce(&mut Self)) {
-        let old = mem::replace(&mut self.is_assoc_ty_bound_banned, true);
-        f(self);
-        self.is_assoc_ty_bound_banned = old;
-    }
-
     fn with_impl_trait(&mut self, outer: Option<Span>, f: impl FnOnce(&mut Self)) {
         let old = mem::replace(&mut self.outer_impl_trait, outer);
         f(self);
         self.outer_impl_trait = old;
     }
 
-    fn visit_assoc_constraint_from_generic_args(&mut self, constraint: &'a AssocConstraint) {
-        match constraint.kind {
-            AssocConstraintKind::Equality { .. } => {}
-            AssocConstraintKind::Bound { .. } => {
-                if self.is_assoc_ty_bound_banned {
-                    self.session.emit_err(ForbiddenAssocConstraint { span: constraint.span });
-                }
-            }
-        }
-        self.visit_assoc_constraint(constraint);
-    }
-
     // Mirrors `visit::walk_ty`, but tracks relevant state.
     fn walk_ty(&mut self, t: &'a Ty) {
         match &t.kind {
@@ -216,7 +192,7 @@ impl<'a> AstValidator<'a> {
                 // We allow these:
                 //  - `Option<impl Trait>`
                 //  - `option::Option<impl Trait>`
-                //  - `option::Option<T>::Foo<impl Trait>
+                //  - `option::Option<T>::Foo<impl Trait>`
                 //
                 // But not these:
                 //  - `<impl Trait>::Foo`
@@ -254,24 +230,24 @@ impl<'a> AstValidator<'a> {
     fn check_lifetime(&self, ident: Ident) {
         let valid_names = [kw::UnderscoreLifetime, kw::StaticLifetime, kw::Empty];
         if !valid_names.contains(&ident.name) && ident.without_first_quote().is_reserved() {
-            self.session.emit_err(KeywordLifetime { span: ident.span });
+            self.session.emit_err(errors::KeywordLifetime { span: ident.span });
         }
     }
 
     fn check_label(&self, ident: Ident) {
         if ident.without_first_quote().is_reserved() {
-            self.session.emit_err(InvalidLabel { span: ident.span, name: ident.name });
+            self.session.emit_err(errors::InvalidLabel { span: ident.span, name: ident.name });
         }
     }
 
-    fn invalid_visibility(&self, vis: &Visibility, note: Option<InvalidVisibilityNote>) {
+    fn invalid_visibility(&self, vis: &Visibility, note: Option<errors::InvalidVisibilityNote>) {
         if let VisibilityKind::Inherited = vis.kind {
             return;
         }
 
-        self.session.emit_err(InvalidVisibility {
+        self.session.emit_err(errors::InvalidVisibility {
             span: vis.span,
-            implied: if vis.kind.is_pub() { Some(vis.span) } else { None },
+            implied: vis.kind.is_pub().then_some(vis.span),
             note,
         });
     }
@@ -290,28 +266,7 @@ impl<'a> AstValidator<'a> {
 
     fn check_trait_fn_not_const(&self, constness: Const) {
         if let Const::Yes(span) = constness {
-            self.session.emit_err(TraitFnConst { span });
-        }
-    }
-
-    fn check_late_bound_lifetime_defs(&self, params: &[GenericParam]) {
-        // Check only lifetime parameters are present and that the lifetime
-        // parameters that are present have no bounds.
-        let non_lt_param_spans: Vec<_> = params
-            .iter()
-            .filter_map(|param| match param.kind {
-                GenericParamKind::Lifetime { .. } => {
-                    if !param.bounds.is_empty() {
-                        let spans: Vec<_> = param.bounds.iter().map(|b| b.span()).collect();
-                        self.session.emit_err(ForbiddenLifetimeBound { spans });
-                    }
-                    None
-                }
-                _ => Some(param.ident.span),
-            })
-            .collect();
-        if !non_lt_param_spans.is_empty() {
-            self.session.emit_err(ForbiddenNonLifetimeParam { spans: non_lt_param_spans });
+            self.session.emit_err(errors::TraitFnConst { span });
         }
     }
 
@@ -328,7 +283,7 @@ impl<'a> AstValidator<'a> {
         let max_num_args: usize = u16::MAX.into();
         if fn_decl.inputs.len() > max_num_args {
             let Param { span, .. } = fn_decl.inputs[0];
-            self.session.emit_fatal(FnParamTooMany { span, max_num_args });
+            self.session.emit_fatal(errors::FnParamTooMany { span, max_num_args });
         }
     }
 
@@ -336,13 +291,13 @@ impl<'a> AstValidator<'a> {
         match &*fn_decl.inputs {
             [Param { ty, span, .. }] => {
                 if let TyKind::CVarArgs = ty.kind {
-                    self.session.emit_err(FnParamCVarArgsOnly { span: *span });
+                    self.session.emit_err(errors::FnParamCVarArgsOnly { span: *span });
                 }
             }
             [ps @ .., _] => {
                 for Param { ty, span, .. } in ps {
                     if let TyKind::CVarArgs = ty.kind {
-                        self.session.emit_err(FnParamCVarArgsNotLast { span: *span });
+                        self.session.emit_err(errors::FnParamCVarArgsNotLast { span: *span });
                     }
                 }
             }
@@ -369,9 +324,9 @@ impl<'a> AstValidator<'a> {
             })
             .for_each(|attr| {
                 if attr.is_doc_comment() {
-                    self.session.emit_err(FnParamDocComment { span: attr.span });
+                    self.session.emit_err(errors::FnParamDocComment { span: attr.span });
                 } else {
-                    self.session.emit_err(FnParamForbiddenAttr { span: attr.span });
+                    self.session.emit_err(errors::FnParamForbiddenAttr { span: attr.span });
                 }
             });
     }
@@ -379,7 +334,7 @@ impl<'a> AstValidator<'a> {
     fn check_decl_self_param(&self, fn_decl: &FnDecl, self_semantic: SelfSemantic) {
         if let (SelfSemantic::No, [param, ..]) = (self_semantic, &*fn_decl.inputs) {
             if param.is_self() {
-                self.session.emit_err(FnParamForbiddenSelf { span: param.span });
+                self.session.emit_err(errors::FnParamForbiddenSelf { span: param.span });
             }
         }
     }
@@ -387,7 +342,7 @@ impl<'a> AstValidator<'a> {
     fn check_defaultness(&self, span: Span, defaultness: Defaultness) {
         if let Defaultness::Default(def_span) = defaultness {
             let span = self.session.source_map().guess_head_span(span);
-            self.session.emit_err(ForbiddenDefault { span, def_span });
+            self.session.emit_err(errors::ForbiddenDefault { span, def_span });
         }
     }
 
@@ -410,27 +365,17 @@ impl<'a> AstValidator<'a> {
             [b0] => b0.span(),
             [b0, .., bl] => b0.span().to(bl.span()),
         };
-        self.err_handler()
-            .struct_span_err(span, &format!("bounds on `type`s in {} have no effect", ctx))
-            .emit();
+        self.err_handler().emit_err(errors::BoundInContext { span, ctx });
     }
 
     fn check_foreign_ty_genericless(&self, generics: &Generics, where_span: Span) {
         let cannot_have = |span, descr, remove_descr| {
-            self.err_handler()
-                .struct_span_err(
-                    span,
-                    &format!("`type`s inside `extern` blocks cannot have {}", descr),
-                )
-                .span_suggestion(
-                    span,
-                    &format!("remove the {}", remove_descr),
-                    "",
-                    Applicability::MaybeIncorrect,
-                )
-                .span_label(self.current_extern_span(), "`extern` block begins here")
-                .note(MORE_EXTERN)
-                .emit();
+            self.err_handler().emit_err(errors::ExternTypesCannotHave {
+                span,
+                descr,
+                remove_descr,
+                block_span: self.current_extern_span(),
+            });
         };
 
         if !generics.params.is_empty() {
@@ -446,20 +391,12 @@ impl<'a> AstValidator<'a> {
         let Some(body) = body else {
             return;
         };
-        self.err_handler()
-            .struct_span_err(ident.span, &format!("incorrect `{}` inside `extern` block", kind))
-            .span_label(ident.span, "cannot have a body")
-            .span_label(body, "the invalid body")
-            .span_label(
-                self.current_extern_span(),
-                format!(
-                    "`extern` blocks define existing foreign {0}s and {0}s \
-                    inside of them cannot have a body",
-                    kind
-                ),
-            )
-            .note(MORE_EXTERN)
-            .emit();
+        self.err_handler().emit_err(errors::BodyInExtern {
+            span: ident.span,
+            body,
+            block: self.current_extern_span(),
+            kind,
+        });
     }
 
     /// An `fn` in `extern { ... }` cannot have a body `{ ... }`.
@@ -467,26 +404,11 @@ impl<'a> AstValidator<'a> {
         let Some(body) = body else {
             return;
         };
-        self.err_handler()
-            .struct_span_err(ident.span, "incorrect function inside `extern` block")
-            .span_label(ident.span, "cannot have a body")
-            .span_suggestion(
-                body.span,
-                "remove the invalid body",
-                ";",
-                Applicability::MaybeIncorrect,
-            )
-            .help(
-                "you might have meant to write a function accessible through FFI, \
-                which can be done by writing `extern fn` outside of the `extern` block",
-            )
-            .span_label(
-                self.current_extern_span(),
-                "`extern` blocks define existing foreign functions and functions \
-                inside of them cannot have a body",
-            )
-            .note(MORE_EXTERN)
-            .emit();
+        self.err_handler().emit_err(errors::FnBodyInExtern {
+            span: ident.span,
+            body: body.span,
+            block: self.current_extern_span(),
+        });
     }
 
     fn current_extern_span(&self) -> Span {
@@ -496,34 +418,21 @@ impl<'a> AstValidator<'a> {
     /// An `fn` in `extern { ... }` cannot have qualifiers, e.g. `async fn`.
     fn check_foreign_fn_headerless(&self, ident: Ident, span: Span, header: FnHeader) {
         if header.has_qualifiers() {
-            self.err_handler()
-                .struct_span_err(ident.span, "functions in `extern` blocks cannot have qualifiers")
-                .span_label(self.current_extern_span(), "in this `extern` block")
-                .span_suggestion_verbose(
-                    span.until(ident.span.shrink_to_lo()),
-                    "remove the qualifiers",
-                    "fn ",
-                    Applicability::MaybeIncorrect,
-                )
-                .emit();
+            self.err_handler().emit_err(errors::FnQualifierInExtern {
+                span: ident.span,
+                block: self.current_extern_span(),
+                sugg_span: span.until(ident.span.shrink_to_lo()),
+            });
         }
     }
 
     /// An item in `extern { ... }` cannot use non-ascii identifier.
     fn check_foreign_item_ascii_only(&self, ident: Ident) {
         if !ident.as_str().is_ascii() {
-            let n = 83942;
-            self.err_handler()
-                .struct_span_err(
-                    ident.span,
-                    "items in `extern` blocks cannot use non-ascii identifiers",
-                )
-                .span_label(self.current_extern_span(), "in this `extern` block")
-                .note(&format!(
-                    "this limitation may be lifted in the future; see issue #{} <https://github.com/rust-lang/rust/issues/{}> for more information",
-                    n, n,
-                ))
-                .emit();
+            self.err_handler().emit_err(errors::ExternItemAscii {
+                span: ident.span,
+                block: self.current_extern_span(),
+            });
         }
     }
 
@@ -546,12 +455,7 @@ impl<'a> AstValidator<'a> {
 
         for Param { ty, span, .. } in &fk.decl().inputs {
             if let TyKind::CVarArgs = ty.kind {
-                self.err_handler()
-                    .struct_span_err(
-                        *span,
-                        "only foreign or `unsafe extern \"C\"` functions may be C-variadic",
-                    )
-                    .emit();
+                self.err_handler().emit_err(errors::BadCVariadic { span: *span });
             }
         }
     }
@@ -560,75 +464,32 @@ impl<'a> AstValidator<'a> {
         if ident.name != kw::Underscore {
             return;
         }
-        self.err_handler()
-            .struct_span_err(ident.span, &format!("`{}` items in this context need a name", kind))
-            .span_label(ident.span, format!("`_` is not a valid name for this `{}` item", kind))
-            .emit();
+        self.err_handler().emit_err(errors::ItemUnderscore { span: ident.span, kind });
     }
 
     fn check_nomangle_item_asciionly(&self, ident: Ident, item_span: Span) {
         if ident.name.as_str().is_ascii() {
             return;
         }
-        let head_span = self.session.source_map().guess_head_span(item_span);
-        struct_span_err!(
-            self.session,
-            head_span,
-            E0754,
-            "`#[no_mangle]` requires ASCII identifier"
-        )
-        .emit();
+        let span = self.session.source_map().guess_head_span(item_span);
+        self.session.emit_err(errors::NoMangleAscii { span });
     }
 
     fn check_mod_file_item_asciionly(&self, ident: Ident) {
         if ident.name.as_str().is_ascii() {
             return;
         }
-        struct_span_err!(
-            self.session,
-            ident.span,
-            E0754,
-            "trying to load file for module `{}` with non-ascii identifier name",
-            ident.name
-        )
-        .help("consider using `#[path]` attribute to specify filesystem path")
-        .emit();
+        self.session.emit_err(errors::ModuleNonAscii { span: ident.span, name: ident.name });
     }
 
-    fn deny_generic_params(&self, generics: &Generics, ident_span: Span) {
+    fn deny_generic_params(&self, generics: &Generics, ident: Span) {
         if !generics.params.is_empty() {
-            struct_span_err!(
-                self.session,
-                generics.span,
-                E0567,
-                "auto traits cannot have generic parameters"
-            )
-            .span_label(ident_span, "auto trait cannot have generic parameters")
-            .span_suggestion(
-                generics.span,
-                "remove the parameters",
-                "",
-                Applicability::MachineApplicable,
-            )
-            .emit();
+            self.session.emit_err(errors::AutoTraitGeneric { span: generics.span, ident });
         }
     }
 
-    fn emit_e0568(&self, span: Span, ident_span: Span) {
-        struct_span_err!(
-            self.session,
-            span,
-            E0568,
-            "auto traits cannot have super traits or lifetime bounds"
-        )
-        .span_label(ident_span, "auto trait cannot have super traits or lifetime bounds")
-        .span_suggestion(
-            span,
-            "remove the super traits or lifetime bounds",
-            "",
-            Applicability::MachineApplicable,
-        )
-        .emit();
+    fn emit_e0568(&self, span: Span, ident: Span) {
+        self.session.emit_err(errors::AutoTraitBounds { span, ident });
     }
 
     fn deny_super_traits(&self, bounds: &GenericBounds, ident_span: Span) {
@@ -644,24 +505,11 @@ impl<'a> AstValidator<'a> {
         }
     }
 
-    fn deny_items(&self, trait_items: &[P<AssocItem>], ident_span: Span) {
+    fn deny_items(&self, trait_items: &[P<AssocItem>], ident: Span) {
         if !trait_items.is_empty() {
             let spans: Vec<_> = trait_items.iter().map(|i| i.ident.span).collect();
-            let total_span = trait_items.first().unwrap().span.to(trait_items.last().unwrap().span);
-            struct_span_err!(
-                self.session,
-                spans,
-                E0380,
-                "auto traits cannot have associated items"
-            )
-            .span_suggestion(
-                total_span,
-                "remove these associated items",
-                "",
-                Applicability::MachineApplicable,
-            )
-            .span_label(ident_span, "auto trait cannot have associated items")
-            .emit();
+            let total = trait_items.first().unwrap().span.to(trait_items.last().unwrap().span);
+            self.session.emit_err(errors::AutoTraitItems { spans, total, ident });
         }
     }
 
@@ -707,29 +555,17 @@ impl<'a> AstValidator<'a> {
         let args_len = arg_spans.len();
         let constraint_len = constraint_spans.len();
         // ...and then error:
-        self.err_handler()
-            .struct_span_err(
-                arg_spans.clone(),
-                "generic arguments must come before the first constraint",
-            )
-            .span_label(constraint_spans[0], &format!("constraint{}", pluralize!(constraint_len)))
-            .span_label(
-                *arg_spans.iter().last().unwrap(),
-                &format!("generic argument{}", pluralize!(args_len)),
-            )
-            .span_labels(constraint_spans, "")
-            .span_labels(arg_spans, "")
-            .span_suggestion_verbose(
-                data.span,
-                &format!(
-                    "move the constraint{} after the generic argument{}",
-                    pluralize!(constraint_len),
-                    pluralize!(args_len)
-                ),
-                self.correct_generic_order_suggestion(&data),
-                Applicability::MachineApplicable,
-            )
-            .emit();
+        self.err_handler().emit_err(errors::ArgsBeforeConstraint {
+            arg_spans: arg_spans.clone(),
+            constraints: constraint_spans[0],
+            args: *arg_spans.iter().last().unwrap(),
+            data: data.span,
+            constraint_spans: errors::EmptyLabelManySpans(constraint_spans),
+            arg_spans2: errors::EmptyLabelManySpans(arg_spans),
+            suggestion: self.correct_generic_order_suggestion(&data),
+            constraint_len,
+            args_len,
+        });
     }
 
     fn visit_ty_common(&mut self, ty: &'a Ty) {
@@ -737,15 +573,8 @@ impl<'a> AstValidator<'a> {
             TyKind::BareFn(bfty) => {
                 self.check_fn_decl(&bfty.decl, SelfSemantic::No);
                 Self::check_decl_no_pat(&bfty.decl, |span, _, _| {
-                    struct_span_err!(
-                        self.session,
-                        span,
-                        E0561,
-                        "patterns aren't allowed in function pointer types"
-                    )
-                    .emit();
+                    self.session.emit_err(errors::PatternFnPointer { span });
                 });
-                self.check_late_bound_lifetime_defs(&bfty.generic_params);
                 if let Extern::Implicit(_) = bfty.ext {
                     let sig_span = self.session.source_map().next_point(ty.span.shrink_to_lo());
                     self.maybe_lint_missing_abi(sig_span, ty.id);
@@ -756,13 +585,8 @@ impl<'a> AstValidator<'a> {
                 for bound in bounds {
                     if let GenericBound::Outlives(lifetime) = bound {
                         if any_lifetime_bounds {
-                            struct_span_err!(
-                                self.session,
-                                lifetime.ident.span,
-                                E0226,
-                                "only a single explicit lifetime bound is permitted"
-                            )
-                            .emit();
+                            self.session
+                                .emit_err(errors::TraitObjectBound { span: lifetime.ident.span });
                             break;
                         }
                         any_lifetime_bounds = true;
@@ -771,29 +595,19 @@ impl<'a> AstValidator<'a> {
             }
             TyKind::ImplTrait(_, bounds) => {
                 if self.is_impl_trait_banned {
-                    struct_span_err!(
-                        self.session,
-                        ty.span,
-                        E0667,
-                        "`impl Trait` is not allowed in path parameters"
-                    )
-                    .emit();
+                    self.session.emit_err(errors::ImplTraitPath { span: ty.span });
                 }
 
                 if let Some(outer_impl_trait_sp) = self.outer_impl_trait {
-                    struct_span_err!(
-                        self.session,
-                        ty.span,
-                        E0666,
-                        "nested `impl Trait` is not allowed"
-                    )
-                    .span_label(outer_impl_trait_sp, "outer `impl Trait`")
-                    .span_label(ty.span, "nested `impl Trait` here")
-                    .emit();
+                    self.session.emit_err(errors::NestedImplTrait {
+                        span: ty.span,
+                        outer: outer_impl_trait_sp,
+                        inner: ty.span,
+                    });
                 }
 
                 if !bounds.iter().any(|b| matches!(b, GenericBound::Trait(..))) {
-                    self.err_handler().span_err(ty.span, "at least one trait must be specified");
+                    self.err_handler().emit_err(errors::AtLeastOneTrait { span: ty.span });
                 }
             }
             _ => {}
@@ -814,7 +628,7 @@ impl<'a> AstValidator<'a> {
                 MISSING_ABI,
                 id,
                 span,
-                "extern declarations without an explicit ABI are deprecated",
+                fluent::ast_passes_extern_without_abi,
                 BuiltinLintDiagnostics::MissingAbi(span, abi::Abi::FALLBACK),
             )
         }
@@ -887,20 +701,13 @@ fn validate_generic_param_order(
         ordered_params += ">";
 
         for (param_ord, (max_param, spans)) in &out_of_order {
-            let mut err = handler.struct_span_err(
-                spans.clone(),
-                &format!(
-                    "{} parameters must be declared prior to {} parameters",
-                    param_ord, max_param,
-                ),
-            );
-            err.span_suggestion(
-                span,
-                "reorder the parameters: lifetimes, then consts and types",
-                &ordered_params,
-                Applicability::MachineApplicable,
-            );
-            err.emit();
+            handler.emit_err(errors::OutOfOrderParams {
+                spans: spans.clone(),
+                sugg_span: span,
+                param_ord,
+                max_param,
+                ordered_params: &ordered_params,
+            });
         }
     }
 }
@@ -1014,25 +821,15 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 self.with_in_trait_impl(true, Some(*constness), |this| {
                     this.invalid_visibility(&item.vis, None);
                     if let TyKind::Err = self_ty.kind {
-                        this.err_handler()
-                            .struct_span_err(
-                                item.span,
-                                "`impl Trait for .. {}` is an obsolete syntax",
-                            )
-                            .help("use `auto trait Trait {}` instead")
-                            .emit();
+                        this.err_handler().emit_err(errors::ObsoleteAuto { span: item.span });
                     }
                     if let (&Unsafe::Yes(span), &ImplPolarity::Negative(sp)) = (unsafety, polarity)
                     {
-                        struct_span_err!(
-                            this.session,
-                            sp.to(t.path.span),
-                            E0198,
-                            "negative impls cannot be unsafe"
-                        )
-                        .span_label(sp, "negative because of this")
-                        .span_label(span, "unsafe because of this")
-                        .emit();
+                        this.session.emit_err(errors::UnsafeNegativeImpl {
+                            span: sp.to(t.path.span),
+                            negative: sp,
+                            r#unsafe: span,
+                        });
                     }
 
                     this.visit_vis(&item.vis);
@@ -1060,52 +857,54 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 self_ty,
                 items: _,
             }) => {
-                let error = |annotation_span, annotation| {
-                    let mut err = self.err_handler().struct_span_err(
-                        self_ty.span,
-                        &format!("inherent impls cannot be {}", annotation),
-                    );
-                    err.span_label(annotation_span, &format!("{} because of this", annotation));
-                    err.span_label(self_ty.span, "inherent impl for this type");
-                    err
-                };
+                let error =
+                    |annotation_span, annotation, only_trait: bool| errors::InherentImplCannot {
+                        span: self_ty.span,
+                        annotation_span,
+                        annotation,
+                        self_ty: self_ty.span,
+                        only_trait: only_trait.then_some(()),
+                    };
 
                 self.invalid_visibility(
                     &item.vis,
-                    Some(InvalidVisibilityNote::IndividualImplItems),
+                    Some(errors::InvalidVisibilityNote::IndividualImplItems),
                 );
                 if let &Unsafe::Yes(span) = unsafety {
-                    error(span, "unsafe").code(error_code!(E0197)).emit();
+                    self.err_handler().emit_err(errors::InherentImplCannotUnsafe {
+                        span: self_ty.span,
+                        annotation_span: span,
+                        annotation: "unsafe",
+                        self_ty: self_ty.span,
+                    });
                 }
                 if let &ImplPolarity::Negative(span) = polarity {
-                    error(span, "negative").emit();
+                    self.err_handler().emit_err(error(span, "negative", false));
                 }
                 if let &Defaultness::Default(def_span) = defaultness {
-                    error(def_span, "`default`")
-                        .note("only trait implementations may be annotated with `default`")
-                        .emit();
+                    self.err_handler().emit_err(error(def_span, "`default`", true));
                 }
                 if let &Const::Yes(span) = constness {
-                    error(span, "`const`")
-                        .note("only trait implementations may be annotated with `const`")
-                        .emit();
+                    self.err_handler().emit_err(error(span, "`const`", true));
                 }
             }
             ItemKind::Fn(box Fn { defaultness, sig, generics, body }) => {
                 self.check_defaultness(item.span, *defaultness);
 
                 if body.is_none() {
-                    self.session.emit_err(FnWithoutBody {
+                    self.session.emit_err(errors::FnWithoutBody {
                         span: item.span,
                         replace_span: self.ending_semi_or_hi(item.span),
                         extern_block_suggestion: match sig.header.ext {
                             Extern::None => None,
-                            Extern::Implicit(start_span) => Some(ExternBlockSuggestion::Implicit {
-                                start_span,
-                                end_span: item.span.shrink_to_hi(),
-                            }),
+                            Extern::Implicit(start_span) => {
+                                Some(errors::ExternBlockSuggestion::Implicit {
+                                    start_span,
+                                    end_span: item.span.shrink_to_hi(),
+                                })
+                            }
                             Extern::Explicit(abi, start_span) => {
-                                Some(ExternBlockSuggestion::Explicit {
+                                Some(errors::ExternBlockSuggestion::Explicit {
                                     start_span,
                                     end_span: item.span.shrink_to_hi(),
                                     abi: abi.symbol_unescaped,
@@ -1127,10 +926,10 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 let old_item = mem::replace(&mut self.extern_mod, Some(item));
                 self.invalid_visibility(
                     &item.vis,
-                    Some(InvalidVisibilityNote::IndividualForeignItems),
+                    Some(errors::InvalidVisibilityNote::IndividualForeignItems),
                 );
                 if let &Unsafe::Yes(span) = unsafety {
-                    self.err_handler().span_err(span, "extern block cannot be declared unsafe");
+                    self.err_handler().emit_err(errors::UnsafeItem { span, kind: "extern block" });
                 }
                 if abi.is_none() {
                     self.maybe_lint_missing_abi(item.span, item.id);
@@ -1170,7 +969,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
             }
             ItemKind::Mod(unsafety, mod_kind) => {
                 if let &Unsafe::Yes(span) = unsafety {
-                    self.err_handler().span_err(span, "module cannot be declared unsafe");
+                    self.err_handler().emit_err(errors::UnsafeItem { span, kind: "module" });
                 }
                 // Ensure that `path` attributes on modules are recorded as used (cf. issue #35584).
                 if !matches!(mod_kind, ModKind::Loaded(_, Inline::Yes, _))
@@ -1181,18 +980,18 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
             }
             ItemKind::Union(vdata, ..) => {
                 if vdata.fields().is_empty() {
-                    self.err_handler().span_err(item.span, "unions cannot have zero fields");
+                    self.err_handler().emit_err(errors::FieldlessUnion { span: item.span });
                 }
             }
             ItemKind::Const(def, .., None) => {
                 self.check_defaultness(item.span, *def);
-                self.session.emit_err(ConstWithoutBody {
+                self.session.emit_err(errors::ConstWithoutBody {
                     span: item.span,
                     replace_span: self.ending_semi_or_hi(item.span),
                 });
             }
             ItemKind::Static(.., None) => {
-                self.session.emit_err(StaticWithoutBody {
+                self.session.emit_err(errors::StaticWithoutBody {
                     span: item.span,
                     replace_span: self.ending_semi_or_hi(item.span),
                 });
@@ -1200,21 +999,15 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
             ItemKind::TyAlias(box TyAlias { defaultness, where_clauses, bounds, ty, .. }) => {
                 self.check_defaultness(item.span, *defaultness);
                 if ty.is_none() {
-                    self.session.emit_err(TyAliasWithoutBody {
+                    self.session.emit_err(errors::TyAliasWithoutBody {
                         span: item.span,
                         replace_span: self.ending_semi_or_hi(item.span),
                     });
                 }
                 self.check_type_no_bounds(bounds, "this context");
                 if where_clauses.1.0 {
-                    let mut err = self.err_handler().struct_span_err(
-                        where_clauses.1.1,
-                        "where clauses are not allowed after the type for type aliases",
-                    );
-                    err.note(
-                        "see issue #89122 <https://github.com/rust-lang/rust/issues/89122> for more information",
-                    );
-                    err.emit();
+                    self.err_handler()
+                        .emit_err(errors::WhereAfterTypeAlias { span: where_clauses.1.1 });
                 }
             }
             _ => {}
@@ -1268,7 +1061,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                         // are allowed to contain nested `impl Trait`.
                         AngleBracketedArg::Constraint(constraint) => {
                             self.with_impl_trait(None, |this| {
-                                this.visit_assoc_constraint_from_generic_args(constraint);
+                                this.visit_assoc_constraint(constraint);
                             });
                         }
                     }
@@ -1296,11 +1089,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 }
                 GenericParamKind::Type { .. } | GenericParamKind::Const { .. } => {
                     if let Some(span) = prev_param_default {
-                        let mut err = self.err_handler().struct_span_err(
-                            span,
-                            "generic parameters with a default must be trailing",
-                        );
-                        err.emit();
+                        self.err_handler().emit_err(errors::GenericDefaultTrailing { span });
                         break;
                     }
                 }
@@ -1318,9 +1107,6 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         for predicate in &generics.where_clause.predicates {
             match predicate {
                 WherePredicate::BoundPredicate(bound_pred) => {
-                    // A type binding, eg `for<'c> Foo: Send+Clone+'c`
-                    self.check_late_bound_lifetime_defs(&bound_pred.bound_generic_params);
-
                     // This is slightly complicated. Our representation for poly-trait-refs contains a single
                     // binder and thus we only allow a single level of quantification. However,
                     // the syntax of Rust permits quantification in two places in where clauses,
@@ -1331,13 +1117,8 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                             match bound {
                                 GenericBound::Trait(t, _) => {
                                     if !t.bound_generic_params.is_empty() {
-                                        struct_span_err!(
-                                            self.err_handler(),
-                                            t.span,
-                                            E0316,
-                                            "nested quantification of lifetimes"
-                                        )
-                                        .emit();
+                                        self.err_handler()
+                                            .emit_err(errors::NestedLifetimes { span: t.span });
                                     }
                                 }
                                 GenericBound::Outlives(_) => {}
@@ -1362,32 +1143,27 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         if let GenericBound::Trait(poly, modify) = bound {
             match (ctxt, modify) {
                 (BoundKind::SuperTraits, TraitBoundModifier::Maybe) => {
-                    let mut err = self
-                        .err_handler()
-                        .struct_span_err(poly.span, "`?Trait` is not permitted in supertraits");
-                    let path_str = pprust::path_to_string(&poly.trait_ref.path);
-                    err.note(&format!("traits are `?{}` by default", path_str));
-                    err.emit();
+                    self.err_handler().emit_err(errors::OptionalTraitSupertrait {
+                        span: poly.span,
+                        path_str: pprust::path_to_string(&poly.trait_ref.path)
+                    });
                 }
                 (BoundKind::TraitObject, TraitBoundModifier::Maybe) => {
-                    let mut err = self.err_handler().struct_span_err(
-                        poly.span,
-                        "`?Trait` is not permitted in trait object types",
-                    );
-                    err.emit();
+                    self.err_handler().emit_err(errors::OptionalTraitObject {span: poly.span});
                 }
                 (_, TraitBoundModifier::MaybeConst) if let Some(reason) = &self.disallow_tilde_const => {
-                    let mut err = self.err_handler().struct_span_err(bound.span(), "`~const` is not allowed here");
-                    match reason {
-                        DisallowTildeConstContext::TraitObject => err.note("trait objects cannot have `~const` trait bounds"),
-                        DisallowTildeConstContext::Fn(FnKind::Closure(..)) => err.note("closures cannot have `~const` trait bounds"),
-                        DisallowTildeConstContext::Fn(FnKind::Fn(_, ident, ..)) => err.span_note(ident.span, "this function is not `const`, so it cannot have `~const` trait bounds"),
+                    let reason = match reason {
+                        DisallowTildeConstContext::TraitObject => errors::TildeConstReason::TraitObject,
+                        DisallowTildeConstContext::Fn(FnKind::Closure(..)) => errors::TildeConstReason::Closure,
+                        DisallowTildeConstContext::Fn(FnKind::Fn(_, ident, ..)) => errors::TildeConstReason::Function { ident: ident.span },
                     };
-                    err.emit();
+                    self.err_handler().emit_err(errors::TildeConstDisallowed {
+                        span: bound.span(),
+                        reason
+                    });
                 }
                 (_, TraitBoundModifier::MaybeConstMaybe) => {
-                    self.err_handler()
-                        .span_err(bound.span(), "`~const` and `?` are mutually exclusive");
+                    self.err_handler().emit_err(errors::OptionalConstExclusive {span: bound.span()});
                 }
                 _ => {}
             }
@@ -1396,19 +1172,6 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         visit::walk_param_bound(self, bound)
     }
 
-    fn visit_poly_trait_ref(&mut self, t: &'a PolyTraitRef) {
-        self.check_late_bound_lifetime_defs(&t.bound_generic_params);
-        visit::walk_poly_trait_ref(self, t);
-    }
-
-    fn visit_variant_data(&mut self, s: &'a VariantData) {
-        self.with_banned_assoc_ty_bound(|this| visit::walk_struct_def(this, s))
-    }
-
-    fn visit_enum_def(&mut self, enum_definition: &'a EnumDef) {
-        self.with_banned_assoc_ty_bound(|this| visit::walk_enum_def(this, enum_definition))
-    }
-
     fn visit_fn(&mut self, fk: FnKind<'a>, span: Span, id: NodeId) {
         // Only associated `fn`s can have `self` parameters.
         let self_semantic = match fk.ctxt() {
@@ -1420,25 +1183,18 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         self.check_c_variadic_type(fk);
 
         // Functions cannot both be `const async`
-        if let Some(FnHeader {
+        if let Some(&FnHeader {
             constness: Const::Yes(cspan),
             asyncness: Async::Yes { span: aspan, .. },
             ..
         }) = fk.header()
         {
-            self.err_handler()
-                .struct_span_err(
-                    vec![*cspan, *aspan],
-                    "functions cannot be both `const` and `async`",
-                )
-                .span_label(*cspan, "`const` because of this")
-                .span_label(*aspan, "`async` because of this")
-                .span_label(span, "") // Point at the fn header.
-                .emit();
-        }
-
-        if let FnKind::Closure(ClosureBinder::For { generic_params, .. }, ..) = fk {
-            self.check_late_bound_lifetime_defs(generic_params);
+            self.err_handler().emit_err(errors::ConstAndAsync {
+                spans: vec![cspan, aspan],
+                cspan,
+                aspan,
+                span,
+            });
         }
 
         if let FnKind::Fn(
@@ -1456,20 +1212,12 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         // Functions without bodies cannot have patterns.
         if let FnKind::Fn(ctxt, _, sig, _, _, None) = fk {
             Self::check_decl_no_pat(&sig.decl, |span, ident, mut_ident| {
-                let (code, msg, label) = match ctxt {
-                    FnCtxt::Foreign => (
-                        error_code!(E0130),
-                        "patterns aren't allowed in foreign function declarations",
-                        "pattern not allowed in foreign function",
-                    ),
-                    _ => (
-                        error_code!(E0642),
-                        "patterns aren't allowed in functions without bodies",
-                        "pattern not allowed in function without body",
-                    ),
-                };
                 if mut_ident && matches!(ctxt, FnCtxt::Assoc(_)) {
                     if let Some(ident) = ident {
+                        let msg = match ctxt {
+                            FnCtxt::Foreign => fluent::ast_passes_pattern_in_foreign,
+                            _ => fluent::ast_passes_pattern_in_bodiless,
+                        };
                         let diag = BuiltinLintDiagnostics::PatternsInFnsWithoutBody(span, ident);
                         self.lint_buffer.buffer_lint_with_diagnostic(
                             PATTERNS_IN_FNS_WITHOUT_BODY,
@@ -1480,11 +1228,12 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                         )
                     }
                 } else {
-                    self.err_handler()
-                        .struct_span_err(span, msg)
-                        .span_label(span, label)
-                        .code(code)
-                        .emit();
+                    match ctxt {
+                        FnCtxt::Foreign => {
+                            self.err_handler().emit_err(errors::PatternInForeign { span })
+                        }
+                        _ => self.err_handler().emit_err(errors::PatternInBodiless { span }),
+                    };
                 }
             });
         }
@@ -1511,7 +1260,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
             match &item.kind {
                 AssocItemKind::Const(_, _, body) => {
                     if body.is_none() {
-                        self.session.emit_err(AssocConstWithoutBody {
+                        self.session.emit_err(errors::AssocConstWithoutBody {
                             span: item.span,
                             replace_span: self.ending_semi_or_hi(item.span),
                         });
@@ -1519,7 +1268,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 }
                 AssocItemKind::Fn(box Fn { body, .. }) => {
                     if body.is_none() {
-                        self.session.emit_err(AssocFnWithoutBody {
+                        self.session.emit_err(errors::AssocFnWithoutBody {
                             span: item.span,
                             replace_span: self.ending_semi_or_hi(item.span),
                         });
@@ -1534,7 +1283,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                     ..
                 }) => {
                     if ty.is_none() {
-                        self.session.emit_err(AssocTypeWithoutBody {
+                        self.session.emit_err(errors::AssocTypeWithoutBody {
                             span: item.span,
                             replace_span: self.ending_semi_or_hi(item.span),
                         });
@@ -1606,11 +1355,7 @@ fn deny_equality_constraints(
     predicate: &WhereEqPredicate,
     generics: &Generics,
 ) {
-    let mut err = this.err_handler().struct_span_err(
-        predicate.span,
-        "equality constraints are not yet supported in `where` clauses",
-    );
-    err.span_label(predicate.span, "not supported");
+    let mut err = errors::EqualityInWhere { span: predicate.span, assoc: None, assoc2: None };
 
     // Given `<A as Foo>::Bar = RhsTy`, suggest `A: Foo<Bar = RhsTy>`.
     if let TyKind::Path(Some(qself), full_path) = &predicate.lhs_ty.kind {
@@ -1649,25 +1394,17 @@ fn deny_equality_constraints(
                                         empty_args => {
                                             *empty_args = AngleBracketedArgs {
                                                 span: ident.span,
-                                                args: vec![arg],
+                                                args: thin_vec![arg],
                                             }
                                             .into();
                                         }
                                     }
-                                    err.span_suggestion_verbose(
-                                        predicate.span,
-                                        &format!(
-                                            "if `{}` is an associated type you're trying to set, \
-                                            use the associated type binding syntax",
-                                            ident
-                                        ),
-                                        format!(
-                                            "{}: {}",
-                                            param,
-                                            pprust::path_to_string(&assoc_path)
-                                        ),
-                                        Applicability::MaybeIncorrect,
-                                    );
+                                    err.assoc = Some(errors::AssociatedSuggestion {
+                                        span: predicate.span,
+                                        ident: *ident,
+                                        param: *param,
+                                        path: pprust::path_to_string(&assoc_path),
+                                    })
                                 }
                                 _ => {}
                             };
@@ -1709,15 +1446,13 @@ fn deny_equality_constraints(
                                         trait_segment.span().shrink_to_hi(),
                                     ),
                                 };
-                                err.multipart_suggestion(
-                                    &format!(
-                                        "if `{}::{}` is an associated type you're trying to set, \
-                                        use the associated type binding syntax",
-                                        trait_segment.ident, potential_assoc.ident,
-                                    ),
-                                    vec![(span, args), (predicate.span, String::new())],
-                                    Applicability::MaybeIncorrect,
-                                );
+                                err.assoc2 = Some(errors::AssociatedSuggestion2 {
+                                    span,
+                                    args,
+                                    predicate: predicate.span,
+                                    trait_segment: trait_segment.ident,
+                                    potential_assoc: potential_assoc.ident,
+                                });
                             }
                         }
                     }
@@ -1725,10 +1460,7 @@ fn deny_equality_constraints(
             }
         }
     }
-    err.note(
-        "see issue #20041 <https://github.com/rust-lang/rust/issues/20041> for more information",
-    );
-    err.emit();
+    this.err_handler().emit_err(err);
 }
 
 pub fn check_crate(session: &Session, krate: &Crate, lints: &mut LintBuffer) -> bool {
@@ -1741,7 +1473,6 @@ pub fn check_crate(session: &Session, krate: &Crate, lints: &mut LintBuffer) ->
         outer_impl_trait: None,
         disallow_tilde_const: None,
         is_impl_trait_banned: false,
-        is_assoc_ty_bound_banned: false,
         forbidden_let_reason: Some(ForbiddenLetReason::GenericForbidden),
         lint_buffer: lints,
     };
@@ -1756,12 +1487,12 @@ pub(crate) enum ForbiddenLetReason {
     /// `let` is not valid and the source environment is not important
     GenericForbidden,
     /// A let chain with the `||` operator
-    #[note(not_supported_or)]
+    #[note(ast_passes_not_supported_or)]
     NotSupportedOr(#[primary_span] Span),
     /// A let chain with invalid parentheses
     ///
     /// For example, `let 1 = 1 && (expr && expr)` is allowed
     /// but `(let 1 = 1 && (let 1 = 1 && (let 1 = 1))) && let a = 1` is not
-    #[note(not_supported_parentheses)]
+    #[note(ast_passes_not_supported_parentheses)]
     NotSupportedParentheses(#[primary_span] Span),
 }
diff --git a/compiler/rustc_ast_passes/src/errors.rs b/compiler/rustc_ast_passes/src/errors.rs
index 09e262452..d007097d9 100644
--- a/compiler/rustc_ast_passes/src/errors.rs
+++ b/compiler/rustc_ast_passes/src/errors.rs
@@ -1,9 +1,12 @@
 //! Errors emitted by ast_passes.
 
+use rustc_ast::ParamKindOrd;
+use rustc_errors::AddToDiagnostic;
 use rustc_macros::{Diagnostic, Subdiagnostic};
-use rustc_span::{Span, Symbol};
+use rustc_span::{symbol::Ident, Span, Symbol};
 
 use crate::ast_validation::ForbiddenLetReason;
+use crate::fluent_generated as fluent;
 
 #[derive(Diagnostic)]
 #[diag(ast_passes_forbidden_let)]
@@ -24,13 +27,6 @@ pub struct ForbiddenLetStable {
 }
 
 #[derive(Diagnostic)]
-#[diag(ast_passes_forbidden_assoc_constraint)]
-pub struct ForbiddenAssocConstraint {
-    #[primary_span]
-    pub span: Span,
-}
-
-#[derive(Diagnostic)]
 #[diag(ast_passes_keyword_lifetime)]
 pub struct KeywordLifetime {
     #[primary_span]
@@ -50,7 +46,7 @@ pub struct InvalidLabel {
 pub struct InvalidVisibility {
     #[primary_span]
     pub span: Span,
-    #[label(implied)]
+    #[label(ast_passes_implied)]
     pub implied: Option<Span>,
     #[subdiagnostic]
     pub note: Option<InvalidVisibilityNote>,
@@ -58,9 +54,9 @@ pub struct InvalidVisibility {
 
 #[derive(Subdiagnostic)]
 pub enum InvalidVisibilityNote {
-    #[note(individual_impl_items)]
+    #[note(ast_passes_individual_impl_items)]
     IndividualImplItems,
-    #[note(individual_foreign_items)]
+    #[note(ast_passes_individual_foreign_items)]
     IndividualForeignItems,
 }
 
@@ -224,3 +220,474 @@ pub enum ExternBlockSuggestion {
         abi: Symbol,
     },
 }
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_bound_in_context)]
+pub struct BoundInContext<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub ctx: &'a str,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_extern_types_cannot)]
+#[note(ast_passes_extern_keyword_link)]
+pub struct ExternTypesCannotHave<'a> {
+    #[primary_span]
+    #[suggestion(code = "", applicability = "maybe-incorrect")]
+    pub span: Span,
+    pub descr: &'a str,
+    pub remove_descr: &'a str,
+    #[label]
+    pub block_span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_body_in_extern)]
+#[note(ast_passes_extern_keyword_link)]
+pub struct BodyInExtern<'a> {
+    #[primary_span]
+    #[label(ast_passes_cannot_have)]
+    pub span: Span,
+    #[label(ast_passes_invalid)]
+    pub body: Span,
+    #[label(ast_passes_existing)]
+    pub block: Span,
+    pub kind: &'a str,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_fn_body_extern)]
+#[help]
+#[note(ast_passes_extern_keyword_link)]
+pub struct FnBodyInExtern {
+    #[primary_span]
+    #[label(ast_passes_cannot_have)]
+    pub span: Span,
+    #[suggestion(code = ";", applicability = "maybe-incorrect")]
+    pub body: Span,
+    #[label]
+    pub block: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_extern_fn_qualifiers)]
+pub struct FnQualifierInExtern {
+    #[primary_span]
+    pub span: Span,
+    #[label]
+    pub block: Span,
+    #[suggestion(code = "fn ", applicability = "maybe-incorrect", style = "verbose")]
+    pub sugg_span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_extern_item_ascii)]
+#[note]
+pub struct ExternItemAscii {
+    #[primary_span]
+    pub span: Span,
+    #[label]
+    pub block: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_bad_c_variadic)]
+pub struct BadCVariadic {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_item_underscore)]
+pub struct ItemUnderscore<'a> {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    pub kind: &'a str,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_nomangle_ascii, code = "E0754")]
+pub struct NoMangleAscii {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_module_nonascii, code = "E0754")]
+#[help]
+pub struct ModuleNonAscii {
+    #[primary_span]
+    pub span: Span,
+    pub name: Symbol,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_auto_generic, code = "E0567")]
+pub struct AutoTraitGeneric {
+    #[primary_span]
+    #[suggestion(code = "", applicability = "machine-applicable")]
+    pub span: Span,
+    #[label]
+    pub ident: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_auto_super_lifetime, code = "E0568")]
+pub struct AutoTraitBounds {
+    #[primary_span]
+    #[suggestion(code = "", applicability = "machine-applicable")]
+    pub span: Span,
+    #[label]
+    pub ident: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_auto_items, code = "E0380")]
+pub struct AutoTraitItems {
+    #[primary_span]
+    pub spans: Vec<Span>,
+    #[suggestion(code = "", applicability = "machine-applicable")]
+    pub total: Span,
+    #[label]
+    pub ident: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_generic_before_constraints)]
+pub struct ArgsBeforeConstraint {
+    #[primary_span]
+    pub arg_spans: Vec<Span>,
+    #[label(ast_passes_constraints)]
+    pub constraints: Span,
+    #[label(ast_passes_args)]
+    pub args: Span,
+    #[suggestion(code = "{suggestion}", applicability = "machine-applicable", style = "verbose")]
+    pub data: Span,
+    pub suggestion: String,
+    pub constraint_len: usize,
+    pub args_len: usize,
+    #[subdiagnostic]
+    pub constraint_spans: EmptyLabelManySpans,
+    #[subdiagnostic]
+    pub arg_spans2: EmptyLabelManySpans,
+}
+
+pub struct EmptyLabelManySpans(pub Vec<Span>);
+
+// The derive for `Vec<Span>` does multiple calls to `span_label`, adding commas between each
+impl AddToDiagnostic for EmptyLabelManySpans {
+    fn add_to_diagnostic_with<F>(self, diag: &mut rustc_errors::Diagnostic, _: F)
+    where
+        F: Fn(
+            &mut rustc_errors::Diagnostic,
+            rustc_errors::SubdiagnosticMessage,
+        ) -> rustc_errors::SubdiagnosticMessage,
+    {
+        diag.span_labels(self.0, "");
+    }
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_pattern_in_fn_pointer, code = "E0561")]
+pub struct PatternFnPointer {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_trait_object_single_bound, code = "E0226")]
+pub struct TraitObjectBound {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_impl_trait_path, code = "E0667")]
+pub struct ImplTraitPath {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_nested_impl_trait, code = "E0666")]
+pub struct NestedImplTrait {
+    #[primary_span]
+    pub span: Span,
+    #[label(ast_passes_outer)]
+    pub outer: Span,
+    #[label(ast_passes_inner)]
+    pub inner: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_at_least_one_trait)]
+pub struct AtLeastOneTrait {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_out_of_order_params)]
+pub struct OutOfOrderParams<'a> {
+    #[primary_span]
+    pub spans: Vec<Span>,
+    #[suggestion(code = "{ordered_params}", applicability = "machine-applicable")]
+    pub sugg_span: Span,
+    pub param_ord: &'a ParamKindOrd,
+    pub max_param: &'a ParamKindOrd,
+    pub ordered_params: &'a str,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_obsolete_auto)]
+#[help]
+pub struct ObsoleteAuto {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_unsafe_negative_impl, code = "E0198")]
+pub struct UnsafeNegativeImpl {
+    #[primary_span]
+    pub span: Span,
+    #[label(ast_passes_negative)]
+    pub negative: Span,
+    #[label(ast_passes_unsafe)]
+    pub r#unsafe: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_inherent_cannot_be)]
+pub struct InherentImplCannot<'a> {
+    #[primary_span]
+    pub span: Span,
+    #[label(ast_passes_because)]
+    pub annotation_span: Span,
+    pub annotation: &'a str,
+    #[label(ast_passes_type)]
+    pub self_ty: Span,
+    #[note(ast_passes_only_trait)]
+    pub only_trait: Option<()>,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_inherent_cannot_be, code = "E0197")]
+pub struct InherentImplCannotUnsafe<'a> {
+    #[primary_span]
+    pub span: Span,
+    #[label(ast_passes_because)]
+    pub annotation_span: Span,
+    pub annotation: &'a str,
+    #[label(ast_passes_type)]
+    pub self_ty: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_unsafe_item)]
+pub struct UnsafeItem {
+    #[primary_span]
+    pub span: Span,
+    pub kind: &'static str,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_fieldless_union)]
+pub struct FieldlessUnion {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_where_after_type_alias)]
+#[note]
+pub struct WhereAfterTypeAlias {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_generic_default_trailing)]
+pub struct GenericDefaultTrailing {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_nested_lifetimes, code = "E0316")]
+pub struct NestedLifetimes {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_optional_trait_supertrait)]
+#[note]
+pub struct OptionalTraitSupertrait {
+    #[primary_span]
+    pub span: Span,
+    pub path_str: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_optional_trait_object)]
+pub struct OptionalTraitObject {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_tilde_const_disallowed)]
+pub struct TildeConstDisallowed {
+    #[primary_span]
+    pub span: Span,
+    #[subdiagnostic]
+    pub reason: TildeConstReason,
+}
+
+#[derive(Subdiagnostic)]
+pub enum TildeConstReason {
+    #[note(ast_passes_trait)]
+    TraitObject,
+    #[note(ast_passes_closure)]
+    Closure,
+    #[note(ast_passes_function)]
+    Function {
+        #[primary_span]
+        ident: Span,
+    },
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_optional_const_exclusive)]
+pub struct OptionalConstExclusive {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_const_and_async)]
+pub struct ConstAndAsync {
+    #[primary_span]
+    pub spans: Vec<Span>,
+    #[label(ast_passes_const)]
+    pub cspan: Span,
+    #[label(ast_passes_async)]
+    pub aspan: Span,
+    #[label]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_pattern_in_foreign, code = "E0130")]
+pub struct PatternInForeign {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_pattern_in_bodiless, code = "E0642")]
+pub struct PatternInBodiless {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_equality_in_where)]
+#[note]
+pub struct EqualityInWhere {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    #[subdiagnostic]
+    pub assoc: Option<AssociatedSuggestion>,
+    #[subdiagnostic]
+    pub assoc2: Option<AssociatedSuggestion2>,
+}
+
+#[derive(Subdiagnostic)]
+#[suggestion(
+    ast_passes_suggestion,
+    code = "{param}: {path}",
+    style = "verbose",
+    applicability = "maybe-incorrect"
+)]
+pub struct AssociatedSuggestion {
+    #[primary_span]
+    pub span: Span,
+    pub ident: Ident,
+    pub param: Ident,
+    pub path: String,
+}
+
+#[derive(Subdiagnostic)]
+#[multipart_suggestion(ast_passes_suggestion_path, applicability = "maybe-incorrect")]
+pub struct AssociatedSuggestion2 {
+    #[suggestion_part(code = "{args}")]
+    pub span: Span,
+    pub args: String,
+    #[suggestion_part(code = "")]
+    pub predicate: Span,
+    pub trait_segment: Ident,
+    pub potential_assoc: Ident,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_stability_outside_std, code = "E0734")]
+pub struct StabilityOutsideStd {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_feature_on_non_nightly, code = "E0554")]
+pub struct FeatureOnNonNightly {
+    #[primary_span]
+    pub span: Span,
+    pub channel: &'static str,
+    #[subdiagnostic]
+    pub stable_features: Vec<StableFeature>,
+    #[suggestion(code = "", applicability = "machine-applicable")]
+    pub sugg: Option<Span>,
+}
+
+pub struct StableFeature {
+    pub name: Symbol,
+    pub since: Symbol,
+}
+
+impl AddToDiagnostic for StableFeature {
+    fn add_to_diagnostic_with<F>(self, diag: &mut rustc_errors::Diagnostic, _: F)
+    where
+        F: Fn(
+            &mut rustc_errors::Diagnostic,
+            rustc_errors::SubdiagnosticMessage,
+        ) -> rustc_errors::SubdiagnosticMessage,
+    {
+        diag.set_arg("name", self.name);
+        diag.set_arg("since", self.since);
+        diag.help(fluent::ast_passes_stable_since);
+    }
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_incompatbile_features)]
+#[help]
+pub struct IncompatibleFeatures {
+    #[primary_span]
+    pub spans: Vec<Span>,
+    pub f1: Symbol,
+    pub f2: Symbol,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_show_span)]
+pub struct ShowSpan {
+    #[primary_span]
+    pub span: Span,
+    pub msg: &'static str,
+}
diff --git a/compiler/rustc_ast_passes/src/feature_gate.rs b/compiler/rustc_ast_passes/src/feature_gate.rs
index 89ba6f936..96042ea30 100644
--- a/compiler/rustc_ast_passes/src/feature_gate.rs
+++ b/compiler/rustc_ast_passes/src/feature_gate.rs
@@ -2,7 +2,7 @@ use rustc_ast as ast;
 use rustc_ast::visit::{self, AssocCtxt, FnCtxt, FnKind, Visitor};
 use rustc_ast::{AssocConstraint, AssocConstraintKind, NodeId};
 use rustc_ast::{PatKind, RangeEnd};
-use rustc_errors::{struct_span_err, Applicability, StashKey};
+use rustc_errors::{Applicability, StashKey};
 use rustc_feature::{AttributeGate, BuiltinAttribute, Features, GateIssue, BUILTIN_ATTRIBUTE_MAP};
 use rustc_session::parse::{feature_err, feature_err_issue, feature_warn};
 use rustc_session::Session;
@@ -10,6 +10,10 @@ use rustc_span::source_map::Spanned;
 use rustc_span::symbol::sym;
 use rustc_span::Span;
 use rustc_target::spec::abi;
+use thin_vec::ThinVec;
+use tracing::debug;
+
+use crate::errors;
 
 macro_rules! gate_feature_fn {
     ($visitor: expr, $has_feature: expr, $span: expr, $name: expr, $explain: expr, $help: expr) => {{
@@ -136,6 +140,34 @@ impl<'a> PostExpansionVisitor<'a> {
         }
         ImplTraitVisitor { vis: self }.visit_ty(ty);
     }
+
+    fn check_late_bound_lifetime_defs(&self, params: &[ast::GenericParam]) {
+        // Check only lifetime parameters are present and that the lifetime
+        // parameters that are present have no bounds.
+        let non_lt_param_spans: Vec<_> = params
+            .iter()
+            .filter_map(|param| match param.kind {
+                ast::GenericParamKind::Lifetime { .. } => None,
+                _ => Some(param.ident.span),
+            })
+            .collect();
+        // FIXME: gate_feature_post doesn't really handle multispans...
+        if !non_lt_param_spans.is_empty() && !self.features.non_lifetime_binders {
+            feature_err(
+                &self.sess.parse_sess,
+                sym::non_lifetime_binders,
+                non_lt_param_spans,
+                crate::fluent_generated::ast_passes_forbidden_non_lifetime_param,
+            )
+            .emit();
+        }
+        for param in params {
+            if !param.bounds.is_empty() {
+                let spans: Vec<_> = param.bounds.iter().map(|b| b.span()).collect();
+                self.sess.emit_err(errors::ForbiddenLifetimeBound { spans });
+            }
+        }
+    }
 }
 
 impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
@@ -147,7 +179,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
             ..
         }) = attr_info
         {
-            gate_feature_fn!(self, has_feature, attr.span, *name, descr);
+            gate_feature_fn!(self, has_feature, attr.span, *name, *descr);
         }
         // Check unstable flavors of the `#[doc]` attribute.
         if attr.has_name(sym::doc) {
@@ -186,13 +218,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
                 || attr.has_name(sym::rustc_const_stable)
                 || attr.has_name(sym::rustc_default_body_unstable)
             {
-                struct_span_err!(
-                    self.sess,
-                    attr.span,
-                    E0734,
-                    "stability attributes may not be used outside of the standard library",
-                )
-                .emit();
+                self.sess.emit_err(errors::StabilityOutsideStd { span: attr.span });
             }
         }
     }
@@ -220,7 +246,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
 
             ast::ItemKind::Struct(..) => {
                 for attr in self.sess.filter_by_name(&i.attrs, sym::repr) {
-                    for item in attr.meta_item_list().unwrap_or_else(Vec::new) {
+                    for item in attr.meta_item_list().unwrap_or_else(ThinVec::new) {
                         if item.has_name(sym::simd) {
                             gate_feature_post!(
                                 &self,
@@ -306,6 +332,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
             ast::TyKind::BareFn(bare_fn_ty) => {
                 // Function pointers cannot be `const`
                 self.check_extern(bare_fn_ty.ext, ast::Const::No);
+                self.check_late_bound_lifetime_defs(&bare_fn_ty.generic_params);
             }
             ast::TyKind::Never => {
                 gate_feature_post!(&self, never_type, ty.span, "the `!` type is experimental");
@@ -318,6 +345,19 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
         visit::walk_ty(self, ty)
     }
 
+    fn visit_generics(&mut self, g: &'a ast::Generics) {
+        for predicate in &g.where_clause.predicates {
+            match predicate {
+                ast::WherePredicate::BoundPredicate(bound_pred) => {
+                    // A type binding, eg `for<'c> Foo: Send+Clone+'c`
+                    self.check_late_bound_lifetime_defs(&bound_pred.bound_generic_params);
+                }
+                _ => {}
+            }
+        }
+        visit::walk_generics(self, g);
+    }
+
     fn visit_fn_ret_ty(&mut self, ret_ty: &'a ast::FnRetTy) {
         if let ast::FnRetTy::Ty(output_ty) = ret_ty {
             if let ast::TyKind::Never = output_ty.kind {
@@ -346,7 +386,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
             ).span_suggestion_verbose(
                 lhs.span.shrink_to_lo(),
                 "you might have meant to introduce a new binding",
-                "let ".to_string(),
+                "let ",
                 Applicability::MachineApplicable,
             ).emit();
         }
@@ -437,12 +477,21 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
         visit::walk_pat(self, pattern)
     }
 
+    fn visit_poly_trait_ref(&mut self, t: &'a ast::PolyTraitRef) {
+        self.check_late_bound_lifetime_defs(&t.bound_generic_params);
+        visit::walk_poly_trait_ref(self, t);
+    }
+
     fn visit_fn(&mut self, fn_kind: FnKind<'a>, span: Span, _: NodeId) {
         if let Some(header) = fn_kind.header() {
             // Stability of const fn methods are covered in `visit_assoc_item` below.
             self.check_extern(header.ext, header.constness);
         }
 
+        if let FnKind::Closure(ast::ClosureBinder::For { generic_params, .. }, ..) = fn_kind {
+            self.check_late_bound_lifetime_defs(generic_params);
+        }
+
         if fn_kind.ctxt() != Some(FnCtxt::Foreign) && fn_kind.decl().c_variadic() {
             gate_feature_post!(&self, c_variadic, span, "C-variadic functions are unstable");
         }
@@ -580,13 +629,13 @@ fn maybe_stage_features(sess: &Session, krate: &ast::Crate) {
             return;
         }
         for attr in krate.attrs.iter().filter(|attr| attr.has_name(sym::feature)) {
-            let mut err = struct_span_err!(
-                sess.parse_sess.span_diagnostic,
-                attr.span,
-                E0554,
-                "`#![feature]` may not be used on the {} release channel",
-                option_env!("CFG_RELEASE_CHANNEL").unwrap_or("(unknown)")
-            );
+            let mut err = errors::FeatureOnNonNightly {
+                span: attr.span,
+                channel: option_env!("CFG_RELEASE_CHANNEL").unwrap_or("(unknown)"),
+                stable_features: vec![],
+                sugg: None,
+            };
+
             let mut all_stable = true;
             for ident in
                 attr.meta_item_list().into_iter().flatten().flat_map(|nested| nested.ident())
@@ -597,24 +646,15 @@ fn maybe_stage_features(sess: &Session, krate: &ast::Crate) {
                     .flat_map(|&(feature, _, since)| if feature == name { since } else { None })
                     .next();
                 if let Some(since) = stable_since {
-                    err.help(&format!(
-                        "the feature `{}` has been stable since {} and no longer requires \
-                                  an attribute to enable",
-                        name, since
-                    ));
+                    err.stable_features.push(errors::StableFeature { name, since });
                 } else {
                     all_stable = false;
                 }
             }
             if all_stable {
-                err.span_suggestion(
-                    attr.span,
-                    "remove the attribute",
-                    "",
-                    Applicability::MachineApplicable,
-                );
+                err.sugg = Some(attr.span);
             }
-            err.emit();
+            sess.parse_sess.span_diagnostic.emit_err(err);
         }
     }
 }
@@ -637,16 +677,7 @@ fn check_incompatible_features(sess: &Session) {
             if let Some((f2_name, f2_span)) = declared_features.clone().find(|(name, _)| name == f2)
             {
                 let spans = vec![f1_span, f2_span];
-                sess.struct_span_err(
-                    spans,
-                    &format!(
-                        "features `{}` and `{}` are incompatible, using them at the same time \
-                        is not allowed",
-                        f1_name, f2_name
-                    ),
-                )
-                .help("remove one of these features")
-                .emit();
+                sess.emit_err(errors::IncompatibleFeatures { spans, f1: f1_name, f2: f2_name });
             }
         }
     }
diff --git a/compiler/rustc_ast_passes/src/lib.rs b/compiler/rustc_ast_passes/src/lib.rs
index f58fffc91..b9dcaee23 100644
--- a/compiler/rustc_ast_passes/src/lib.rs
+++ b/compiler/rustc_ast_passes/src/lib.rs
@@ -10,12 +10,16 @@
 #![feature(iter_is_partitioned)]
 #![feature(let_chains)]
 #![recursion_limit = "256"]
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 
-#[macro_use]
-extern crate tracing;
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
+use rustc_macros::fluent_messages;
 
 pub mod ast_validation;
 mod errors;
 pub mod feature_gate;
 pub mod node_count;
 pub mod show_span;
+
+fluent_messages! { "../locales/en-US.ftl" }
diff --git a/compiler/rustc_ast_passes/src/show_span.rs b/compiler/rustc_ast_passes/src/show_span.rs
index 27637e311..280cf3284 100644
--- a/compiler/rustc_ast_passes/src/show_span.rs
+++ b/compiler/rustc_ast_passes/src/show_span.rs
@@ -9,6 +9,8 @@ use rustc_ast as ast;
 use rustc_ast::visit;
 use rustc_ast::visit::Visitor;
 
+use crate::errors;
+
 enum Mode {
     Expression,
     Pattern,
@@ -36,21 +38,21 @@ struct ShowSpanVisitor<'a> {
 impl<'a> Visitor<'a> for ShowSpanVisitor<'a> {
     fn visit_expr(&mut self, e: &'a ast::Expr) {
         if let Mode::Expression = self.mode {
-            self.span_diagnostic.span_warn(e.span, "expression");
+            self.span_diagnostic.emit_warning(errors::ShowSpan { span: e.span, msg: "expression" });
         }
         visit::walk_expr(self, e);
     }
 
     fn visit_pat(&mut self, p: &'a ast::Pat) {
         if let Mode::Pattern = self.mode {
-            self.span_diagnostic.span_warn(p.span, "pattern");
+            self.span_diagnostic.emit_warning(errors::ShowSpan { span: p.span, msg: "pattern" });
         }
         visit::walk_pat(self, p);
     }
 
     fn visit_ty(&mut self, t: &'a ast::Ty) {
         if let Mode::Type = self.mode {
-            self.span_diagnostic.span_warn(t.span, "type");
+            self.span_diagnostic.emit_warning(errors::ShowSpan { span: t.span, msg: "type" });
         }
         visit::walk_ty(self, t);
     }
diff --git a/compiler/rustc_ast_pretty/Cargo.toml b/compiler/rustc_ast_pretty/Cargo.toml
index a3e3e823b..980a8fa93 100644
--- a/compiler/rustc_ast_pretty/Cargo.toml
+++ b/compiler/rustc_ast_pretty/Cargo.toml
@@ -6,5 +6,6 @@ edition = "2021"
 [lib]
 
 [dependencies]
-rustc_span = { path = "../rustc_span" }
 rustc_ast = { path = "../rustc_ast" }
+rustc_span = { path = "../rustc_span" }
+thin-vec = "0.2.12"
diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs
index 6a8064b0e..694d688bf 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state.rs
@@ -4,7 +4,7 @@ mod item;
 
 use crate::pp::Breaks::{Consistent, Inconsistent};
 use crate::pp::{self, Breaks};
-
+use rustc_ast::attr::AttrIdGenerator;
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, BinOpToken, CommentKind, Delimiter, Nonterminal, Token, TokenKind};
 use rustc_ast::tokenstream::{TokenStream, TokenTree};
@@ -20,9 +20,8 @@ use rustc_span::edition::Edition;
 use rustc_span::source_map::{SourceMap, Spanned};
 use rustc_span::symbol::{kw, sym, Ident, IdentPrinter, Symbol};
 use rustc_span::{BytePos, FileName, Span, DUMMY_SP};
-
-use rustc_ast::attr::AttrIdGenerator;
 use std::borrow::Cow;
+use thin_vec::ThinVec;
 
 pub use self::delimited::IterDelimited;
 
@@ -131,7 +130,7 @@ pub fn print_crate<'a>(
 
         // Currently, in Rust 2018 we don't have `extern crate std;` at the crate
         // root, so this is not needed, and actually breaks things.
-        if edition == Edition::Edition2015 {
+        if edition.is_rust_2015() {
             // `#![no_std]`
             let fake_attr = attr::mk_attr_word(g, ast::AttrStyle::Inner, sym::no_std, DUMMY_SP);
             s.print_attribute(&fake_attr);
@@ -1567,8 +1566,18 @@ impl<'a> State<'a> {
 
             match bound {
                 GenericBound::Trait(tref, modifier) => {
-                    if modifier == &TraitBoundModifier::Maybe {
-                        self.word("?");
+                    match modifier {
+                        TraitBoundModifier::None => {}
+                        TraitBoundModifier::Maybe => {
+                            self.word("?");
+                        }
+                        TraitBoundModifier::MaybeConst => {
+                            self.word_space("~const");
+                        }
+                        TraitBoundModifier::MaybeConstMaybe => {
+                            self.word_space("~const");
+                            self.word("?");
+                        }
                     }
                     self.print_poly_trait_ref(tref);
                 }
@@ -1712,10 +1721,10 @@ impl<'a> State<'a> {
         self.ibox(INDENT_UNIT);
         self.print_formal_generic_params(generic_params);
         let generics = ast::Generics {
-            params: Vec::new(),
+            params: ThinVec::new(),
             where_clause: ast::WhereClause {
                 has_where_token: false,
-                predicates: Vec::new(),
+                predicates: ThinVec::new(),
                 span: DUMMY_SP,
             },
             span: DUMMY_SP,
diff --git a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
index 2a18e5164..cacfe9eb2 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
@@ -6,6 +6,11 @@ use rustc_ast::token;
 use rustc_ast::util::literal::escape_byte_str_symbol;
 use rustc_ast::util::parser::{self, AssocOp, Fixity};
 use rustc_ast::{self as ast, BlockCheckMode};
+use rustc_ast::{
+    FormatAlignment, FormatArgPosition, FormatArgsPiece, FormatCount, FormatDebugHex, FormatSign,
+    FormatTrait,
+};
+use std::fmt::Write;
 
 impl<'a> State<'a> {
     fn print_else(&mut self, els: Option<&ast::Expr>) {
@@ -527,9 +532,23 @@ impl<'a> State<'a> {
                 }
             }
             ast::ExprKind::InlineAsm(a) => {
+                // FIXME: This should have its own syntax, distinct from a macro invocation.
                 self.word("asm!");
                 self.print_inline_asm(a);
             }
+            ast::ExprKind::FormatArgs(fmt) => {
+                // FIXME: This should have its own syntax, distinct from a macro invocation.
+                self.word("format_args!");
+                self.popen();
+                self.rbox(0, Inconsistent);
+                self.word(reconstruct_format_args_template_string(&fmt.template));
+                for arg in fmt.arguments.all_args() {
+                    self.word_space(",");
+                    self.print_expr(&arg.expr);
+                }
+                self.end();
+                self.pclose();
+            }
             ast::ExprKind::MacCall(m) => self.print_mac(m),
             ast::ExprKind::Paren(e) => {
                 self.popen();
@@ -629,3 +648,88 @@ impl<'a> State<'a> {
         }
     }
 }
+
+pub fn reconstruct_format_args_template_string(pieces: &[FormatArgsPiece]) -> String {
+    let mut template = "\"".to_string();
+    for piece in pieces {
+        match piece {
+            FormatArgsPiece::Literal(s) => {
+                for c in s.as_str().escape_debug() {
+                    template.push(c);
+                    if let '{' | '}' = c {
+                        template.push(c);
+                    }
+                }
+            }
+            FormatArgsPiece::Placeholder(p) => {
+                template.push('{');
+                let (Ok(n) | Err(n)) = p.argument.index;
+                write!(template, "{n}").unwrap();
+                if p.format_options != Default::default() || p.format_trait != FormatTrait::Display
+                {
+                    template.push_str(":");
+                }
+                if let Some(fill) = p.format_options.fill {
+                    template.push(fill);
+                }
+                match p.format_options.alignment {
+                    Some(FormatAlignment::Left) => template.push_str("<"),
+                    Some(FormatAlignment::Right) => template.push_str(">"),
+                    Some(FormatAlignment::Center) => template.push_str("^"),
+                    None => {}
+                }
+                match p.format_options.sign {
+                    Some(FormatSign::Plus) => template.push('+'),
+                    Some(FormatSign::Minus) => template.push('-'),
+                    None => {}
+                }
+                if p.format_options.alternate {
+                    template.push('#');
+                }
+                if p.format_options.zero_pad {
+                    template.push('0');
+                }
+                if let Some(width) = &p.format_options.width {
+                    match width {
+                        FormatCount::Literal(n) => write!(template, "{n}").unwrap(),
+                        FormatCount::Argument(FormatArgPosition {
+                            index: Ok(n) | Err(n), ..
+                        }) => {
+                            write!(template, "{n}$").unwrap();
+                        }
+                    }
+                }
+                if let Some(precision) = &p.format_options.precision {
+                    template.push('.');
+                    match precision {
+                        FormatCount::Literal(n) => write!(template, "{n}").unwrap(),
+                        FormatCount::Argument(FormatArgPosition {
+                            index: Ok(n) | Err(n), ..
+                        }) => {
+                            write!(template, "{n}$").unwrap();
+                        }
+                    }
+                }
+                match p.format_options.debug_hex {
+                    Some(FormatDebugHex::Lower) => template.push('x'),
+                    Some(FormatDebugHex::Upper) => template.push('X'),
+                    None => {}
+                }
+                template.push_str(match p.format_trait {
+                    FormatTrait::Display => "",
+                    FormatTrait::Debug => "?",
+                    FormatTrait::LowerExp => "e",
+                    FormatTrait::UpperExp => "E",
+                    FormatTrait::Octal => "o",
+                    FormatTrait::Pointer => "p",
+                    FormatTrait::Binary => "b",
+                    FormatTrait::LowerHex => "x",
+                    FormatTrait::UpperHex => "X",
+                });
+                template.push('}');
+            }
+        }
+    }
+    template.push('"');
+    template
+}
diff --git a/compiler/rustc_ast_pretty/src/pprust/tests.rs b/compiler/rustc_ast_pretty/src/pprust/tests.rs
index 6c8d42f33..3b2b60a86 100644
--- a/compiler/rustc_ast_pretty/src/pprust/tests.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/tests.rs
@@ -3,6 +3,7 @@ use super::*;
 use rustc_ast as ast;
 use rustc_span::create_default_session_globals_then;
 use rustc_span::symbol::Ident;
+use thin_vec::ThinVec;
 
 fn fun_to_string(
     decl: &ast::FnDecl,
@@ -27,8 +28,10 @@ fn test_fun_to_string() {
     create_default_session_globals_then(|| {
         let abba_ident = Ident::from_str("abba");
 
-        let decl =
-            ast::FnDecl { inputs: Vec::new(), output: ast::FnRetTy::Default(rustc_span::DUMMY_SP) };
+        let decl = ast::FnDecl {
+            inputs: ThinVec::new(),
+            output: ast::FnRetTy::Default(rustc_span::DUMMY_SP),
+        };
         let generics = ast::Generics::default();
         assert_eq!(
             fun_to_string(&decl, ast::FnHeader::default(), abba_ident, &generics),
diff --git a/compiler/rustc_error_messages/locales/en-US/attr.ftl b/compiler/rustc_attr/locales/en-US.ftl
index a7f8c993d..a7f8c993d 100644
--- a/compiler/rustc_error_messages/locales/en-US/attr.ftl
+++ b/compiler/rustc_attr/locales/en-US.ftl
diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs
index 40531c1c1..3d240108b 100644
--- a/compiler/rustc_attr/src/builtin.rs
+++ b/compiler/rustc_attr/src/builtin.rs
@@ -731,7 +731,7 @@ pub fn eval_condition(
                             sess,
                             sym::cfg_target_compact,
                             cfg.span,
-                            &"compact `cfg(target(..))` is experimental and subject to change"
+                            "compact `cfg(target(..))` is experimental and subject to change"
                         ).emit();
                     }
 
diff --git a/compiler/rustc_attr/src/lib.rs b/compiler/rustc_attr/src/lib.rs
index 4580ffcc6..5fede0a58 100644
--- a/compiler/rustc_attr/src/lib.rs
+++ b/compiler/rustc_attr/src/lib.rs
@@ -11,6 +11,9 @@
 #[macro_use]
 extern crate rustc_macros;
 
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
+use rustc_macros::fluent_messages;
+
 mod builtin;
 mod session_diagnostics;
 
@@ -22,3 +25,5 @@ pub use StabilityLevel::*;
 pub use rustc_ast::attr::*;
 
 pub(crate) use rustc_ast::HashStableContext;
+
+fluent_messages! { "../locales/en-US.ftl" }
diff --git a/compiler/rustc_attr/src/session_diagnostics.rs b/compiler/rustc_attr/src/session_diagnostics.rs
index 3ba7a3c53..ee79545e3 100644
--- a/compiler/rustc_attr/src/session_diagnostics.rs
+++ b/compiler/rustc_attr/src/session_diagnostics.rs
@@ -2,11 +2,12 @@ use std::num::IntErrorKind;
 
 use rustc_ast as ast;
 use rustc_errors::{
-    error_code, fluent, Applicability, DiagnosticBuilder, ErrorGuaranteed, Handler, IntoDiagnostic,
+    error_code, Applicability, DiagnosticBuilder, ErrorGuaranteed, Handler, IntoDiagnostic,
 };
 use rustc_macros::Diagnostic;
 use rustc_span::{Span, Symbol};
 
+use crate::fluent_generated as fluent;
 use crate::UnsupportedLiteralReason;
 
 #[derive(Diagnostic)]
@@ -59,7 +60,7 @@ impl<'a> IntoDiagnostic<'a> for UnknownMetaItem<'_> {
         );
         diag.set_arg("item", self.item);
         diag.set_arg("expected", expected.join(", "));
-        diag.span_label(self.span, fluent::label);
+        diag.span_label(self.span, fluent::attr_label);
         diag
     }
 }
@@ -99,31 +100,31 @@ pub(crate) struct InvalidIssueString {
 // translatable.
 #[derive(Subdiagnostic)]
 pub(crate) enum InvalidIssueStringCause {
-    #[label(must_not_be_zero)]
+    #[label(attr_must_not_be_zero)]
     MustNotBeZero {
         #[primary_span]
         span: Span,
     },
 
-    #[label(empty)]
+    #[label(attr_empty)]
     Empty {
         #[primary_span]
         span: Span,
     },
 
-    #[label(invalid_digit)]
+    #[label(attr_invalid_digit)]
     InvalidDigit {
         #[primary_span]
         span: Span,
     },
 
-    #[label(pos_overflow)]
+    #[label(attr_pos_overflow)]
     PosOverflow {
         #[primary_span]
         span: Span,
     },
 
-    #[label(neg_overflow)]
+    #[label(attr_neg_overflow)]
     NegOverflow {
         #[primary_span]
         span: Span,
@@ -275,7 +276,7 @@ pub(crate) struct IncorrectReprFormatGeneric<'a> {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum IncorrectReprFormatGenericCause<'a> {
-    #[suggestion(suggestion, code = "{name}({int})", applicability = "machine-applicable")]
+    #[suggestion(attr_suggestion, code = "{name}({int})", applicability = "machine-applicable")]
     Int {
         #[primary_span]
         span: Span,
@@ -287,7 +288,7 @@ pub(crate) enum IncorrectReprFormatGenericCause<'a> {
         int: u128,
     },
 
-    #[suggestion(suggestion, code = "{name}({symbol})", applicability = "machine-applicable")]
+    #[suggestion(attr_suggestion, code = "{name}({symbol})", applicability = "machine-applicable")]
     Symbol {
         #[primary_span]
         span: Span,
diff --git a/compiler/rustc_baked_icu_data/Cargo.toml b/compiler/rustc_baked_icu_data/Cargo.toml
index 3477306db..184fea868 100644
--- a/compiler/rustc_baked_icu_data/Cargo.toml
+++ b/compiler/rustc_baked_icu_data/Cargo.toml
@@ -4,12 +4,11 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
-icu_list = "1.0.0"
-icu_locid = "1.0.0"
-icu_provider = "1.0.1"
-icu_provider_adapters = "1.0.0"
-litemap = "0.6.0"
-zerovec = "0.9.0"
+icu_list = "1.1.0"
+icu_locid = "1.1.0"
+icu_provider = "1.1.0"
+icu_provider_adapters = "1.1.0"
+zerovec = "0.9.2"
 
 [features]
 rustc_use_parallel_compiler = ['icu_provider/sync']
diff --git a/compiler/rustc_baked_icu_data/src/data/any.rs b/compiler/rustc_baked_icu_data/src/data/any.rs
index e8e99be93..230288766 100644
--- a/compiler/rustc_baked_icu_data/src/data/any.rs
+++ b/compiler/rustc_baked_icu_data/src/data/any.rs
@@ -1,42 +1,2 @@
 // @generated
-impl AnyProvider for BakedDataProvider {
-    fn load_any(&self, key: DataKey, req: DataRequest) -> Result<AnyResponse, DataError> {
-        const ANDLISTV1MARKER: ::icu_provider::DataKeyHash =
-            ::icu_list::provider::AndListV1Marker::KEY.hashed();
-        const COLLATIONFALLBACKSUPPLEMENTV1MARKER: ::icu_provider::DataKeyHash =
-            ::icu_provider_adapters::fallback::provider::CollationFallbackSupplementV1Marker::KEY
-                .hashed();
-        const LOCALEFALLBACKLIKELYSUBTAGSV1MARKER: ::icu_provider::DataKeyHash =
-            ::icu_provider_adapters::fallback::provider::LocaleFallbackLikelySubtagsV1Marker::KEY
-                .hashed();
-        const LOCALEFALLBACKPARENTSV1MARKER: ::icu_provider::DataKeyHash =
-            ::icu_provider_adapters::fallback::provider::LocaleFallbackParentsV1Marker::KEY
-                .hashed();
-        #[allow(clippy::match_single_binding)]
-        match key.hashed() {
-            ANDLISTV1MARKER => list::and_v1::DATA
-                .get_by(|k| req.locale.strict_cmp(k.as_bytes()).reverse())
-                .copied()
-                .map(AnyPayload::from_static_ref)
-                .ok_or(DataErrorKind::MissingLocale),
-            COLLATIONFALLBACKSUPPLEMENTV1MARKER => fallback::supplement::co_v1::DATA
-                .get_by(|k| req.locale.strict_cmp(k.as_bytes()).reverse())
-                .copied()
-                .map(AnyPayload::from_static_ref)
-                .ok_or(DataErrorKind::MissingLocale),
-            LOCALEFALLBACKLIKELYSUBTAGSV1MARKER => fallback::likelysubtags_v1::DATA
-                .get_by(|k| req.locale.strict_cmp(k.as_bytes()).reverse())
-                .copied()
-                .map(AnyPayload::from_static_ref)
-                .ok_or(DataErrorKind::MissingLocale),
-            LOCALEFALLBACKPARENTSV1MARKER => fallback::parents_v1::DATA
-                .get_by(|k| req.locale.strict_cmp(k.as_bytes()).reverse())
-                .copied()
-                .map(AnyPayload::from_static_ref)
-                .ok_or(DataErrorKind::MissingLocale),
-            _ => Err(DataErrorKind::MissingDataKey),
-        }
-        .map_err(|e| e.with_req(key, req))
-        .map(|payload| AnyResponse { payload: Some(payload), metadata: Default::default() })
-    }
-}
+impl_any_provider!(BakedDataProvider);
diff --git a/compiler/rustc_baked_icu_data/src/data/fallback/likelysubtags_v1.rs b/compiler/rustc_baked_icu_data/src/data/fallback/likelysubtags_v1.rs
deleted file mode 100644
index 0a90c832e..000000000
--- a/compiler/rustc_baked_icu_data/src/data/fallback/likelysubtags_v1.rs
+++ /dev/null
@@ -1,733 +0,0 @@
-// @generated
-type DataStruct = < :: icu_provider_adapters :: fallback :: provider :: LocaleFallbackLikelySubtagsV1Marker as :: icu_provider :: DataMarker > :: Yokeable ;
-pub static DATA: litemap::LiteMap<&str, &DataStruct, &[(&str, &DataStruct)]> =
-    litemap::LiteMap::from_sorted_store_unchecked(&[("und", UND)]);
-static UND: &DataStruct =
-    &::icu_provider_adapters::fallback::provider::LocaleFallbackLikelySubtagsV1 {
-        l2s: unsafe {
-            #[allow(unused_unsafe)]
-            ::zerovec::ZeroMap::from_parts_unchecked(
-                unsafe {
-                    ::zerovec::ZeroVec::from_bytes_unchecked(&[
-                        97u8, 98u8, 0u8, 97u8, 98u8, 113u8, 97u8, 100u8, 112u8, 97u8, 100u8, 121u8,
-                        97u8, 101u8, 0u8, 97u8, 101u8, 98u8, 97u8, 104u8, 111u8, 97u8, 106u8,
-                        116u8, 97u8, 107u8, 107u8, 97u8, 108u8, 116u8, 97u8, 109u8, 0u8, 97u8,
-                        112u8, 99u8, 97u8, 112u8, 100u8, 97u8, 114u8, 0u8, 97u8, 114u8, 99u8, 97u8,
-                        114u8, 113u8, 97u8, 114u8, 115u8, 97u8, 114u8, 121u8, 97u8, 114u8, 122u8,
-                        97u8, 115u8, 0u8, 97u8, 115u8, 101u8, 97u8, 118u8, 0u8, 97u8, 118u8, 108u8,
-                        97u8, 119u8, 97u8, 98u8, 97u8, 0u8, 98u8, 97u8, 108u8, 98u8, 97u8, 112u8,
-                        98u8, 97u8, 120u8, 98u8, 99u8, 113u8, 98u8, 101u8, 0u8, 98u8, 101u8, 106u8,
-                        98u8, 102u8, 113u8, 98u8, 102u8, 116u8, 98u8, 102u8, 121u8, 98u8, 103u8,
-                        0u8, 98u8, 103u8, 99u8, 98u8, 103u8, 110u8, 98u8, 103u8, 120u8, 98u8,
-                        104u8, 98u8, 98u8, 104u8, 105u8, 98u8, 104u8, 111u8, 98u8, 106u8, 105u8,
-                        98u8, 106u8, 106u8, 98u8, 108u8, 116u8, 98u8, 110u8, 0u8, 98u8, 111u8, 0u8,
-                        98u8, 112u8, 121u8, 98u8, 113u8, 105u8, 98u8, 114u8, 97u8, 98u8, 114u8,
-                        104u8, 98u8, 114u8, 120u8, 98u8, 115u8, 113u8, 98u8, 115u8, 116u8, 98u8,
-                        116u8, 118u8, 98u8, 117u8, 97u8, 98u8, 121u8, 110u8, 99u8, 99u8, 112u8,
-                        99u8, 101u8, 0u8, 99u8, 104u8, 109u8, 99u8, 104u8, 114u8, 99u8, 106u8,
-                        97u8, 99u8, 106u8, 109u8, 99u8, 107u8, 98u8, 99u8, 109u8, 103u8, 99u8,
-                        111u8, 112u8, 99u8, 114u8, 0u8, 99u8, 114u8, 104u8, 99u8, 114u8, 107u8,
-                        99u8, 114u8, 108u8, 99u8, 115u8, 119u8, 99u8, 116u8, 100u8, 99u8, 117u8,
-                        0u8, 99u8, 118u8, 0u8, 100u8, 97u8, 114u8, 100u8, 99u8, 99u8, 100u8, 103u8,
-                        108u8, 100u8, 109u8, 102u8, 100u8, 111u8, 105u8, 100u8, 114u8, 104u8,
-                        100u8, 114u8, 115u8, 100u8, 116u8, 121u8, 100u8, 118u8, 0u8, 100u8, 122u8,
-                        0u8, 101u8, 103u8, 121u8, 101u8, 107u8, 121u8, 101u8, 108u8, 0u8, 101u8,
-                        115u8, 103u8, 101u8, 116u8, 116u8, 102u8, 97u8, 0u8, 102u8, 105u8, 97u8,
-                        102u8, 117u8, 98u8, 103u8, 97u8, 110u8, 103u8, 98u8, 109u8, 103u8, 98u8,
-                        122u8, 103u8, 101u8, 122u8, 103u8, 103u8, 110u8, 103u8, 106u8, 107u8,
-                        103u8, 106u8, 117u8, 103u8, 108u8, 107u8, 103u8, 109u8, 118u8, 103u8,
-                        111u8, 102u8, 103u8, 111u8, 109u8, 103u8, 111u8, 110u8, 103u8, 111u8,
-                        116u8, 103u8, 114u8, 99u8, 103u8, 114u8, 116u8, 103u8, 117u8, 0u8, 103u8,
-                        118u8, 114u8, 103u8, 119u8, 99u8, 103u8, 119u8, 116u8, 104u8, 97u8, 107u8,
-                        104u8, 97u8, 122u8, 104u8, 100u8, 121u8, 104u8, 101u8, 0u8, 104u8, 105u8,
-                        0u8, 104u8, 108u8, 117u8, 104u8, 109u8, 100u8, 104u8, 110u8, 100u8, 104u8,
-                        110u8, 101u8, 104u8, 110u8, 106u8, 104u8, 110u8, 111u8, 104u8, 111u8, 99u8,
-                        104u8, 111u8, 106u8, 104u8, 115u8, 110u8, 104u8, 121u8, 0u8, 105u8, 105u8,
-                        0u8, 105u8, 110u8, 104u8, 105u8, 117u8, 0u8, 105u8, 119u8, 0u8, 106u8,
-                        97u8, 0u8, 106u8, 105u8, 0u8, 106u8, 109u8, 108u8, 107u8, 97u8, 0u8, 107u8,
-                        97u8, 97u8, 107u8, 97u8, 119u8, 107u8, 98u8, 100u8, 107u8, 98u8, 121u8,
-                        107u8, 100u8, 116u8, 107u8, 102u8, 114u8, 107u8, 102u8, 121u8, 107u8,
-                        104u8, 98u8, 107u8, 104u8, 110u8, 107u8, 104u8, 116u8, 107u8, 104u8, 119u8,
-                        107u8, 106u8, 103u8, 107u8, 107u8, 0u8, 107u8, 109u8, 0u8, 107u8, 110u8,
-                        0u8, 107u8, 111u8, 0u8, 107u8, 111u8, 105u8, 107u8, 111u8, 107u8, 107u8,
-                        113u8, 121u8, 107u8, 114u8, 99u8, 107u8, 114u8, 117u8, 107u8, 115u8, 0u8,
-                        107u8, 116u8, 98u8, 107u8, 117u8, 109u8, 107u8, 118u8, 0u8, 107u8, 118u8,
-                        120u8, 107u8, 120u8, 99u8, 107u8, 120u8, 108u8, 107u8, 120u8, 109u8, 107u8,
-                        120u8, 112u8, 107u8, 121u8, 0u8, 107u8, 122u8, 104u8, 108u8, 97u8, 98u8,
-                        108u8, 97u8, 100u8, 108u8, 97u8, 104u8, 108u8, 98u8, 101u8, 108u8, 99u8,
-                        112u8, 108u8, 101u8, 112u8, 108u8, 101u8, 122u8, 108u8, 105u8, 102u8,
-                        108u8, 105u8, 115u8, 108u8, 107u8, 105u8, 108u8, 109u8, 110u8, 108u8,
-                        111u8, 0u8, 108u8, 114u8, 99u8, 108u8, 117u8, 122u8, 108u8, 119u8, 108u8,
-                        108u8, 122u8, 104u8, 109u8, 97u8, 103u8, 109u8, 97u8, 105u8, 109u8, 100u8,
-                        101u8, 109u8, 100u8, 102u8, 109u8, 100u8, 120u8, 109u8, 102u8, 97u8, 109u8,
-                        103u8, 112u8, 109u8, 107u8, 0u8, 109u8, 107u8, 105u8, 109u8, 108u8, 0u8,
-                        109u8, 110u8, 0u8, 109u8, 110u8, 105u8, 109u8, 110u8, 119u8, 109u8, 114u8,
-                        0u8, 109u8, 114u8, 100u8, 109u8, 114u8, 106u8, 109u8, 114u8, 111u8, 109u8,
-                        116u8, 114u8, 109u8, 118u8, 121u8, 109u8, 119u8, 114u8, 109u8, 119u8,
-                        119u8, 109u8, 121u8, 0u8, 109u8, 121u8, 109u8, 109u8, 121u8, 118u8, 109u8,
-                        121u8, 122u8, 109u8, 122u8, 110u8, 110u8, 97u8, 110u8, 110u8, 101u8, 0u8,
-                        110u8, 101u8, 119u8, 110u8, 110u8, 112u8, 110u8, 111u8, 100u8, 110u8,
-                        111u8, 101u8, 110u8, 111u8, 110u8, 110u8, 113u8, 111u8, 110u8, 115u8,
-                        107u8, 110u8, 115u8, 116u8, 111u8, 106u8, 0u8, 111u8, 106u8, 115u8, 111u8,
-                        114u8, 0u8, 111u8, 114u8, 117u8, 111u8, 115u8, 0u8, 111u8, 115u8, 97u8,
-                        111u8, 116u8, 97u8, 111u8, 116u8, 107u8, 111u8, 117u8, 105u8, 112u8, 97u8,
-                        0u8, 112u8, 97u8, 108u8, 112u8, 101u8, 111u8, 112u8, 104u8, 108u8, 112u8,
-                        104u8, 110u8, 112u8, 107u8, 97u8, 112u8, 110u8, 116u8, 112u8, 112u8, 97u8,
-                        112u8, 114u8, 97u8, 112u8, 114u8, 100u8, 112u8, 115u8, 0u8, 114u8, 97u8,
-                        106u8, 114u8, 104u8, 103u8, 114u8, 105u8, 102u8, 114u8, 106u8, 115u8,
-                        114u8, 107u8, 116u8, 114u8, 109u8, 116u8, 114u8, 117u8, 0u8, 114u8, 117u8,
-                        101u8, 114u8, 121u8, 117u8, 115u8, 97u8, 0u8, 115u8, 97u8, 104u8, 115u8,
-                        97u8, 116u8, 115u8, 97u8, 122u8, 115u8, 99u8, 107u8, 115u8, 99u8, 108u8,
-                        115u8, 100u8, 0u8, 115u8, 100u8, 104u8, 115u8, 103u8, 97u8, 115u8, 103u8,
-                        119u8, 115u8, 104u8, 105u8, 115u8, 104u8, 110u8, 115u8, 104u8, 117u8,
-                        115u8, 105u8, 0u8, 115u8, 107u8, 114u8, 115u8, 109u8, 112u8, 115u8, 111u8,
-                        103u8, 115u8, 111u8, 117u8, 115u8, 114u8, 0u8, 115u8, 114u8, 98u8, 115u8,
-                        114u8, 120u8, 115u8, 119u8, 98u8, 115u8, 119u8, 118u8, 115u8, 121u8, 108u8,
-                        115u8, 121u8, 114u8, 116u8, 97u8, 0u8, 116u8, 97u8, 106u8, 116u8, 99u8,
-                        121u8, 116u8, 100u8, 100u8, 116u8, 100u8, 103u8, 116u8, 100u8, 104u8,
-                        116u8, 101u8, 0u8, 116u8, 103u8, 0u8, 116u8, 104u8, 0u8, 116u8, 104u8,
-                        108u8, 116u8, 104u8, 113u8, 116u8, 104u8, 114u8, 116u8, 105u8, 0u8, 116u8,
-                        105u8, 103u8, 116u8, 107u8, 116u8, 116u8, 114u8, 119u8, 116u8, 115u8,
-                        100u8, 116u8, 115u8, 102u8, 116u8, 115u8, 106u8, 116u8, 116u8, 0u8, 116u8,
-                        116u8, 115u8, 116u8, 120u8, 103u8, 116u8, 120u8, 111u8, 116u8, 121u8,
-                        118u8, 117u8, 100u8, 105u8, 117u8, 100u8, 109u8, 117u8, 103u8, 0u8, 117u8,
-                        103u8, 97u8, 117u8, 107u8, 0u8, 117u8, 110u8, 114u8, 117u8, 110u8, 120u8,
-                        117u8, 114u8, 0u8, 118u8, 97u8, 105u8, 119u8, 97u8, 108u8, 119u8, 98u8,
-                        113u8, 119u8, 98u8, 114u8, 119u8, 110u8, 105u8, 119u8, 115u8, 103u8, 119u8,
-                        116u8, 109u8, 119u8, 117u8, 117u8, 120u8, 99u8, 111u8, 120u8, 99u8, 114u8,
-                        120u8, 108u8, 99u8, 120u8, 108u8, 100u8, 120u8, 109u8, 102u8, 120u8, 109u8,
-                        110u8, 120u8, 109u8, 114u8, 120u8, 110u8, 97u8, 120u8, 110u8, 114u8, 120u8,
-                        112u8, 114u8, 120u8, 115u8, 97u8, 120u8, 115u8, 114u8, 121u8, 105u8, 0u8,
-                        121u8, 117u8, 101u8, 122u8, 100u8, 106u8, 122u8, 103u8, 104u8, 122u8,
-                        104u8, 0u8, 122u8, 104u8, 120u8, 122u8, 107u8, 116u8,
-                    ])
-                },
-                unsafe {
-                    ::zerovec::ZeroVec::from_bytes_unchecked(&[
-                        67u8, 121u8, 114u8, 108u8, 67u8, 121u8, 114u8, 108u8, 84u8, 105u8, 98u8,
-                        116u8, 67u8, 121u8, 114u8, 108u8, 65u8, 118u8, 115u8, 116u8, 65u8, 114u8,
-                        97u8, 98u8, 65u8, 104u8, 111u8, 109u8, 65u8, 114u8, 97u8, 98u8, 88u8,
-                        115u8, 117u8, 120u8, 67u8, 121u8, 114u8, 108u8, 69u8, 116u8, 104u8, 105u8,
-                        65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8,
-                        65u8, 114u8, 109u8, 105u8, 65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8,
-                        98u8, 65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 66u8, 101u8, 110u8,
-                        103u8, 83u8, 103u8, 110u8, 119u8, 67u8, 121u8, 114u8, 108u8, 65u8, 114u8,
-                        97u8, 98u8, 68u8, 101u8, 118u8, 97u8, 67u8, 121u8, 114u8, 108u8, 65u8,
-                        114u8, 97u8, 98u8, 68u8, 101u8, 118u8, 97u8, 66u8, 97u8, 109u8, 117u8,
-                        69u8, 116u8, 104u8, 105u8, 67u8, 121u8, 114u8, 108u8, 65u8, 114u8, 97u8,
-                        98u8, 84u8, 97u8, 109u8, 108u8, 65u8, 114u8, 97u8, 98u8, 68u8, 101u8,
-                        118u8, 97u8, 67u8, 121u8, 114u8, 108u8, 68u8, 101u8, 118u8, 97u8, 65u8,
-                        114u8, 97u8, 98u8, 71u8, 114u8, 101u8, 107u8, 68u8, 101u8, 118u8, 97u8,
-                        68u8, 101u8, 118u8, 97u8, 68u8, 101u8, 118u8, 97u8, 69u8, 116u8, 104u8,
-                        105u8, 68u8, 101u8, 118u8, 97u8, 84u8, 97u8, 118u8, 116u8, 66u8, 101u8,
-                        110u8, 103u8, 84u8, 105u8, 98u8, 116u8, 66u8, 101u8, 110u8, 103u8, 65u8,
-                        114u8, 97u8, 98u8, 68u8, 101u8, 118u8, 97u8, 65u8, 114u8, 97u8, 98u8, 68u8,
-                        101u8, 118u8, 97u8, 66u8, 97u8, 115u8, 115u8, 69u8, 116u8, 104u8, 105u8,
-                        68u8, 101u8, 118u8, 97u8, 67u8, 121u8, 114u8, 108u8, 69u8, 116u8, 104u8,
-                        105u8, 67u8, 97u8, 107u8, 109u8, 67u8, 121u8, 114u8, 108u8, 67u8, 121u8,
-                        114u8, 108u8, 67u8, 104u8, 101u8, 114u8, 65u8, 114u8, 97u8, 98u8, 67u8,
-                        104u8, 97u8, 109u8, 65u8, 114u8, 97u8, 98u8, 83u8, 111u8, 121u8, 111u8,
-                        67u8, 111u8, 112u8, 116u8, 67u8, 97u8, 110u8, 115u8, 67u8, 121u8, 114u8,
-                        108u8, 67u8, 97u8, 110u8, 115u8, 67u8, 97u8, 110u8, 115u8, 67u8, 97u8,
-                        110u8, 115u8, 80u8, 97u8, 117u8, 99u8, 67u8, 121u8, 114u8, 108u8, 67u8,
-                        121u8, 114u8, 108u8, 67u8, 121u8, 114u8, 108u8, 65u8, 114u8, 97u8, 98u8,
-                        65u8, 114u8, 97u8, 98u8, 77u8, 101u8, 100u8, 102u8, 68u8, 101u8, 118u8,
-                        97u8, 77u8, 111u8, 110u8, 103u8, 69u8, 116u8, 104u8, 105u8, 68u8, 101u8,
-                        118u8, 97u8, 84u8, 104u8, 97u8, 97u8, 84u8, 105u8, 98u8, 116u8, 69u8,
-                        103u8, 121u8, 112u8, 75u8, 97u8, 108u8, 105u8, 71u8, 114u8, 101u8, 107u8,
-                        71u8, 111u8, 110u8, 109u8, 73u8, 116u8, 97u8, 108u8, 65u8, 114u8, 97u8,
-                        98u8, 65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 72u8, 97u8, 110u8,
-                        115u8, 68u8, 101u8, 118u8, 97u8, 65u8, 114u8, 97u8, 98u8, 69u8, 116u8,
-                        104u8, 105u8, 68u8, 101u8, 118u8, 97u8, 65u8, 114u8, 97u8, 98u8, 65u8,
-                        114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 69u8, 116u8, 104u8, 105u8,
-                        69u8, 116u8, 104u8, 105u8, 68u8, 101u8, 118u8, 97u8, 84u8, 101u8, 108u8,
-                        117u8, 71u8, 111u8, 116u8, 104u8, 67u8, 112u8, 114u8, 116u8, 66u8, 101u8,
-                        110u8, 103u8, 71u8, 117u8, 106u8, 114u8, 68u8, 101u8, 118u8, 97u8, 65u8,
-                        114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 72u8, 97u8, 110u8, 115u8, 65u8,
-                        114u8, 97u8, 98u8, 69u8, 116u8, 104u8, 105u8, 72u8, 101u8, 98u8, 114u8,
-                        68u8, 101u8, 118u8, 97u8, 72u8, 108u8, 117u8, 119u8, 80u8, 108u8, 114u8,
-                        100u8, 65u8, 114u8, 97u8, 98u8, 68u8, 101u8, 118u8, 97u8, 72u8, 109u8,
-                        110u8, 112u8, 65u8, 114u8, 97u8, 98u8, 68u8, 101u8, 118u8, 97u8, 68u8,
-                        101u8, 118u8, 97u8, 72u8, 97u8, 110u8, 115u8, 65u8, 114u8, 109u8, 110u8,
-                        89u8, 105u8, 105u8, 105u8, 67u8, 121u8, 114u8, 108u8, 67u8, 97u8, 110u8,
-                        115u8, 72u8, 101u8, 98u8, 114u8, 74u8, 112u8, 97u8, 110u8, 72u8, 101u8,
-                        98u8, 114u8, 68u8, 101u8, 118u8, 97u8, 71u8, 101u8, 111u8, 114u8, 67u8,
-                        121u8, 114u8, 108u8, 75u8, 97u8, 119u8, 105u8, 67u8, 121u8, 114u8, 108u8,
-                        65u8, 114u8, 97u8, 98u8, 84u8, 104u8, 97u8, 105u8, 68u8, 101u8, 118u8,
-                        97u8, 68u8, 101u8, 118u8, 97u8, 84u8, 97u8, 108u8, 117u8, 68u8, 101u8,
-                        118u8, 97u8, 77u8, 121u8, 109u8, 114u8, 65u8, 114u8, 97u8, 98u8, 76u8,
-                        97u8, 111u8, 111u8, 67u8, 121u8, 114u8, 108u8, 75u8, 104u8, 109u8, 114u8,
-                        75u8, 110u8, 100u8, 97u8, 75u8, 111u8, 114u8, 101u8, 67u8, 121u8, 114u8,
-                        108u8, 68u8, 101u8, 118u8, 97u8, 69u8, 116u8, 104u8, 105u8, 67u8, 121u8,
-                        114u8, 108u8, 68u8, 101u8, 118u8, 97u8, 65u8, 114u8, 97u8, 98u8, 69u8,
-                        116u8, 104u8, 105u8, 67u8, 121u8, 114u8, 108u8, 67u8, 121u8, 114u8, 108u8,
-                        65u8, 114u8, 97u8, 98u8, 69u8, 116u8, 104u8, 105u8, 68u8, 101u8, 118u8,
-                        97u8, 84u8, 104u8, 97u8, 105u8, 65u8, 114u8, 97u8, 98u8, 67u8, 121u8,
-                        114u8, 108u8, 65u8, 114u8, 97u8, 98u8, 76u8, 105u8, 110u8, 97u8, 72u8,
-                        101u8, 98u8, 114u8, 65u8, 114u8, 97u8, 98u8, 67u8, 121u8, 114u8, 108u8,
-                        84u8, 104u8, 97u8, 105u8, 76u8, 101u8, 112u8, 99u8, 67u8, 121u8, 114u8,
-                        108u8, 68u8, 101u8, 118u8, 97u8, 76u8, 105u8, 115u8, 117u8, 65u8, 114u8,
-                        97u8, 98u8, 84u8, 101u8, 108u8, 117u8, 76u8, 97u8, 111u8, 111u8, 65u8,
-                        114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 84u8, 104u8, 97u8, 105u8, 72u8,
-                        97u8, 110u8, 115u8, 68u8, 101u8, 118u8, 97u8, 68u8, 101u8, 118u8, 97u8,
-                        65u8, 114u8, 97u8, 98u8, 67u8, 121u8, 114u8, 108u8, 69u8, 116u8, 104u8,
-                        105u8, 65u8, 114u8, 97u8, 98u8, 68u8, 101u8, 118u8, 97u8, 67u8, 121u8,
-                        114u8, 108u8, 65u8, 114u8, 97u8, 98u8, 77u8, 108u8, 121u8, 109u8, 67u8,
-                        121u8, 114u8, 108u8, 66u8, 101u8, 110u8, 103u8, 77u8, 121u8, 109u8, 114u8,
-                        68u8, 101u8, 118u8, 97u8, 68u8, 101u8, 118u8, 97u8, 67u8, 121u8, 114u8,
-                        108u8, 77u8, 114u8, 111u8, 111u8, 68u8, 101u8, 118u8, 97u8, 65u8, 114u8,
-                        97u8, 98u8, 68u8, 101u8, 118u8, 97u8, 72u8, 109u8, 110u8, 112u8, 77u8,
-                        121u8, 109u8, 114u8, 69u8, 116u8, 104u8, 105u8, 67u8, 121u8, 114u8, 108u8,
-                        77u8, 97u8, 110u8, 100u8, 65u8, 114u8, 97u8, 98u8, 72u8, 97u8, 110u8,
-                        115u8, 68u8, 101u8, 118u8, 97u8, 68u8, 101u8, 118u8, 97u8, 87u8, 99u8,
-                        104u8, 111u8, 76u8, 97u8, 110u8, 97u8, 68u8, 101u8, 118u8, 97u8, 82u8,
-                        117u8, 110u8, 114u8, 78u8, 107u8, 111u8, 111u8, 67u8, 97u8, 110u8, 115u8,
-                        84u8, 110u8, 115u8, 97u8, 67u8, 97u8, 110u8, 115u8, 67u8, 97u8, 110u8,
-                        115u8, 79u8, 114u8, 121u8, 97u8, 65u8, 114u8, 97u8, 98u8, 67u8, 121u8,
-                        114u8, 108u8, 79u8, 115u8, 103u8, 101u8, 65u8, 114u8, 97u8, 98u8, 79u8,
-                        114u8, 107u8, 104u8, 79u8, 117u8, 103u8, 114u8, 71u8, 117u8, 114u8, 117u8,
-                        80u8, 104u8, 108u8, 105u8, 88u8, 112u8, 101u8, 111u8, 65u8, 114u8, 97u8,
-                        98u8, 80u8, 104u8, 110u8, 120u8, 66u8, 114u8, 97u8, 104u8, 71u8, 114u8,
-                        101u8, 107u8, 68u8, 101u8, 118u8, 97u8, 75u8, 104u8, 97u8, 114u8, 65u8,
-                        114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 68u8, 101u8, 118u8, 97u8, 82u8,
-                        111u8, 104u8, 103u8, 84u8, 102u8, 110u8, 103u8, 68u8, 101u8, 118u8, 97u8,
-                        66u8, 101u8, 110u8, 103u8, 65u8, 114u8, 97u8, 98u8, 67u8, 121u8, 114u8,
-                        108u8, 67u8, 121u8, 114u8, 108u8, 75u8, 97u8, 110u8, 97u8, 68u8, 101u8,
-                        118u8, 97u8, 67u8, 121u8, 114u8, 108u8, 79u8, 108u8, 99u8, 107u8, 83u8,
-                        97u8, 117u8, 114u8, 68u8, 101u8, 118u8, 97u8, 65u8, 114u8, 97u8, 98u8,
-                        65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 79u8, 103u8, 97u8, 109u8,
-                        69u8, 116u8, 104u8, 105u8, 84u8, 102u8, 110u8, 103u8, 77u8, 121u8, 109u8,
-                        114u8, 65u8, 114u8, 97u8, 98u8, 83u8, 105u8, 110u8, 104u8, 65u8, 114u8,
-                        97u8, 98u8, 83u8, 97u8, 109u8, 114u8, 83u8, 111u8, 103u8, 100u8, 84u8,
-                        104u8, 97u8, 105u8, 67u8, 121u8, 114u8, 108u8, 83u8, 111u8, 114u8, 97u8,
-                        68u8, 101u8, 118u8, 97u8, 65u8, 114u8, 97u8, 98u8, 68u8, 101u8, 118u8,
-                        97u8, 66u8, 101u8, 110u8, 103u8, 83u8, 121u8, 114u8, 99u8, 84u8, 97u8,
-                        109u8, 108u8, 68u8, 101u8, 118u8, 97u8, 75u8, 110u8, 100u8, 97u8, 84u8,
-                        97u8, 108u8, 101u8, 68u8, 101u8, 118u8, 97u8, 68u8, 101u8, 118u8, 97u8,
-                        84u8, 101u8, 108u8, 117u8, 67u8, 121u8, 114u8, 108u8, 84u8, 104u8, 97u8,
-                        105u8, 68u8, 101u8, 118u8, 97u8, 68u8, 101u8, 118u8, 97u8, 68u8, 101u8,
-                        118u8, 97u8, 69u8, 116u8, 104u8, 105u8, 69u8, 116u8, 104u8, 105u8, 68u8,
-                        101u8, 118u8, 97u8, 65u8, 114u8, 97u8, 98u8, 71u8, 114u8, 101u8, 107u8,
-                        68u8, 101u8, 118u8, 97u8, 84u8, 105u8, 98u8, 116u8, 67u8, 121u8, 114u8,
-                        108u8, 84u8, 104u8, 97u8, 105u8, 84u8, 97u8, 110u8, 103u8, 84u8, 111u8,
-                        116u8, 111u8, 67u8, 121u8, 114u8, 108u8, 65u8, 103u8, 104u8, 98u8, 67u8,
-                        121u8, 114u8, 108u8, 65u8, 114u8, 97u8, 98u8, 85u8, 103u8, 97u8, 114u8,
-                        67u8, 121u8, 114u8, 108u8, 66u8, 101u8, 110u8, 103u8, 66u8, 101u8, 110u8,
-                        103u8, 65u8, 114u8, 97u8, 98u8, 86u8, 97u8, 105u8, 105u8, 69u8, 116u8,
-                        104u8, 105u8, 84u8, 101u8, 108u8, 117u8, 68u8, 101u8, 118u8, 97u8, 65u8,
-                        114u8, 97u8, 98u8, 71u8, 111u8, 110u8, 103u8, 68u8, 101u8, 118u8, 97u8,
-                        72u8, 97u8, 110u8, 115u8, 67u8, 104u8, 114u8, 115u8, 67u8, 97u8, 114u8,
-                        105u8, 76u8, 121u8, 99u8, 105u8, 76u8, 121u8, 100u8, 105u8, 71u8, 101u8,
-                        111u8, 114u8, 77u8, 97u8, 110u8, 105u8, 77u8, 101u8, 114u8, 99u8, 78u8,
-                        97u8, 114u8, 98u8, 68u8, 101u8, 118u8, 97u8, 80u8, 114u8, 116u8, 105u8,
-                        83u8, 97u8, 114u8, 98u8, 68u8, 101u8, 118u8, 97u8, 72u8, 101u8, 98u8,
-                        114u8, 72u8, 97u8, 110u8, 116u8, 65u8, 114u8, 97u8, 98u8, 84u8, 102u8,
-                        110u8, 103u8, 72u8, 97u8, 110u8, 115u8, 78u8, 115u8, 104u8, 117u8, 75u8,
-                        105u8, 116u8, 115u8,
-                    ])
-                },
-            )
-        },
-        lr2s: unsafe {
-            #[allow(unused_unsafe)]
-            ::zerovec::ZeroMap2d::from_parts_unchecked(
-                unsafe {
-                    ::zerovec::ZeroVec::from_bytes_unchecked(&[
-                        97u8, 122u8, 0u8, 104u8, 97u8, 0u8, 107u8, 107u8, 0u8, 107u8, 117u8, 0u8,
-                        107u8, 121u8, 0u8, 109u8, 97u8, 110u8, 109u8, 110u8, 0u8, 109u8, 115u8,
-                        0u8, 112u8, 97u8, 0u8, 114u8, 105u8, 102u8, 115u8, 100u8, 0u8, 115u8,
-                        114u8, 0u8, 116u8, 103u8, 0u8, 117u8, 103u8, 0u8, 117u8, 110u8, 114u8,
-                        117u8, 122u8, 0u8, 121u8, 117u8, 101u8, 122u8, 104u8, 0u8,
-                    ])
-                },
-                unsafe {
-                    ::zerovec::ZeroVec::from_bytes_unchecked(&[
-                        3u8, 0u8, 0u8, 0u8, 5u8, 0u8, 0u8, 0u8, 9u8, 0u8, 0u8, 0u8, 10u8, 0u8, 0u8,
-                        0u8, 12u8, 0u8, 0u8, 0u8, 13u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 15u8,
-                        0u8, 0u8, 0u8, 16u8, 0u8, 0u8, 0u8, 17u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8,
-                        0u8, 22u8, 0u8, 0u8, 0u8, 23u8, 0u8, 0u8, 0u8, 25u8, 0u8, 0u8, 0u8, 26u8,
-                        0u8, 0u8, 0u8, 28u8, 0u8, 0u8, 0u8, 29u8, 0u8, 0u8, 0u8, 44u8, 0u8, 0u8,
-                        0u8,
-                    ])
-                },
-                unsafe {
-                    ::zerovec::ZeroVec::from_bytes_unchecked(&[
-                        73u8, 81u8, 0u8, 73u8, 82u8, 0u8, 82u8, 85u8, 0u8, 67u8, 77u8, 0u8, 83u8,
-                        68u8, 0u8, 65u8, 70u8, 0u8, 67u8, 78u8, 0u8, 73u8, 82u8, 0u8, 77u8, 78u8,
-                        0u8, 76u8, 66u8, 0u8, 67u8, 78u8, 0u8, 84u8, 82u8, 0u8, 71u8, 78u8, 0u8,
-                        67u8, 78u8, 0u8, 67u8, 67u8, 0u8, 80u8, 75u8, 0u8, 78u8, 76u8, 0u8, 73u8,
-                        78u8, 0u8, 77u8, 69u8, 0u8, 82u8, 79u8, 0u8, 82u8, 85u8, 0u8, 84u8, 82u8,
-                        0u8, 80u8, 75u8, 0u8, 75u8, 90u8, 0u8, 77u8, 78u8, 0u8, 78u8, 80u8, 0u8,
-                        65u8, 70u8, 0u8, 67u8, 78u8, 0u8, 67u8, 78u8, 0u8, 65u8, 85u8, 0u8, 66u8,
-                        78u8, 0u8, 71u8, 66u8, 0u8, 71u8, 70u8, 0u8, 72u8, 75u8, 0u8, 73u8, 68u8,
-                        0u8, 77u8, 79u8, 0u8, 80u8, 65u8, 0u8, 80u8, 70u8, 0u8, 80u8, 72u8, 0u8,
-                        83u8, 82u8, 0u8, 84u8, 72u8, 0u8, 84u8, 87u8, 0u8, 85u8, 83u8, 0u8, 86u8,
-                        78u8, 0u8,
-                    ])
-                },
-                unsafe {
-                    ::zerovec::ZeroVec::from_bytes_unchecked(&[
-                        65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 67u8, 121u8, 114u8,
-                        108u8, 65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8,
-                        98u8, 65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8,
-                        98u8, 65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 76u8, 97u8, 116u8,
-                        110u8, 78u8, 107u8, 111u8, 111u8, 77u8, 111u8, 110u8, 103u8, 65u8, 114u8,
-                        97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 76u8, 97u8, 116u8, 110u8, 68u8, 101u8,
-                        118u8, 97u8, 76u8, 97u8, 116u8, 110u8, 76u8, 97u8, 116u8, 110u8, 76u8,
-                        97u8, 116u8, 110u8, 76u8, 97u8, 116u8, 110u8, 65u8, 114u8, 97u8, 98u8,
-                        67u8, 121u8, 114u8, 108u8, 67u8, 121u8, 114u8, 108u8, 68u8, 101u8, 118u8,
-                        97u8, 65u8, 114u8, 97u8, 98u8, 67u8, 121u8, 114u8, 108u8, 72u8, 97u8,
-                        110u8, 115u8, 72u8, 97u8, 110u8, 116u8, 72u8, 97u8, 110u8, 116u8, 72u8,
-                        97u8, 110u8, 116u8, 72u8, 97u8, 110u8, 116u8, 72u8, 97u8, 110u8, 116u8,
-                        72u8, 97u8, 110u8, 116u8, 72u8, 97u8, 110u8, 116u8, 72u8, 97u8, 110u8,
-                        116u8, 72u8, 97u8, 110u8, 116u8, 72u8, 97u8, 110u8, 116u8, 72u8, 97u8,
-                        110u8, 116u8, 72u8, 97u8, 110u8, 116u8, 72u8, 97u8, 110u8, 116u8, 72u8,
-                        97u8, 110u8, 116u8, 72u8, 97u8, 110u8, 116u8,
-                    ])
-                },
-            )
-        },
-        l2r: unsafe {
-            #[allow(unused_unsafe)]
-            ::zerovec::ZeroMap::from_parts_unchecked(
-                unsafe {
-                    ::zerovec::ZeroVec::from_bytes_unchecked(&[
-                        97u8, 97u8, 0u8, 97u8, 98u8, 0u8, 97u8, 98u8, 114u8, 97u8, 99u8, 101u8,
-                        97u8, 99u8, 104u8, 97u8, 100u8, 97u8, 97u8, 100u8, 112u8, 97u8, 100u8,
-                        121u8, 97u8, 101u8, 0u8, 97u8, 101u8, 98u8, 97u8, 102u8, 0u8, 97u8, 103u8,
-                        113u8, 97u8, 104u8, 111u8, 97u8, 106u8, 116u8, 97u8, 107u8, 0u8, 97u8,
-                        107u8, 107u8, 97u8, 108u8, 110u8, 97u8, 108u8, 116u8, 97u8, 109u8, 0u8,
-                        97u8, 109u8, 111u8, 97u8, 110u8, 0u8, 97u8, 110u8, 110u8, 97u8, 111u8,
-                        122u8, 97u8, 112u8, 100u8, 97u8, 114u8, 0u8, 97u8, 114u8, 99u8, 97u8,
-                        114u8, 110u8, 97u8, 114u8, 111u8, 97u8, 114u8, 113u8, 97u8, 114u8, 115u8,
-                        97u8, 114u8, 121u8, 97u8, 114u8, 122u8, 97u8, 115u8, 0u8, 97u8, 115u8,
-                        97u8, 97u8, 115u8, 101u8, 97u8, 115u8, 116u8, 97u8, 116u8, 106u8, 97u8,
-                        118u8, 0u8, 97u8, 119u8, 97u8, 97u8, 121u8, 0u8, 97u8, 122u8, 0u8, 98u8,
-                        97u8, 0u8, 98u8, 97u8, 108u8, 98u8, 97u8, 110u8, 98u8, 97u8, 112u8, 98u8,
-                        97u8, 114u8, 98u8, 97u8, 115u8, 98u8, 97u8, 120u8, 98u8, 98u8, 99u8, 98u8,
-                        98u8, 106u8, 98u8, 99u8, 105u8, 98u8, 101u8, 0u8, 98u8, 101u8, 106u8, 98u8,
-                        101u8, 109u8, 98u8, 101u8, 119u8, 98u8, 101u8, 122u8, 98u8, 102u8, 100u8,
-                        98u8, 102u8, 113u8, 98u8, 102u8, 116u8, 98u8, 102u8, 121u8, 98u8, 103u8,
-                        0u8, 98u8, 103u8, 99u8, 98u8, 103u8, 110u8, 98u8, 103u8, 120u8, 98u8,
-                        104u8, 98u8, 98u8, 104u8, 105u8, 98u8, 104u8, 111u8, 98u8, 105u8, 0u8,
-                        98u8, 105u8, 107u8, 98u8, 105u8, 110u8, 98u8, 106u8, 106u8, 98u8, 106u8,
-                        110u8, 98u8, 106u8, 116u8, 98u8, 107u8, 109u8, 98u8, 107u8, 117u8, 98u8,
-                        108u8, 97u8, 98u8, 108u8, 103u8, 98u8, 108u8, 116u8, 98u8, 109u8, 0u8,
-                        98u8, 109u8, 113u8, 98u8, 110u8, 0u8, 98u8, 111u8, 0u8, 98u8, 112u8, 121u8,
-                        98u8, 113u8, 105u8, 98u8, 113u8, 118u8, 98u8, 114u8, 0u8, 98u8, 114u8,
-                        97u8, 98u8, 114u8, 104u8, 98u8, 114u8, 120u8, 98u8, 115u8, 0u8, 98u8,
-                        115u8, 113u8, 98u8, 115u8, 115u8, 98u8, 116u8, 111u8, 98u8, 116u8, 118u8,
-                        98u8, 117u8, 97u8, 98u8, 117u8, 99u8, 98u8, 117u8, 103u8, 98u8, 117u8,
-                        109u8, 98u8, 118u8, 98u8, 98u8, 121u8, 110u8, 98u8, 121u8, 118u8, 98u8,
-                        122u8, 101u8, 99u8, 97u8, 0u8, 99u8, 97u8, 100u8, 99u8, 99u8, 104u8, 99u8,
-                        99u8, 112u8, 99u8, 101u8, 0u8, 99u8, 101u8, 98u8, 99u8, 103u8, 103u8, 99u8,
-                        104u8, 0u8, 99u8, 104u8, 107u8, 99u8, 104u8, 109u8, 99u8, 104u8, 111u8,
-                        99u8, 104u8, 112u8, 99u8, 104u8, 114u8, 99u8, 105u8, 99u8, 99u8, 106u8,
-                        97u8, 99u8, 106u8, 109u8, 99u8, 107u8, 98u8, 99u8, 108u8, 99u8, 99u8,
-                        109u8, 103u8, 99u8, 111u8, 0u8, 99u8, 111u8, 112u8, 99u8, 112u8, 115u8,
-                        99u8, 114u8, 0u8, 99u8, 114u8, 103u8, 99u8, 114u8, 104u8, 99u8, 114u8,
-                        107u8, 99u8, 114u8, 108u8, 99u8, 114u8, 115u8, 99u8, 115u8, 0u8, 99u8,
-                        115u8, 98u8, 99u8, 115u8, 119u8, 99u8, 116u8, 100u8, 99u8, 117u8, 0u8,
-                        99u8, 118u8, 0u8, 99u8, 121u8, 0u8, 100u8, 97u8, 0u8, 100u8, 97u8, 102u8,
-                        100u8, 97u8, 107u8, 100u8, 97u8, 114u8, 100u8, 97u8, 118u8, 100u8, 99u8,
-                        99u8, 100u8, 101u8, 0u8, 100u8, 101u8, 110u8, 100u8, 103u8, 114u8, 100u8,
-                        106u8, 101u8, 100u8, 109u8, 102u8, 100u8, 110u8, 106u8, 100u8, 111u8,
-                        105u8, 100u8, 114u8, 104u8, 100u8, 115u8, 98u8, 100u8, 116u8, 109u8, 100u8,
-                        116u8, 112u8, 100u8, 116u8, 121u8, 100u8, 117u8, 97u8, 100u8, 118u8, 0u8,
-                        100u8, 121u8, 111u8, 100u8, 121u8, 117u8, 100u8, 122u8, 0u8, 101u8, 98u8,
-                        117u8, 101u8, 101u8, 0u8, 101u8, 102u8, 105u8, 101u8, 103u8, 108u8, 101u8,
-                        103u8, 121u8, 101u8, 107u8, 121u8, 101u8, 108u8, 0u8, 101u8, 110u8, 0u8,
-                        101u8, 111u8, 0u8, 101u8, 115u8, 0u8, 101u8, 115u8, 103u8, 101u8, 115u8,
-                        117u8, 101u8, 116u8, 0u8, 101u8, 116u8, 116u8, 101u8, 117u8, 0u8, 101u8,
-                        119u8, 111u8, 101u8, 120u8, 116u8, 102u8, 97u8, 0u8, 102u8, 97u8, 110u8,
-                        102u8, 102u8, 0u8, 102u8, 102u8, 109u8, 102u8, 105u8, 0u8, 102u8, 105u8,
-                        97u8, 102u8, 105u8, 108u8, 102u8, 105u8, 116u8, 102u8, 106u8, 0u8, 102u8,
-                        111u8, 0u8, 102u8, 111u8, 110u8, 102u8, 114u8, 0u8, 102u8, 114u8, 99u8,
-                        102u8, 114u8, 112u8, 102u8, 114u8, 114u8, 102u8, 114u8, 115u8, 102u8,
-                        117u8, 98u8, 102u8, 117u8, 100u8, 102u8, 117u8, 102u8, 102u8, 117u8, 113u8,
-                        102u8, 117u8, 114u8, 102u8, 117u8, 118u8, 102u8, 118u8, 114u8, 102u8,
-                        121u8, 0u8, 103u8, 97u8, 0u8, 103u8, 97u8, 97u8, 103u8, 97u8, 103u8, 103u8,
-                        97u8, 110u8, 103u8, 97u8, 121u8, 103u8, 98u8, 109u8, 103u8, 98u8, 122u8,
-                        103u8, 99u8, 114u8, 103u8, 100u8, 0u8, 103u8, 101u8, 122u8, 103u8, 103u8,
-                        110u8, 103u8, 105u8, 108u8, 103u8, 106u8, 107u8, 103u8, 106u8, 117u8,
-                        103u8, 108u8, 0u8, 103u8, 108u8, 107u8, 103u8, 110u8, 0u8, 103u8, 111u8,
-                        109u8, 103u8, 111u8, 110u8, 103u8, 111u8, 114u8, 103u8, 111u8, 115u8,
-                        103u8, 111u8, 116u8, 103u8, 114u8, 99u8, 103u8, 114u8, 116u8, 103u8, 115u8,
-                        119u8, 103u8, 117u8, 0u8, 103u8, 117u8, 98u8, 103u8, 117u8, 99u8, 103u8,
-                        117u8, 114u8, 103u8, 117u8, 122u8, 103u8, 118u8, 0u8, 103u8, 118u8, 114u8,
-                        103u8, 119u8, 105u8, 104u8, 97u8, 0u8, 104u8, 97u8, 107u8, 104u8, 97u8,
-                        119u8, 104u8, 97u8, 122u8, 104u8, 101u8, 0u8, 104u8, 105u8, 0u8, 104u8,
-                        105u8, 102u8, 104u8, 105u8, 108u8, 104u8, 108u8, 117u8, 104u8, 109u8,
-                        100u8, 104u8, 110u8, 100u8, 104u8, 110u8, 101u8, 104u8, 110u8, 106u8,
-                        104u8, 110u8, 110u8, 104u8, 110u8, 111u8, 104u8, 111u8, 0u8, 104u8, 111u8,
-                        99u8, 104u8, 111u8, 106u8, 104u8, 114u8, 0u8, 104u8, 115u8, 98u8, 104u8,
-                        115u8, 110u8, 104u8, 116u8, 0u8, 104u8, 117u8, 0u8, 104u8, 117u8, 114u8,
-                        104u8, 121u8, 0u8, 104u8, 122u8, 0u8, 105u8, 97u8, 0u8, 105u8, 98u8, 97u8,
-                        105u8, 98u8, 98u8, 105u8, 100u8, 0u8, 105u8, 102u8, 101u8, 105u8, 103u8,
-                        0u8, 105u8, 105u8, 0u8, 105u8, 107u8, 0u8, 105u8, 108u8, 111u8, 105u8,
-                        110u8, 0u8, 105u8, 110u8, 104u8, 105u8, 111u8, 0u8, 105u8, 115u8, 0u8,
-                        105u8, 116u8, 0u8, 105u8, 117u8, 0u8, 105u8, 119u8, 0u8, 105u8, 122u8,
-                        104u8, 106u8, 97u8, 0u8, 106u8, 97u8, 109u8, 106u8, 98u8, 111u8, 106u8,
-                        103u8, 111u8, 106u8, 105u8, 0u8, 106u8, 109u8, 99u8, 106u8, 109u8, 108u8,
-                        106u8, 117u8, 116u8, 106u8, 118u8, 0u8, 106u8, 119u8, 0u8, 107u8, 97u8,
-                        0u8, 107u8, 97u8, 97u8, 107u8, 97u8, 98u8, 107u8, 97u8, 99u8, 107u8, 97u8,
-                        106u8, 107u8, 97u8, 109u8, 107u8, 97u8, 111u8, 107u8, 97u8, 119u8, 107u8,
-                        98u8, 100u8, 107u8, 98u8, 121u8, 107u8, 99u8, 103u8, 107u8, 99u8, 107u8,
-                        107u8, 100u8, 101u8, 107u8, 100u8, 104u8, 107u8, 100u8, 116u8, 107u8,
-                        101u8, 97u8, 107u8, 101u8, 110u8, 107u8, 102u8, 111u8, 107u8, 102u8, 114u8,
-                        107u8, 102u8, 121u8, 107u8, 103u8, 0u8, 107u8, 103u8, 101u8, 107u8, 103u8,
-                        112u8, 107u8, 104u8, 97u8, 107u8, 104u8, 98u8, 107u8, 104u8, 110u8, 107u8,
-                        104u8, 113u8, 107u8, 104u8, 116u8, 107u8, 104u8, 119u8, 107u8, 105u8, 0u8,
-                        107u8, 105u8, 117u8, 107u8, 106u8, 0u8, 107u8, 106u8, 103u8, 107u8, 107u8,
-                        0u8, 107u8, 107u8, 106u8, 107u8, 108u8, 0u8, 107u8, 108u8, 110u8, 107u8,
-                        109u8, 0u8, 107u8, 109u8, 98u8, 107u8, 110u8, 0u8, 107u8, 110u8, 102u8,
-                        107u8, 111u8, 0u8, 107u8, 111u8, 105u8, 107u8, 111u8, 107u8, 107u8, 111u8,
-                        115u8, 107u8, 112u8, 101u8, 107u8, 114u8, 99u8, 107u8, 114u8, 105u8, 107u8,
-                        114u8, 106u8, 107u8, 114u8, 108u8, 107u8, 114u8, 117u8, 107u8, 115u8, 0u8,
-                        107u8, 115u8, 98u8, 107u8, 115u8, 102u8, 107u8, 115u8, 104u8, 107u8, 116u8,
-                        114u8, 107u8, 117u8, 0u8, 107u8, 117u8, 109u8, 107u8, 118u8, 0u8, 107u8,
-                        118u8, 114u8, 107u8, 118u8, 120u8, 107u8, 119u8, 0u8, 107u8, 119u8, 107u8,
-                        107u8, 120u8, 108u8, 107u8, 120u8, 109u8, 107u8, 120u8, 112u8, 107u8,
-                        121u8, 0u8, 107u8, 122u8, 106u8, 107u8, 122u8, 116u8, 108u8, 97u8, 0u8,
-                        108u8, 97u8, 98u8, 108u8, 97u8, 100u8, 108u8, 97u8, 103u8, 108u8, 97u8,
-                        104u8, 108u8, 97u8, 106u8, 108u8, 98u8, 0u8, 108u8, 98u8, 101u8, 108u8,
-                        98u8, 119u8, 108u8, 99u8, 112u8, 108u8, 101u8, 112u8, 108u8, 101u8, 122u8,
-                        108u8, 103u8, 0u8, 108u8, 105u8, 0u8, 108u8, 105u8, 102u8, 108u8, 105u8,
-                        106u8, 108u8, 105u8, 108u8, 108u8, 105u8, 115u8, 108u8, 106u8, 112u8,
-                        108u8, 107u8, 105u8, 108u8, 107u8, 116u8, 108u8, 109u8, 110u8, 108u8,
-                        109u8, 111u8, 108u8, 110u8, 0u8, 108u8, 111u8, 0u8, 108u8, 111u8, 108u8,
-                        108u8, 111u8, 122u8, 108u8, 114u8, 99u8, 108u8, 116u8, 0u8, 108u8, 116u8,
-                        103u8, 108u8, 117u8, 0u8, 108u8, 117u8, 97u8, 108u8, 117u8, 111u8, 108u8,
-                        117u8, 121u8, 108u8, 117u8, 122u8, 108u8, 118u8, 0u8, 108u8, 119u8, 108u8,
-                        108u8, 122u8, 104u8, 108u8, 122u8, 122u8, 109u8, 97u8, 100u8, 109u8, 97u8,
-                        102u8, 109u8, 97u8, 103u8, 109u8, 97u8, 105u8, 109u8, 97u8, 107u8, 109u8,
-                        97u8, 110u8, 109u8, 97u8, 115u8, 109u8, 97u8, 122u8, 109u8, 100u8, 102u8,
-                        109u8, 100u8, 104u8, 109u8, 100u8, 114u8, 109u8, 101u8, 110u8, 109u8,
-                        101u8, 114u8, 109u8, 102u8, 97u8, 109u8, 102u8, 101u8, 109u8, 103u8, 0u8,
-                        109u8, 103u8, 104u8, 109u8, 103u8, 111u8, 109u8, 103u8, 112u8, 109u8,
-                        103u8, 121u8, 109u8, 104u8, 0u8, 109u8, 105u8, 0u8, 109u8, 105u8, 99u8,
-                        109u8, 105u8, 110u8, 109u8, 107u8, 0u8, 109u8, 108u8, 0u8, 109u8, 108u8,
-                        115u8, 109u8, 110u8, 0u8, 109u8, 110u8, 105u8, 109u8, 110u8, 119u8, 109u8,
-                        111u8, 0u8, 109u8, 111u8, 101u8, 109u8, 111u8, 104u8, 109u8, 111u8, 115u8,
-                        109u8, 114u8, 0u8, 109u8, 114u8, 100u8, 109u8, 114u8, 106u8, 109u8, 114u8,
-                        111u8, 109u8, 115u8, 0u8, 109u8, 116u8, 0u8, 109u8, 116u8, 114u8, 109u8,
-                        117u8, 97u8, 109u8, 117u8, 115u8, 109u8, 118u8, 121u8, 109u8, 119u8, 107u8,
-                        109u8, 119u8, 114u8, 109u8, 119u8, 118u8, 109u8, 119u8, 119u8, 109u8,
-                        120u8, 99u8, 109u8, 121u8, 0u8, 109u8, 121u8, 118u8, 109u8, 121u8, 120u8,
-                        109u8, 121u8, 122u8, 109u8, 122u8, 110u8, 110u8, 97u8, 0u8, 110u8, 97u8,
-                        110u8, 110u8, 97u8, 112u8, 110u8, 97u8, 113u8, 110u8, 98u8, 0u8, 110u8,
-                        99u8, 104u8, 110u8, 100u8, 0u8, 110u8, 100u8, 99u8, 110u8, 100u8, 115u8,
-                        110u8, 101u8, 0u8, 110u8, 101u8, 119u8, 110u8, 103u8, 0u8, 110u8, 103u8,
-                        108u8, 110u8, 104u8, 101u8, 110u8, 104u8, 119u8, 110u8, 105u8, 106u8,
-                        110u8, 105u8, 117u8, 110u8, 106u8, 111u8, 110u8, 108u8, 0u8, 110u8, 109u8,
-                        103u8, 110u8, 110u8, 0u8, 110u8, 110u8, 104u8, 110u8, 110u8, 112u8, 110u8,
-                        111u8, 0u8, 110u8, 111u8, 100u8, 110u8, 111u8, 101u8, 110u8, 111u8, 110u8,
-                        110u8, 113u8, 111u8, 110u8, 114u8, 0u8, 110u8, 115u8, 107u8, 110u8, 115u8,
-                        111u8, 110u8, 115u8, 116u8, 110u8, 117u8, 115u8, 110u8, 118u8, 0u8, 110u8,
-                        120u8, 113u8, 110u8, 121u8, 0u8, 110u8, 121u8, 109u8, 110u8, 121u8, 110u8,
-                        110u8, 122u8, 105u8, 111u8, 99u8, 0u8, 111u8, 106u8, 0u8, 111u8, 106u8,
-                        115u8, 111u8, 107u8, 97u8, 111u8, 109u8, 0u8, 111u8, 114u8, 0u8, 111u8,
-                        115u8, 0u8, 111u8, 115u8, 97u8, 111u8, 116u8, 107u8, 111u8, 117u8, 105u8,
-                        112u8, 97u8, 0u8, 112u8, 97u8, 103u8, 112u8, 97u8, 108u8, 112u8, 97u8,
-                        109u8, 112u8, 97u8, 112u8, 112u8, 97u8, 117u8, 112u8, 99u8, 100u8, 112u8,
-                        99u8, 109u8, 112u8, 100u8, 99u8, 112u8, 100u8, 116u8, 112u8, 101u8, 111u8,
-                        112u8, 102u8, 108u8, 112u8, 104u8, 110u8, 112u8, 105u8, 115u8, 112u8,
-                        107u8, 97u8, 112u8, 107u8, 111u8, 112u8, 108u8, 0u8, 112u8, 109u8, 115u8,
-                        112u8, 110u8, 116u8, 112u8, 111u8, 110u8, 112u8, 112u8, 97u8, 112u8, 113u8,
-                        109u8, 112u8, 114u8, 97u8, 112u8, 114u8, 100u8, 112u8, 114u8, 103u8, 112u8,
-                        115u8, 0u8, 112u8, 116u8, 0u8, 112u8, 117u8, 117u8, 113u8, 117u8, 0u8,
-                        113u8, 117u8, 99u8, 113u8, 117u8, 103u8, 114u8, 97u8, 106u8, 114u8, 99u8,
-                        102u8, 114u8, 101u8, 106u8, 114u8, 103u8, 110u8, 114u8, 104u8, 103u8,
-                        114u8, 105u8, 97u8, 114u8, 105u8, 102u8, 114u8, 106u8, 115u8, 114u8, 107u8,
-                        116u8, 114u8, 109u8, 0u8, 114u8, 109u8, 102u8, 114u8, 109u8, 111u8, 114u8,
-                        109u8, 116u8, 114u8, 109u8, 117u8, 114u8, 110u8, 0u8, 114u8, 110u8, 103u8,
-                        114u8, 111u8, 0u8, 114u8, 111u8, 98u8, 114u8, 111u8, 102u8, 114u8, 116u8,
-                        109u8, 114u8, 117u8, 0u8, 114u8, 117u8, 101u8, 114u8, 117u8, 103u8, 114u8,
-                        119u8, 0u8, 114u8, 119u8, 107u8, 114u8, 121u8, 117u8, 115u8, 97u8, 0u8,
-                        115u8, 97u8, 102u8, 115u8, 97u8, 104u8, 115u8, 97u8, 113u8, 115u8, 97u8,
-                        115u8, 115u8, 97u8, 116u8, 115u8, 97u8, 118u8, 115u8, 97u8, 122u8, 115u8,
-                        98u8, 112u8, 115u8, 99u8, 0u8, 115u8, 99u8, 107u8, 115u8, 99u8, 110u8,
-                        115u8, 99u8, 111u8, 115u8, 100u8, 0u8, 115u8, 100u8, 99u8, 115u8, 100u8,
-                        104u8, 115u8, 101u8, 0u8, 115u8, 101u8, 102u8, 115u8, 101u8, 104u8, 115u8,
-                        101u8, 105u8, 115u8, 101u8, 115u8, 115u8, 103u8, 0u8, 115u8, 103u8, 97u8,
-                        115u8, 103u8, 115u8, 115u8, 104u8, 105u8, 115u8, 104u8, 110u8, 115u8,
-                        105u8, 0u8, 115u8, 105u8, 100u8, 115u8, 107u8, 0u8, 115u8, 107u8, 114u8,
-                        115u8, 108u8, 0u8, 115u8, 108u8, 105u8, 115u8, 108u8, 121u8, 115u8, 109u8,
-                        0u8, 115u8, 109u8, 97u8, 115u8, 109u8, 100u8, 115u8, 109u8, 106u8, 115u8,
-                        109u8, 110u8, 115u8, 109u8, 112u8, 115u8, 109u8, 115u8, 115u8, 110u8, 0u8,
-                        115u8, 110u8, 98u8, 115u8, 110u8, 107u8, 115u8, 111u8, 0u8, 115u8, 111u8,
-                        103u8, 115u8, 111u8, 117u8, 115u8, 113u8, 0u8, 115u8, 114u8, 0u8, 115u8,
-                        114u8, 98u8, 115u8, 114u8, 110u8, 115u8, 114u8, 114u8, 115u8, 114u8, 120u8,
-                        115u8, 115u8, 0u8, 115u8, 115u8, 121u8, 115u8, 116u8, 0u8, 115u8, 116u8,
-                        113u8, 115u8, 117u8, 0u8, 115u8, 117u8, 107u8, 115u8, 117u8, 115u8, 115u8,
-                        118u8, 0u8, 115u8, 119u8, 0u8, 115u8, 119u8, 98u8, 115u8, 119u8, 99u8,
-                        115u8, 119u8, 103u8, 115u8, 119u8, 118u8, 115u8, 120u8, 110u8, 115u8,
-                        121u8, 108u8, 115u8, 121u8, 114u8, 115u8, 122u8, 108u8, 116u8, 97u8, 0u8,
-                        116u8, 97u8, 106u8, 116u8, 98u8, 119u8, 116u8, 99u8, 121u8, 116u8, 100u8,
-                        100u8, 116u8, 100u8, 103u8, 116u8, 100u8, 104u8, 116u8, 100u8, 117u8,
-                        116u8, 101u8, 0u8, 116u8, 101u8, 109u8, 116u8, 101u8, 111u8, 116u8, 101u8,
-                        116u8, 116u8, 103u8, 0u8, 116u8, 104u8, 0u8, 116u8, 104u8, 108u8, 116u8,
-                        104u8, 113u8, 116u8, 104u8, 114u8, 116u8, 105u8, 0u8, 116u8, 105u8, 103u8,
-                        116u8, 105u8, 118u8, 116u8, 107u8, 0u8, 116u8, 107u8, 108u8, 116u8, 107u8,
-                        114u8, 116u8, 107u8, 116u8, 116u8, 108u8, 0u8, 116u8, 108u8, 121u8, 116u8,
-                        109u8, 104u8, 116u8, 110u8, 0u8, 116u8, 111u8, 0u8, 116u8, 111u8, 103u8,
-                        116u8, 111u8, 107u8, 116u8, 112u8, 105u8, 116u8, 114u8, 0u8, 116u8, 114u8,
-                        117u8, 116u8, 114u8, 118u8, 116u8, 114u8, 119u8, 116u8, 115u8, 0u8, 116u8,
-                        115u8, 100u8, 116u8, 115u8, 102u8, 116u8, 115u8, 103u8, 116u8, 115u8,
-                        106u8, 116u8, 116u8, 0u8, 116u8, 116u8, 106u8, 116u8, 116u8, 115u8, 116u8,
-                        116u8, 116u8, 116u8, 117u8, 109u8, 116u8, 118u8, 108u8, 116u8, 119u8,
-                        113u8, 116u8, 120u8, 103u8, 116u8, 120u8, 111u8, 116u8, 121u8, 0u8, 116u8,
-                        121u8, 118u8, 116u8, 122u8, 109u8, 117u8, 100u8, 105u8, 117u8, 100u8,
-                        109u8, 117u8, 103u8, 0u8, 117u8, 103u8, 97u8, 117u8, 107u8, 0u8, 117u8,
-                        108u8, 105u8, 117u8, 109u8, 98u8, 117u8, 110u8, 114u8, 117u8, 110u8, 120u8,
-                        117u8, 114u8, 0u8, 117u8, 122u8, 0u8, 118u8, 97u8, 105u8, 118u8, 101u8,
-                        0u8, 118u8, 101u8, 99u8, 118u8, 101u8, 112u8, 118u8, 105u8, 0u8, 118u8,
-                        105u8, 99u8, 118u8, 108u8, 115u8, 118u8, 109u8, 102u8, 118u8, 109u8, 119u8,
-                        118u8, 111u8, 0u8, 118u8, 111u8, 116u8, 118u8, 114u8, 111u8, 118u8, 117u8,
-                        110u8, 119u8, 97u8, 0u8, 119u8, 97u8, 101u8, 119u8, 97u8, 108u8, 119u8,
-                        97u8, 114u8, 119u8, 98u8, 112u8, 119u8, 98u8, 113u8, 119u8, 98u8, 114u8,
-                        119u8, 108u8, 115u8, 119u8, 110u8, 105u8, 119u8, 111u8, 0u8, 119u8, 115u8,
-                        103u8, 119u8, 116u8, 109u8, 119u8, 117u8, 117u8, 120u8, 97u8, 118u8, 120u8,
-                        99u8, 111u8, 120u8, 99u8, 114u8, 120u8, 104u8, 0u8, 120u8, 108u8, 99u8,
-                        120u8, 108u8, 100u8, 120u8, 109u8, 102u8, 120u8, 109u8, 110u8, 120u8,
-                        109u8, 114u8, 120u8, 110u8, 97u8, 120u8, 110u8, 114u8, 120u8, 111u8, 103u8,
-                        120u8, 112u8, 114u8, 120u8, 115u8, 97u8, 120u8, 115u8, 114u8, 121u8, 97u8,
-                        111u8, 121u8, 97u8, 112u8, 121u8, 97u8, 118u8, 121u8, 98u8, 98u8, 121u8,
-                        105u8, 0u8, 121u8, 111u8, 0u8, 121u8, 114u8, 108u8, 121u8, 117u8, 97u8,
-                        121u8, 117u8, 101u8, 122u8, 97u8, 0u8, 122u8, 97u8, 103u8, 122u8, 100u8,
-                        106u8, 122u8, 101u8, 97u8, 122u8, 103u8, 104u8, 122u8, 104u8, 0u8, 122u8,
-                        104u8, 120u8, 122u8, 107u8, 116u8, 122u8, 108u8, 109u8, 122u8, 109u8,
-                        105u8, 122u8, 117u8, 0u8, 122u8, 122u8, 97u8,
-                    ])
-                },
-                unsafe {
-                    ::zerovec::ZeroVec::from_bytes_unchecked(&[
-                        69u8, 84u8, 0u8, 71u8, 69u8, 0u8, 71u8, 72u8, 0u8, 73u8, 68u8, 0u8, 85u8,
-                        71u8, 0u8, 71u8, 72u8, 0u8, 66u8, 84u8, 0u8, 82u8, 85u8, 0u8, 73u8, 82u8,
-                        0u8, 84u8, 78u8, 0u8, 90u8, 65u8, 0u8, 67u8, 77u8, 0u8, 73u8, 78u8, 0u8,
-                        84u8, 78u8, 0u8, 71u8, 72u8, 0u8, 73u8, 81u8, 0u8, 88u8, 75u8, 0u8, 82u8,
-                        85u8, 0u8, 69u8, 84u8, 0u8, 78u8, 71u8, 0u8, 69u8, 83u8, 0u8, 78u8, 71u8,
-                        0u8, 73u8, 68u8, 0u8, 84u8, 71u8, 0u8, 69u8, 71u8, 0u8, 73u8, 82u8, 0u8,
-                        67u8, 76u8, 0u8, 66u8, 79u8, 0u8, 68u8, 90u8, 0u8, 83u8, 65u8, 0u8, 77u8,
-                        65u8, 0u8, 69u8, 71u8, 0u8, 73u8, 78u8, 0u8, 84u8, 90u8, 0u8, 85u8, 83u8,
-                        0u8, 69u8, 83u8, 0u8, 67u8, 65u8, 0u8, 82u8, 85u8, 0u8, 73u8, 78u8, 0u8,
-                        66u8, 79u8, 0u8, 65u8, 90u8, 0u8, 82u8, 85u8, 0u8, 80u8, 75u8, 0u8, 73u8,
-                        68u8, 0u8, 78u8, 80u8, 0u8, 65u8, 84u8, 0u8, 67u8, 77u8, 0u8, 67u8, 77u8,
-                        0u8, 73u8, 68u8, 0u8, 67u8, 77u8, 0u8, 67u8, 73u8, 0u8, 66u8, 89u8, 0u8,
-                        83u8, 68u8, 0u8, 90u8, 77u8, 0u8, 73u8, 68u8, 0u8, 84u8, 90u8, 0u8, 67u8,
-                        77u8, 0u8, 73u8, 78u8, 0u8, 80u8, 75u8, 0u8, 73u8, 78u8, 0u8, 66u8, 71u8,
-                        0u8, 73u8, 78u8, 0u8, 80u8, 75u8, 0u8, 84u8, 82u8, 0u8, 73u8, 78u8, 0u8,
-                        73u8, 78u8, 0u8, 73u8, 78u8, 0u8, 86u8, 85u8, 0u8, 80u8, 72u8, 0u8, 78u8,
-                        71u8, 0u8, 73u8, 78u8, 0u8, 73u8, 68u8, 0u8, 83u8, 78u8, 0u8, 67u8, 77u8,
-                        0u8, 80u8, 72u8, 0u8, 67u8, 65u8, 0u8, 77u8, 89u8, 0u8, 86u8, 78u8, 0u8,
-                        77u8, 76u8, 0u8, 77u8, 76u8, 0u8, 66u8, 68u8, 0u8, 67u8, 78u8, 0u8, 73u8,
-                        78u8, 0u8, 73u8, 82u8, 0u8, 67u8, 73u8, 0u8, 70u8, 82u8, 0u8, 73u8, 78u8,
-                        0u8, 80u8, 75u8, 0u8, 73u8, 78u8, 0u8, 66u8, 65u8, 0u8, 76u8, 82u8, 0u8,
-                        67u8, 77u8, 0u8, 80u8, 72u8, 0u8, 80u8, 75u8, 0u8, 82u8, 85u8, 0u8, 89u8,
-                        84u8, 0u8, 73u8, 68u8, 0u8, 67u8, 77u8, 0u8, 71u8, 81u8, 0u8, 69u8, 82u8,
-                        0u8, 67u8, 77u8, 0u8, 77u8, 76u8, 0u8, 69u8, 83u8, 0u8, 85u8, 83u8, 0u8,
-                        78u8, 71u8, 0u8, 66u8, 68u8, 0u8, 82u8, 85u8, 0u8, 80u8, 72u8, 0u8, 85u8,
-                        71u8, 0u8, 71u8, 85u8, 0u8, 70u8, 77u8, 0u8, 82u8, 85u8, 0u8, 85u8, 83u8,
-                        0u8, 67u8, 65u8, 0u8, 85u8, 83u8, 0u8, 85u8, 83u8, 0u8, 75u8, 72u8, 0u8,
-                        86u8, 78u8, 0u8, 73u8, 81u8, 0u8, 67u8, 65u8, 0u8, 77u8, 78u8, 0u8, 70u8,
-                        82u8, 0u8, 69u8, 71u8, 0u8, 80u8, 72u8, 0u8, 67u8, 65u8, 0u8, 67u8, 65u8,
-                        0u8, 85u8, 65u8, 0u8, 67u8, 65u8, 0u8, 67u8, 65u8, 0u8, 83u8, 67u8, 0u8,
-                        67u8, 90u8, 0u8, 80u8, 76u8, 0u8, 67u8, 65u8, 0u8, 77u8, 77u8, 0u8, 82u8,
-                        85u8, 0u8, 82u8, 85u8, 0u8, 71u8, 66u8, 0u8, 68u8, 75u8, 0u8, 67u8, 73u8,
-                        0u8, 85u8, 83u8, 0u8, 82u8, 85u8, 0u8, 75u8, 69u8, 0u8, 73u8, 78u8, 0u8,
-                        68u8, 69u8, 0u8, 67u8, 65u8, 0u8, 67u8, 65u8, 0u8, 78u8, 69u8, 0u8, 78u8,
-                        71u8, 0u8, 67u8, 73u8, 0u8, 73u8, 78u8, 0u8, 67u8, 78u8, 0u8, 68u8, 69u8,
-                        0u8, 77u8, 76u8, 0u8, 77u8, 89u8, 0u8, 78u8, 80u8, 0u8, 67u8, 77u8, 0u8,
-                        77u8, 86u8, 0u8, 83u8, 78u8, 0u8, 66u8, 70u8, 0u8, 66u8, 84u8, 0u8, 75u8,
-                        69u8, 0u8, 71u8, 72u8, 0u8, 78u8, 71u8, 0u8, 73u8, 84u8, 0u8, 69u8, 71u8,
-                        0u8, 77u8, 77u8, 0u8, 71u8, 82u8, 0u8, 85u8, 83u8, 0u8, 48u8, 48u8, 49u8,
-                        69u8, 83u8, 0u8, 73u8, 78u8, 0u8, 85u8, 83u8, 0u8, 69u8, 69u8, 0u8, 73u8,
-                        84u8, 0u8, 69u8, 83u8, 0u8, 67u8, 77u8, 0u8, 69u8, 83u8, 0u8, 73u8, 82u8,
-                        0u8, 71u8, 81u8, 0u8, 83u8, 78u8, 0u8, 77u8, 76u8, 0u8, 70u8, 73u8, 0u8,
-                        83u8, 68u8, 0u8, 80u8, 72u8, 0u8, 83u8, 69u8, 0u8, 70u8, 74u8, 0u8, 70u8,
-                        79u8, 0u8, 66u8, 74u8, 0u8, 70u8, 82u8, 0u8, 85u8, 83u8, 0u8, 70u8, 82u8,
-                        0u8, 68u8, 69u8, 0u8, 68u8, 69u8, 0u8, 67u8, 77u8, 0u8, 87u8, 70u8, 0u8,
-                        71u8, 78u8, 0u8, 78u8, 69u8, 0u8, 73u8, 84u8, 0u8, 78u8, 71u8, 0u8, 83u8,
-                        68u8, 0u8, 78u8, 76u8, 0u8, 73u8, 69u8, 0u8, 71u8, 72u8, 0u8, 77u8, 68u8,
-                        0u8, 67u8, 78u8, 0u8, 73u8, 68u8, 0u8, 73u8, 78u8, 0u8, 73u8, 82u8, 0u8,
-                        71u8, 70u8, 0u8, 71u8, 66u8, 0u8, 69u8, 84u8, 0u8, 78u8, 80u8, 0u8, 75u8,
-                        73u8, 0u8, 80u8, 75u8, 0u8, 80u8, 75u8, 0u8, 69u8, 83u8, 0u8, 73u8, 82u8,
-                        0u8, 80u8, 89u8, 0u8, 73u8, 78u8, 0u8, 73u8, 78u8, 0u8, 73u8, 68u8, 0u8,
-                        78u8, 76u8, 0u8, 85u8, 65u8, 0u8, 67u8, 89u8, 0u8, 73u8, 78u8, 0u8, 67u8,
-                        72u8, 0u8, 73u8, 78u8, 0u8, 66u8, 82u8, 0u8, 67u8, 79u8, 0u8, 71u8, 72u8,
-                        0u8, 75u8, 69u8, 0u8, 73u8, 77u8, 0u8, 78u8, 80u8, 0u8, 67u8, 65u8, 0u8,
-                        78u8, 71u8, 0u8, 67u8, 78u8, 0u8, 85u8, 83u8, 0u8, 65u8, 70u8, 0u8, 73u8,
-                        76u8, 0u8, 73u8, 78u8, 0u8, 70u8, 74u8, 0u8, 80u8, 72u8, 0u8, 84u8, 82u8,
-                        0u8, 67u8, 78u8, 0u8, 80u8, 75u8, 0u8, 73u8, 78u8, 0u8, 85u8, 83u8, 0u8,
-                        80u8, 72u8, 0u8, 80u8, 75u8, 0u8, 80u8, 71u8, 0u8, 73u8, 78u8, 0u8, 73u8,
-                        78u8, 0u8, 72u8, 82u8, 0u8, 68u8, 69u8, 0u8, 67u8, 78u8, 0u8, 72u8, 84u8,
-                        0u8, 72u8, 85u8, 0u8, 67u8, 65u8, 0u8, 65u8, 77u8, 0u8, 78u8, 65u8, 0u8,
-                        48u8, 48u8, 49u8, 77u8, 89u8, 0u8, 78u8, 71u8, 0u8, 73u8, 68u8, 0u8, 84u8,
-                        71u8, 0u8, 78u8, 71u8, 0u8, 67u8, 78u8, 0u8, 85u8, 83u8, 0u8, 80u8, 72u8,
-                        0u8, 73u8, 68u8, 0u8, 82u8, 85u8, 0u8, 48u8, 48u8, 49u8, 73u8, 83u8, 0u8,
-                        73u8, 84u8, 0u8, 67u8, 65u8, 0u8, 73u8, 76u8, 0u8, 82u8, 85u8, 0u8, 74u8,
-                        80u8, 0u8, 74u8, 77u8, 0u8, 48u8, 48u8, 49u8, 67u8, 77u8, 0u8, 85u8, 65u8,
-                        0u8, 84u8, 90u8, 0u8, 78u8, 80u8, 0u8, 68u8, 75u8, 0u8, 73u8, 68u8, 0u8,
-                        73u8, 68u8, 0u8, 71u8, 69u8, 0u8, 85u8, 90u8, 0u8, 68u8, 90u8, 0u8, 77u8,
-                        77u8, 0u8, 78u8, 71u8, 0u8, 75u8, 69u8, 0u8, 77u8, 76u8, 0u8, 73u8, 68u8,
-                        0u8, 82u8, 85u8, 0u8, 78u8, 69u8, 0u8, 78u8, 71u8, 0u8, 90u8, 87u8, 0u8,
-                        84u8, 90u8, 0u8, 84u8, 71u8, 0u8, 84u8, 72u8, 0u8, 67u8, 86u8, 0u8, 67u8,
-                        77u8, 0u8, 67u8, 73u8, 0u8, 73u8, 78u8, 0u8, 73u8, 78u8, 0u8, 67u8, 68u8,
-                        0u8, 73u8, 68u8, 0u8, 66u8, 82u8, 0u8, 73u8, 78u8, 0u8, 67u8, 78u8, 0u8,
-                        73u8, 78u8, 0u8, 77u8, 76u8, 0u8, 73u8, 78u8, 0u8, 80u8, 75u8, 0u8, 75u8,
-                        69u8, 0u8, 84u8, 82u8, 0u8, 78u8, 65u8, 0u8, 76u8, 65u8, 0u8, 75u8, 90u8,
-                        0u8, 67u8, 77u8, 0u8, 71u8, 76u8, 0u8, 75u8, 69u8, 0u8, 75u8, 72u8, 0u8,
-                        65u8, 79u8, 0u8, 73u8, 78u8, 0u8, 71u8, 87u8, 0u8, 75u8, 82u8, 0u8, 82u8,
-                        85u8, 0u8, 73u8, 78u8, 0u8, 70u8, 77u8, 0u8, 76u8, 82u8, 0u8, 82u8, 85u8,
-                        0u8, 83u8, 76u8, 0u8, 80u8, 72u8, 0u8, 82u8, 85u8, 0u8, 73u8, 78u8, 0u8,
-                        73u8, 78u8, 0u8, 84u8, 90u8, 0u8, 67u8, 77u8, 0u8, 68u8, 69u8, 0u8, 77u8,
-                        89u8, 0u8, 84u8, 82u8, 0u8, 82u8, 85u8, 0u8, 82u8, 85u8, 0u8, 73u8, 68u8,
-                        0u8, 80u8, 75u8, 0u8, 71u8, 66u8, 0u8, 67u8, 65u8, 0u8, 73u8, 78u8, 0u8,
-                        84u8, 72u8, 0u8, 80u8, 75u8, 0u8, 75u8, 71u8, 0u8, 77u8, 89u8, 0u8, 77u8,
-                        89u8, 0u8, 86u8, 65u8, 0u8, 71u8, 82u8, 0u8, 73u8, 76u8, 0u8, 84u8, 90u8,
-                        0u8, 80u8, 75u8, 0u8, 85u8, 71u8, 0u8, 76u8, 85u8, 0u8, 82u8, 85u8, 0u8,
-                        73u8, 68u8, 0u8, 67u8, 78u8, 0u8, 73u8, 78u8, 0u8, 82u8, 85u8, 0u8, 85u8,
-                        71u8, 0u8, 78u8, 76u8, 0u8, 78u8, 80u8, 0u8, 73u8, 84u8, 0u8, 67u8, 65u8,
-                        0u8, 67u8, 78u8, 0u8, 73u8, 68u8, 0u8, 73u8, 82u8, 0u8, 85u8, 83u8, 0u8,
-                        73u8, 78u8, 0u8, 73u8, 84u8, 0u8, 67u8, 68u8, 0u8, 76u8, 65u8, 0u8, 67u8,
-                        68u8, 0u8, 90u8, 77u8, 0u8, 73u8, 82u8, 0u8, 76u8, 84u8, 0u8, 76u8, 86u8,
-                        0u8, 67u8, 68u8, 0u8, 67u8, 68u8, 0u8, 75u8, 69u8, 0u8, 75u8, 69u8, 0u8,
-                        73u8, 82u8, 0u8, 76u8, 86u8, 0u8, 84u8, 72u8, 0u8, 67u8, 78u8, 0u8, 84u8,
-                        82u8, 0u8, 73u8, 68u8, 0u8, 67u8, 77u8, 0u8, 73u8, 78u8, 0u8, 73u8, 78u8,
-                        0u8, 73u8, 68u8, 0u8, 71u8, 77u8, 0u8, 75u8, 69u8, 0u8, 77u8, 88u8, 0u8,
-                        82u8, 85u8, 0u8, 80u8, 72u8, 0u8, 73u8, 68u8, 0u8, 83u8, 76u8, 0u8, 75u8,
-                        69u8, 0u8, 84u8, 72u8, 0u8, 77u8, 85u8, 0u8, 77u8, 71u8, 0u8, 77u8, 90u8,
-                        0u8, 67u8, 77u8, 0u8, 78u8, 80u8, 0u8, 84u8, 90u8, 0u8, 77u8, 72u8, 0u8,
-                        78u8, 90u8, 0u8, 67u8, 65u8, 0u8, 73u8, 68u8, 0u8, 77u8, 75u8, 0u8, 73u8,
-                        78u8, 0u8, 83u8, 68u8, 0u8, 77u8, 78u8, 0u8, 73u8, 78u8, 0u8, 77u8, 77u8,
-                        0u8, 82u8, 79u8, 0u8, 67u8, 65u8, 0u8, 67u8, 65u8, 0u8, 66u8, 70u8, 0u8,
-                        73u8, 78u8, 0u8, 78u8, 80u8, 0u8, 82u8, 85u8, 0u8, 66u8, 68u8, 0u8, 77u8,
-                        89u8, 0u8, 77u8, 84u8, 0u8, 73u8, 78u8, 0u8, 67u8, 77u8, 0u8, 85u8, 83u8,
-                        0u8, 80u8, 75u8, 0u8, 77u8, 76u8, 0u8, 73u8, 78u8, 0u8, 73u8, 68u8, 0u8,
-                        85u8, 83u8, 0u8, 90u8, 87u8, 0u8, 77u8, 77u8, 0u8, 82u8, 85u8, 0u8, 85u8,
-                        71u8, 0u8, 73u8, 82u8, 0u8, 73u8, 82u8, 0u8, 78u8, 82u8, 0u8, 67u8, 78u8,
-                        0u8, 73u8, 84u8, 0u8, 78u8, 65u8, 0u8, 78u8, 79u8, 0u8, 77u8, 88u8, 0u8,
-                        90u8, 87u8, 0u8, 77u8, 90u8, 0u8, 68u8, 69u8, 0u8, 78u8, 80u8, 0u8, 78u8,
-                        80u8, 0u8, 78u8, 65u8, 0u8, 77u8, 90u8, 0u8, 77u8, 88u8, 0u8, 77u8, 88u8,
-                        0u8, 73u8, 68u8, 0u8, 78u8, 85u8, 0u8, 73u8, 78u8, 0u8, 78u8, 76u8, 0u8,
-                        67u8, 77u8, 0u8, 78u8, 79u8, 0u8, 67u8, 77u8, 0u8, 73u8, 78u8, 0u8, 78u8,
-                        79u8, 0u8, 84u8, 72u8, 0u8, 73u8, 78u8, 0u8, 83u8, 69u8, 0u8, 71u8, 78u8,
-                        0u8, 90u8, 65u8, 0u8, 67u8, 65u8, 0u8, 90u8, 65u8, 0u8, 73u8, 78u8, 0u8,
-                        83u8, 83u8, 0u8, 85u8, 83u8, 0u8, 67u8, 78u8, 0u8, 77u8, 87u8, 0u8, 84u8,
-                        90u8, 0u8, 85u8, 71u8, 0u8, 71u8, 72u8, 0u8, 70u8, 82u8, 0u8, 67u8, 65u8,
-                        0u8, 67u8, 65u8, 0u8, 67u8, 65u8, 0u8, 69u8, 84u8, 0u8, 73u8, 78u8, 0u8,
-                        71u8, 69u8, 0u8, 85u8, 83u8, 0u8, 77u8, 78u8, 0u8, 49u8, 52u8, 51u8, 73u8,
-                        78u8, 0u8, 80u8, 72u8, 0u8, 73u8, 82u8, 0u8, 80u8, 72u8, 0u8, 65u8, 87u8,
-                        0u8, 80u8, 87u8, 0u8, 70u8, 82u8, 0u8, 78u8, 71u8, 0u8, 85u8, 83u8, 0u8,
-                        67u8, 65u8, 0u8, 73u8, 82u8, 0u8, 68u8, 69u8, 0u8, 76u8, 66u8, 0u8, 83u8,
-                        66u8, 0u8, 73u8, 78u8, 0u8, 75u8, 69u8, 0u8, 80u8, 76u8, 0u8, 73u8, 84u8,
-                        0u8, 71u8, 82u8, 0u8, 70u8, 77u8, 0u8, 73u8, 78u8, 0u8, 67u8, 65u8, 0u8,
-                        80u8, 75u8, 0u8, 73u8, 82u8, 0u8, 48u8, 48u8, 49u8, 65u8, 70u8, 0u8, 66u8,
-                        82u8, 0u8, 71u8, 65u8, 0u8, 80u8, 69u8, 0u8, 71u8, 84u8, 0u8, 69u8, 67u8,
-                        0u8, 73u8, 78u8, 0u8, 82u8, 69u8, 0u8, 73u8, 68u8, 0u8, 73u8, 84u8, 0u8,
-                        77u8, 77u8, 0u8, 73u8, 78u8, 0u8, 77u8, 65u8, 0u8, 78u8, 80u8, 0u8, 66u8,
-                        68u8, 0u8, 67u8, 72u8, 0u8, 70u8, 73u8, 0u8, 67u8, 72u8, 0u8, 73u8, 82u8,
-                        0u8, 83u8, 69u8, 0u8, 66u8, 73u8, 0u8, 77u8, 90u8, 0u8, 82u8, 79u8, 0u8,
-                        73u8, 68u8, 0u8, 84u8, 90u8, 0u8, 70u8, 74u8, 0u8, 82u8, 85u8, 0u8, 85u8,
-                        65u8, 0u8, 83u8, 66u8, 0u8, 82u8, 87u8, 0u8, 84u8, 90u8, 0u8, 74u8, 80u8,
-                        0u8, 73u8, 78u8, 0u8, 71u8, 72u8, 0u8, 82u8, 85u8, 0u8, 75u8, 69u8, 0u8,
-                        73u8, 68u8, 0u8, 73u8, 78u8, 0u8, 83u8, 78u8, 0u8, 73u8, 78u8, 0u8, 84u8,
-                        90u8, 0u8, 73u8, 84u8, 0u8, 73u8, 78u8, 0u8, 73u8, 84u8, 0u8, 71u8, 66u8,
-                        0u8, 80u8, 75u8, 0u8, 73u8, 84u8, 0u8, 73u8, 82u8, 0u8, 78u8, 79u8, 0u8,
-                        67u8, 73u8, 0u8, 77u8, 90u8, 0u8, 77u8, 88u8, 0u8, 77u8, 76u8, 0u8, 67u8,
-                        70u8, 0u8, 73u8, 69u8, 0u8, 76u8, 84u8, 0u8, 77u8, 65u8, 0u8, 77u8, 77u8,
-                        0u8, 76u8, 75u8, 0u8, 69u8, 84u8, 0u8, 83u8, 75u8, 0u8, 80u8, 75u8, 0u8,
-                        83u8, 73u8, 0u8, 80u8, 76u8, 0u8, 73u8, 68u8, 0u8, 87u8, 83u8, 0u8, 83u8,
-                        69u8, 0u8, 65u8, 79u8, 0u8, 83u8, 69u8, 0u8, 70u8, 73u8, 0u8, 73u8, 76u8,
-                        0u8, 70u8, 73u8, 0u8, 90u8, 87u8, 0u8, 77u8, 89u8, 0u8, 77u8, 76u8, 0u8,
-                        83u8, 79u8, 0u8, 85u8, 90u8, 0u8, 84u8, 72u8, 0u8, 65u8, 76u8, 0u8, 82u8,
-                        83u8, 0u8, 73u8, 78u8, 0u8, 83u8, 82u8, 0u8, 83u8, 78u8, 0u8, 73u8, 78u8,
-                        0u8, 90u8, 65u8, 0u8, 69u8, 82u8, 0u8, 90u8, 65u8, 0u8, 68u8, 69u8, 0u8,
-                        73u8, 68u8, 0u8, 84u8, 90u8, 0u8, 71u8, 78u8, 0u8, 83u8, 69u8, 0u8, 84u8,
-                        90u8, 0u8, 89u8, 84u8, 0u8, 67u8, 68u8, 0u8, 68u8, 69u8, 0u8, 73u8, 78u8,
-                        0u8, 73u8, 68u8, 0u8, 66u8, 68u8, 0u8, 73u8, 81u8, 0u8, 80u8, 76u8, 0u8,
-                        73u8, 78u8, 0u8, 78u8, 80u8, 0u8, 80u8, 72u8, 0u8, 73u8, 78u8, 0u8, 67u8,
-                        78u8, 0u8, 78u8, 80u8, 0u8, 78u8, 80u8, 0u8, 77u8, 89u8, 0u8, 73u8, 78u8,
-                        0u8, 83u8, 76u8, 0u8, 85u8, 71u8, 0u8, 84u8, 76u8, 0u8, 84u8, 74u8, 0u8,
-                        84u8, 72u8, 0u8, 78u8, 80u8, 0u8, 78u8, 80u8, 0u8, 78u8, 80u8, 0u8, 69u8,
-                        84u8, 0u8, 69u8, 82u8, 0u8, 78u8, 71u8, 0u8, 84u8, 77u8, 0u8, 84u8, 75u8,
-                        0u8, 65u8, 90u8, 0u8, 78u8, 80u8, 0u8, 80u8, 72u8, 0u8, 65u8, 90u8, 0u8,
-                        78u8, 69u8, 0u8, 90u8, 65u8, 0u8, 84u8, 79u8, 0u8, 77u8, 87u8, 0u8, 48u8,
-                        48u8, 49u8, 80u8, 71u8, 0u8, 84u8, 82u8, 0u8, 84u8, 82u8, 0u8, 84u8, 87u8,
-                        0u8, 80u8, 75u8, 0u8, 90u8, 65u8, 0u8, 71u8, 82u8, 0u8, 78u8, 80u8, 0u8,
-                        80u8, 72u8, 0u8, 66u8, 84u8, 0u8, 82u8, 85u8, 0u8, 85u8, 71u8, 0u8, 84u8,
-                        72u8, 0u8, 65u8, 90u8, 0u8, 77u8, 87u8, 0u8, 84u8, 86u8, 0u8, 78u8, 69u8,
-                        0u8, 67u8, 78u8, 0u8, 73u8, 78u8, 0u8, 80u8, 70u8, 0u8, 82u8, 85u8, 0u8,
-                        77u8, 65u8, 0u8, 82u8, 85u8, 0u8, 82u8, 85u8, 0u8, 67u8, 78u8, 0u8, 83u8,
-                        89u8, 0u8, 85u8, 65u8, 0u8, 70u8, 77u8, 0u8, 65u8, 79u8, 0u8, 73u8, 78u8,
-                        0u8, 73u8, 78u8, 0u8, 80u8, 75u8, 0u8, 85u8, 90u8, 0u8, 76u8, 82u8, 0u8,
-                        90u8, 65u8, 0u8, 73u8, 84u8, 0u8, 82u8, 85u8, 0u8, 86u8, 78u8, 0u8, 83u8,
-                        88u8, 0u8, 66u8, 69u8, 0u8, 68u8, 69u8, 0u8, 77u8, 90u8, 0u8, 48u8, 48u8,
-                        49u8, 82u8, 85u8, 0u8, 69u8, 69u8, 0u8, 84u8, 90u8, 0u8, 66u8, 69u8, 0u8,
-                        67u8, 72u8, 0u8, 69u8, 84u8, 0u8, 80u8, 72u8, 0u8, 65u8, 85u8, 0u8, 73u8,
-                        78u8, 0u8, 73u8, 78u8, 0u8, 87u8, 70u8, 0u8, 75u8, 77u8, 0u8, 83u8, 78u8,
-                        0u8, 73u8, 78u8, 0u8, 73u8, 78u8, 0u8, 67u8, 78u8, 0u8, 66u8, 82u8, 0u8,
-                        85u8, 90u8, 0u8, 84u8, 82u8, 0u8, 90u8, 65u8, 0u8, 84u8, 82u8, 0u8, 84u8,
-                        82u8, 0u8, 71u8, 69u8, 0u8, 67u8, 78u8, 0u8, 83u8, 68u8, 0u8, 83u8, 65u8,
-                        0u8, 73u8, 78u8, 0u8, 85u8, 71u8, 0u8, 73u8, 82u8, 0u8, 89u8, 69u8, 0u8,
-                        78u8, 80u8, 0u8, 77u8, 90u8, 0u8, 70u8, 77u8, 0u8, 67u8, 77u8, 0u8, 67u8,
-                        77u8, 0u8, 48u8, 48u8, 49u8, 78u8, 71u8, 0u8, 66u8, 82u8, 0u8, 77u8, 88u8,
-                        0u8, 72u8, 75u8, 0u8, 67u8, 78u8, 0u8, 83u8, 68u8, 0u8, 75u8, 77u8, 0u8,
-                        78u8, 76u8, 0u8, 77u8, 65u8, 0u8, 67u8, 78u8, 0u8, 67u8, 78u8, 0u8, 67u8,
-                        78u8, 0u8, 84u8, 71u8, 0u8, 77u8, 89u8, 0u8, 90u8, 65u8, 0u8, 84u8, 82u8,
-                        0u8,
-                    ])
-                },
-            )
-        },
-        ls2r: unsafe {
-            #[allow(unused_unsafe)]
-            ::zerovec::ZeroMap2d::from_parts_unchecked(
-                unsafe {
-                    ::zerovec::ZeroVec::from_bytes_unchecked(&[
-                        97u8, 114u8, 99u8, 97u8, 122u8, 0u8, 99u8, 117u8, 0u8, 101u8, 110u8, 0u8,
-                        102u8, 102u8, 0u8, 103u8, 114u8, 99u8, 107u8, 107u8, 0u8, 107u8, 117u8,
-                        0u8, 107u8, 121u8, 0u8, 108u8, 105u8, 102u8, 109u8, 97u8, 110u8, 109u8,
-                        110u8, 0u8, 112u8, 97u8, 0u8, 112u8, 97u8, 108u8, 115u8, 100u8, 0u8, 116u8,
-                        103u8, 0u8, 117u8, 103u8, 0u8, 117u8, 110u8, 114u8, 117u8, 122u8, 0u8,
-                        121u8, 117u8, 101u8, 122u8, 104u8, 0u8,
-                    ])
-                },
-                unsafe {
-                    ::zerovec::ZeroVec::from_bytes_unchecked(&[
-                        2u8, 0u8, 0u8, 0u8, 3u8, 0u8, 0u8, 0u8, 4u8, 0u8, 0u8, 0u8, 5u8, 0u8, 0u8,
-                        0u8, 6u8, 0u8, 0u8, 0u8, 7u8, 0u8, 0u8, 0u8, 8u8, 0u8, 0u8, 0u8, 10u8, 0u8,
-                        0u8, 0u8, 12u8, 0u8, 0u8, 0u8, 13u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8,
-                        15u8, 0u8, 0u8, 0u8, 16u8, 0u8, 0u8, 0u8, 17u8, 0u8, 0u8, 0u8, 20u8, 0u8,
-                        0u8, 0u8, 21u8, 0u8, 0u8, 0u8, 22u8, 0u8, 0u8, 0u8, 23u8, 0u8, 0u8, 0u8,
-                        24u8, 0u8, 0u8, 0u8, 25u8, 0u8, 0u8, 0u8, 28u8, 0u8, 0u8, 0u8,
-                    ])
-                },
-                unsafe {
-                    ::zerovec::ZeroVec::from_bytes_unchecked(&[
-                        78u8, 98u8, 97u8, 116u8, 80u8, 97u8, 108u8, 109u8, 65u8, 114u8, 97u8, 98u8,
-                        71u8, 108u8, 97u8, 103u8, 83u8, 104u8, 97u8, 119u8, 65u8, 100u8, 108u8,
-                        109u8, 76u8, 105u8, 110u8, 98u8, 65u8, 114u8, 97u8, 98u8, 65u8, 114u8,
-                        97u8, 98u8, 89u8, 101u8, 122u8, 105u8, 65u8, 114u8, 97u8, 98u8, 76u8, 97u8,
-                        116u8, 110u8, 76u8, 105u8, 109u8, 98u8, 78u8, 107u8, 111u8, 111u8, 77u8,
-                        111u8, 110u8, 103u8, 65u8, 114u8, 97u8, 98u8, 80u8, 104u8, 108u8, 112u8,
-                        68u8, 101u8, 118u8, 97u8, 75u8, 104u8, 111u8, 106u8, 83u8, 105u8, 110u8,
-                        100u8, 65u8, 114u8, 97u8, 98u8, 67u8, 121u8, 114u8, 108u8, 68u8, 101u8,
-                        118u8, 97u8, 65u8, 114u8, 97u8, 98u8, 72u8, 97u8, 110u8, 115u8, 66u8,
-                        111u8, 112u8, 111u8, 72u8, 97u8, 110u8, 98u8, 72u8, 97u8, 110u8, 116u8,
-                    ])
-                },
-                unsafe {
-                    ::zerovec::ZeroVec::from_bytes_unchecked(&[
-                        74u8, 79u8, 0u8, 83u8, 89u8, 0u8, 73u8, 82u8, 0u8, 66u8, 71u8, 0u8, 71u8,
-                        66u8, 0u8, 71u8, 78u8, 0u8, 71u8, 82u8, 0u8, 67u8, 78u8, 0u8, 73u8, 81u8,
-                        0u8, 71u8, 69u8, 0u8, 67u8, 78u8, 0u8, 84u8, 82u8, 0u8, 73u8, 78u8, 0u8,
-                        71u8, 78u8, 0u8, 67u8, 78u8, 0u8, 80u8, 75u8, 0u8, 67u8, 78u8, 0u8, 73u8,
-                        78u8, 0u8, 73u8, 78u8, 0u8, 73u8, 78u8, 0u8, 80u8, 75u8, 0u8, 75u8, 90u8,
-                        0u8, 78u8, 80u8, 0u8, 65u8, 70u8, 0u8, 67u8, 78u8, 0u8, 84u8, 87u8, 0u8,
-                        84u8, 87u8, 0u8, 84u8, 87u8, 0u8,
-                    ])
-                },
-            )
-        },
-    };
diff --git a/compiler/rustc_baked_icu_data/src/data/fallback/likelysubtags_v1/mod.rs b/compiler/rustc_baked_icu_data/src/data/fallback/likelysubtags_v1/mod.rs
new file mode 100644
index 000000000..57f7496dc
--- /dev/null
+++ b/compiler/rustc_baked_icu_data/src/data/fallback/likelysubtags_v1/mod.rs
@@ -0,0 +1,6 @@
+// @generated
+type DataStruct = < :: icu_provider_adapters :: fallback :: provider :: LocaleFallbackLikelySubtagsV1Marker as :: icu_provider :: DataMarker > :: Yokeable ;
+pub fn lookup(locale: &icu_provider::DataLocale) -> Option<&'static DataStruct> {
+    locale.is_empty().then(|| &UND)
+}
+static UND: DataStruct = include!("und.rs.data");
diff --git a/compiler/rustc_baked_icu_data/src/data/fallback/likelysubtags_v1/und.rs.data b/compiler/rustc_baked_icu_data/src/data/fallback/likelysubtags_v1/und.rs.data
new file mode 100644
index 000000000..4fd177834
--- /dev/null
+++ b/compiler/rustc_baked_icu_data/src/data/fallback/likelysubtags_v1/und.rs.data
@@ -0,0 +1,728 @@
+::icu_provider_adapters::fallback::provider::LocaleFallbackLikelySubtagsV1 {
+    l2s: unsafe {
+        #[allow(unused_unsafe)]
+        ::zerovec::ZeroMap::from_parts_unchecked(
+            unsafe {
+                ::zerovec::ZeroVec::from_bytes_unchecked(&[
+                    97u8, 98u8, 0u8, 97u8, 98u8, 113u8, 97u8, 100u8, 112u8, 97u8, 100u8, 121u8,
+                    97u8, 101u8, 0u8, 97u8, 101u8, 98u8, 97u8, 104u8, 111u8, 97u8, 106u8,
+                    116u8, 97u8, 107u8, 107u8, 97u8, 108u8, 116u8, 97u8, 109u8, 0u8, 97u8,
+                    112u8, 99u8, 97u8, 112u8, 100u8, 97u8, 114u8, 0u8, 97u8, 114u8, 99u8, 97u8,
+                    114u8, 113u8, 97u8, 114u8, 115u8, 97u8, 114u8, 121u8, 97u8, 114u8, 122u8,
+                    97u8, 115u8, 0u8, 97u8, 115u8, 101u8, 97u8, 118u8, 0u8, 97u8, 118u8, 108u8,
+                    97u8, 119u8, 97u8, 98u8, 97u8, 0u8, 98u8, 97u8, 108u8, 98u8, 97u8, 112u8,
+                    98u8, 97u8, 120u8, 98u8, 99u8, 113u8, 98u8, 101u8, 0u8, 98u8, 101u8, 106u8,
+                    98u8, 102u8, 113u8, 98u8, 102u8, 116u8, 98u8, 102u8, 121u8, 98u8, 103u8,
+                    0u8, 98u8, 103u8, 99u8, 98u8, 103u8, 110u8, 98u8, 103u8, 120u8, 98u8,
+                    104u8, 98u8, 98u8, 104u8, 105u8, 98u8, 104u8, 111u8, 98u8, 106u8, 105u8,
+                    98u8, 106u8, 106u8, 98u8, 108u8, 116u8, 98u8, 110u8, 0u8, 98u8, 111u8, 0u8,
+                    98u8, 112u8, 121u8, 98u8, 113u8, 105u8, 98u8, 114u8, 97u8, 98u8, 114u8,
+                    104u8, 98u8, 114u8, 120u8, 98u8, 115u8, 113u8, 98u8, 115u8, 116u8, 98u8,
+                    116u8, 118u8, 98u8, 117u8, 97u8, 98u8, 121u8, 110u8, 99u8, 99u8, 112u8,
+                    99u8, 101u8, 0u8, 99u8, 104u8, 109u8, 99u8, 104u8, 114u8, 99u8, 106u8,
+                    97u8, 99u8, 106u8, 109u8, 99u8, 107u8, 98u8, 99u8, 109u8, 103u8, 99u8,
+                    111u8, 112u8, 99u8, 114u8, 0u8, 99u8, 114u8, 104u8, 99u8, 114u8, 107u8,
+                    99u8, 114u8, 108u8, 99u8, 115u8, 119u8, 99u8, 116u8, 100u8, 99u8, 117u8,
+                    0u8, 99u8, 118u8, 0u8, 100u8, 97u8, 114u8, 100u8, 99u8, 99u8, 100u8, 103u8,
+                    108u8, 100u8, 109u8, 102u8, 100u8, 111u8, 105u8, 100u8, 114u8, 104u8,
+                    100u8, 114u8, 115u8, 100u8, 116u8, 121u8, 100u8, 118u8, 0u8, 100u8, 122u8,
+                    0u8, 101u8, 103u8, 121u8, 101u8, 107u8, 121u8, 101u8, 108u8, 0u8, 101u8,
+                    115u8, 103u8, 101u8, 116u8, 116u8, 102u8, 97u8, 0u8, 102u8, 105u8, 97u8,
+                    102u8, 117u8, 98u8, 103u8, 97u8, 110u8, 103u8, 98u8, 109u8, 103u8, 98u8,
+                    122u8, 103u8, 101u8, 122u8, 103u8, 103u8, 110u8, 103u8, 106u8, 107u8,
+                    103u8, 106u8, 117u8, 103u8, 108u8, 107u8, 103u8, 109u8, 118u8, 103u8,
+                    111u8, 102u8, 103u8, 111u8, 109u8, 103u8, 111u8, 110u8, 103u8, 111u8,
+                    116u8, 103u8, 114u8, 99u8, 103u8, 114u8, 116u8, 103u8, 117u8, 0u8, 103u8,
+                    118u8, 114u8, 103u8, 119u8, 99u8, 103u8, 119u8, 116u8, 104u8, 97u8, 107u8,
+                    104u8, 97u8, 122u8, 104u8, 100u8, 121u8, 104u8, 101u8, 0u8, 104u8, 105u8,
+                    0u8, 104u8, 108u8, 117u8, 104u8, 109u8, 100u8, 104u8, 110u8, 100u8, 104u8,
+                    110u8, 101u8, 104u8, 110u8, 106u8, 104u8, 110u8, 111u8, 104u8, 111u8, 99u8,
+                    104u8, 111u8, 106u8, 104u8, 115u8, 110u8, 104u8, 121u8, 0u8, 105u8, 105u8,
+                    0u8, 105u8, 110u8, 104u8, 105u8, 117u8, 0u8, 105u8, 119u8, 0u8, 106u8,
+                    97u8, 0u8, 106u8, 105u8, 0u8, 106u8, 109u8, 108u8, 107u8, 97u8, 0u8, 107u8,
+                    97u8, 97u8, 107u8, 97u8, 119u8, 107u8, 98u8, 100u8, 107u8, 98u8, 121u8,
+                    107u8, 100u8, 116u8, 107u8, 102u8, 114u8, 107u8, 102u8, 121u8, 107u8,
+                    104u8, 98u8, 107u8, 104u8, 110u8, 107u8, 104u8, 116u8, 107u8, 104u8, 119u8,
+                    107u8, 106u8, 103u8, 107u8, 107u8, 0u8, 107u8, 109u8, 0u8, 107u8, 110u8,
+                    0u8, 107u8, 111u8, 0u8, 107u8, 111u8, 105u8, 107u8, 111u8, 107u8, 107u8,
+                    113u8, 121u8, 107u8, 114u8, 99u8, 107u8, 114u8, 117u8, 107u8, 115u8, 0u8,
+                    107u8, 116u8, 98u8, 107u8, 117u8, 109u8, 107u8, 118u8, 0u8, 107u8, 118u8,
+                    120u8, 107u8, 120u8, 99u8, 107u8, 120u8, 108u8, 107u8, 120u8, 109u8, 107u8,
+                    120u8, 112u8, 107u8, 121u8, 0u8, 107u8, 122u8, 104u8, 108u8, 97u8, 98u8,
+                    108u8, 97u8, 100u8, 108u8, 97u8, 104u8, 108u8, 98u8, 101u8, 108u8, 99u8,
+                    112u8, 108u8, 101u8, 112u8, 108u8, 101u8, 122u8, 108u8, 105u8, 102u8,
+                    108u8, 105u8, 115u8, 108u8, 107u8, 105u8, 108u8, 109u8, 110u8, 108u8,
+                    111u8, 0u8, 108u8, 114u8, 99u8, 108u8, 117u8, 122u8, 108u8, 119u8, 108u8,
+                    108u8, 122u8, 104u8, 109u8, 97u8, 103u8, 109u8, 97u8, 105u8, 109u8, 100u8,
+                    101u8, 109u8, 100u8, 102u8, 109u8, 100u8, 120u8, 109u8, 102u8, 97u8, 109u8,
+                    103u8, 112u8, 109u8, 107u8, 0u8, 109u8, 107u8, 105u8, 109u8, 108u8, 0u8,
+                    109u8, 110u8, 0u8, 109u8, 110u8, 105u8, 109u8, 110u8, 119u8, 109u8, 114u8,
+                    0u8, 109u8, 114u8, 100u8, 109u8, 114u8, 106u8, 109u8, 114u8, 111u8, 109u8,
+                    116u8, 114u8, 109u8, 118u8, 121u8, 109u8, 119u8, 114u8, 109u8, 119u8,
+                    119u8, 109u8, 121u8, 0u8, 109u8, 121u8, 109u8, 109u8, 121u8, 118u8, 109u8,
+                    121u8, 122u8, 109u8, 122u8, 110u8, 110u8, 97u8, 110u8, 110u8, 101u8, 0u8,
+                    110u8, 101u8, 119u8, 110u8, 110u8, 112u8, 110u8, 111u8, 100u8, 110u8,
+                    111u8, 101u8, 110u8, 111u8, 110u8, 110u8, 113u8, 111u8, 110u8, 115u8,
+                    107u8, 110u8, 115u8, 116u8, 111u8, 106u8, 0u8, 111u8, 106u8, 115u8, 111u8,
+                    114u8, 0u8, 111u8, 114u8, 117u8, 111u8, 115u8, 0u8, 111u8, 115u8, 97u8,
+                    111u8, 116u8, 97u8, 111u8, 116u8, 107u8, 111u8, 117u8, 105u8, 112u8, 97u8,
+                    0u8, 112u8, 97u8, 108u8, 112u8, 101u8, 111u8, 112u8, 104u8, 108u8, 112u8,
+                    104u8, 110u8, 112u8, 107u8, 97u8, 112u8, 110u8, 116u8, 112u8, 112u8, 97u8,
+                    112u8, 114u8, 97u8, 112u8, 114u8, 100u8, 112u8, 115u8, 0u8, 114u8, 97u8,
+                    106u8, 114u8, 104u8, 103u8, 114u8, 105u8, 102u8, 114u8, 106u8, 115u8,
+                    114u8, 107u8, 116u8, 114u8, 109u8, 116u8, 114u8, 117u8, 0u8, 114u8, 117u8,
+                    101u8, 114u8, 121u8, 117u8, 115u8, 97u8, 0u8, 115u8, 97u8, 104u8, 115u8,
+                    97u8, 116u8, 115u8, 97u8, 122u8, 115u8, 99u8, 107u8, 115u8, 99u8, 108u8,
+                    115u8, 100u8, 0u8, 115u8, 100u8, 104u8, 115u8, 103u8, 97u8, 115u8, 103u8,
+                    119u8, 115u8, 104u8, 105u8, 115u8, 104u8, 110u8, 115u8, 104u8, 117u8,
+                    115u8, 105u8, 0u8, 115u8, 107u8, 114u8, 115u8, 109u8, 112u8, 115u8, 111u8,
+                    103u8, 115u8, 111u8, 117u8, 115u8, 114u8, 0u8, 115u8, 114u8, 98u8, 115u8,
+                    114u8, 120u8, 115u8, 119u8, 98u8, 115u8, 119u8, 118u8, 115u8, 121u8, 108u8,
+                    115u8, 121u8, 114u8, 116u8, 97u8, 0u8, 116u8, 97u8, 106u8, 116u8, 99u8,
+                    121u8, 116u8, 100u8, 100u8, 116u8, 100u8, 103u8, 116u8, 100u8, 104u8,
+                    116u8, 101u8, 0u8, 116u8, 103u8, 0u8, 116u8, 104u8, 0u8, 116u8, 104u8,
+                    108u8, 116u8, 104u8, 113u8, 116u8, 104u8, 114u8, 116u8, 105u8, 0u8, 116u8,
+                    105u8, 103u8, 116u8, 107u8, 116u8, 116u8, 114u8, 119u8, 116u8, 115u8,
+                    100u8, 116u8, 115u8, 102u8, 116u8, 115u8, 106u8, 116u8, 116u8, 0u8, 116u8,
+                    116u8, 115u8, 116u8, 120u8, 103u8, 116u8, 120u8, 111u8, 116u8, 121u8,
+                    118u8, 117u8, 100u8, 105u8, 117u8, 100u8, 109u8, 117u8, 103u8, 0u8, 117u8,
+                    103u8, 97u8, 117u8, 107u8, 0u8, 117u8, 110u8, 114u8, 117u8, 110u8, 120u8,
+                    117u8, 114u8, 0u8, 118u8, 97u8, 105u8, 119u8, 97u8, 108u8, 119u8, 98u8,
+                    113u8, 119u8, 98u8, 114u8, 119u8, 110u8, 105u8, 119u8, 115u8, 103u8, 119u8,
+                    116u8, 109u8, 119u8, 117u8, 117u8, 120u8, 99u8, 111u8, 120u8, 99u8, 114u8,
+                    120u8, 108u8, 99u8, 120u8, 108u8, 100u8, 120u8, 109u8, 102u8, 120u8, 109u8,
+                    110u8, 120u8, 109u8, 114u8, 120u8, 110u8, 97u8, 120u8, 110u8, 114u8, 120u8,
+                    112u8, 114u8, 120u8, 115u8, 97u8, 120u8, 115u8, 114u8, 121u8, 105u8, 0u8,
+                    121u8, 117u8, 101u8, 122u8, 100u8, 106u8, 122u8, 103u8, 104u8, 122u8,
+                    104u8, 0u8, 122u8, 104u8, 120u8, 122u8, 107u8, 116u8,
+                ])
+            },
+            unsafe {
+                ::zerovec::ZeroVec::from_bytes_unchecked(&[
+                    67u8, 121u8, 114u8, 108u8, 67u8, 121u8, 114u8, 108u8, 84u8, 105u8, 98u8,
+                    116u8, 67u8, 121u8, 114u8, 108u8, 65u8, 118u8, 115u8, 116u8, 65u8, 114u8,
+                    97u8, 98u8, 65u8, 104u8, 111u8, 109u8, 65u8, 114u8, 97u8, 98u8, 88u8,
+                    115u8, 117u8, 120u8, 67u8, 121u8, 114u8, 108u8, 69u8, 116u8, 104u8, 105u8,
+                    65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8,
+                    65u8, 114u8, 109u8, 105u8, 65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8,
+                    98u8, 65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 66u8, 101u8, 110u8,
+                    103u8, 83u8, 103u8, 110u8, 119u8, 67u8, 121u8, 114u8, 108u8, 65u8, 114u8,
+                    97u8, 98u8, 68u8, 101u8, 118u8, 97u8, 67u8, 121u8, 114u8, 108u8, 65u8,
+                    114u8, 97u8, 98u8, 68u8, 101u8, 118u8, 97u8, 66u8, 97u8, 109u8, 117u8,
+                    69u8, 116u8, 104u8, 105u8, 67u8, 121u8, 114u8, 108u8, 65u8, 114u8, 97u8,
+                    98u8, 84u8, 97u8, 109u8, 108u8, 65u8, 114u8, 97u8, 98u8, 68u8, 101u8,
+                    118u8, 97u8, 67u8, 121u8, 114u8, 108u8, 68u8, 101u8, 118u8, 97u8, 65u8,
+                    114u8, 97u8, 98u8, 71u8, 114u8, 101u8, 107u8, 68u8, 101u8, 118u8, 97u8,
+                    68u8, 101u8, 118u8, 97u8, 68u8, 101u8, 118u8, 97u8, 69u8, 116u8, 104u8,
+                    105u8, 68u8, 101u8, 118u8, 97u8, 84u8, 97u8, 118u8, 116u8, 66u8, 101u8,
+                    110u8, 103u8, 84u8, 105u8, 98u8, 116u8, 66u8, 101u8, 110u8, 103u8, 65u8,
+                    114u8, 97u8, 98u8, 68u8, 101u8, 118u8, 97u8, 65u8, 114u8, 97u8, 98u8, 68u8,
+                    101u8, 118u8, 97u8, 66u8, 97u8, 115u8, 115u8, 69u8, 116u8, 104u8, 105u8,
+                    68u8, 101u8, 118u8, 97u8, 67u8, 121u8, 114u8, 108u8, 69u8, 116u8, 104u8,
+                    105u8, 67u8, 97u8, 107u8, 109u8, 67u8, 121u8, 114u8, 108u8, 67u8, 121u8,
+                    114u8, 108u8, 67u8, 104u8, 101u8, 114u8, 65u8, 114u8, 97u8, 98u8, 67u8,
+                    104u8, 97u8, 109u8, 65u8, 114u8, 97u8, 98u8, 83u8, 111u8, 121u8, 111u8,
+                    67u8, 111u8, 112u8, 116u8, 67u8, 97u8, 110u8, 115u8, 67u8, 121u8, 114u8,
+                    108u8, 67u8, 97u8, 110u8, 115u8, 67u8, 97u8, 110u8, 115u8, 67u8, 97u8,
+                    110u8, 115u8, 80u8, 97u8, 117u8, 99u8, 67u8, 121u8, 114u8, 108u8, 67u8,
+                    121u8, 114u8, 108u8, 67u8, 121u8, 114u8, 108u8, 65u8, 114u8, 97u8, 98u8,
+                    65u8, 114u8, 97u8, 98u8, 77u8, 101u8, 100u8, 102u8, 68u8, 101u8, 118u8,
+                    97u8, 77u8, 111u8, 110u8, 103u8, 69u8, 116u8, 104u8, 105u8, 68u8, 101u8,
+                    118u8, 97u8, 84u8, 104u8, 97u8, 97u8, 84u8, 105u8, 98u8, 116u8, 69u8,
+                    103u8, 121u8, 112u8, 75u8, 97u8, 108u8, 105u8, 71u8, 114u8, 101u8, 107u8,
+                    71u8, 111u8, 110u8, 109u8, 73u8, 116u8, 97u8, 108u8, 65u8, 114u8, 97u8,
+                    98u8, 65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 72u8, 97u8, 110u8,
+                    115u8, 68u8, 101u8, 118u8, 97u8, 65u8, 114u8, 97u8, 98u8, 69u8, 116u8,
+                    104u8, 105u8, 68u8, 101u8, 118u8, 97u8, 65u8, 114u8, 97u8, 98u8, 65u8,
+                    114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 69u8, 116u8, 104u8, 105u8,
+                    69u8, 116u8, 104u8, 105u8, 68u8, 101u8, 118u8, 97u8, 84u8, 101u8, 108u8,
+                    117u8, 71u8, 111u8, 116u8, 104u8, 67u8, 112u8, 114u8, 116u8, 66u8, 101u8,
+                    110u8, 103u8, 71u8, 117u8, 106u8, 114u8, 68u8, 101u8, 118u8, 97u8, 65u8,
+                    114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 72u8, 97u8, 110u8, 115u8, 65u8,
+                    114u8, 97u8, 98u8, 69u8, 116u8, 104u8, 105u8, 72u8, 101u8, 98u8, 114u8,
+                    68u8, 101u8, 118u8, 97u8, 72u8, 108u8, 117u8, 119u8, 80u8, 108u8, 114u8,
+                    100u8, 65u8, 114u8, 97u8, 98u8, 68u8, 101u8, 118u8, 97u8, 72u8, 109u8,
+                    110u8, 112u8, 65u8, 114u8, 97u8, 98u8, 68u8, 101u8, 118u8, 97u8, 68u8,
+                    101u8, 118u8, 97u8, 72u8, 97u8, 110u8, 115u8, 65u8, 114u8, 109u8, 110u8,
+                    89u8, 105u8, 105u8, 105u8, 67u8, 121u8, 114u8, 108u8, 67u8, 97u8, 110u8,
+                    115u8, 72u8, 101u8, 98u8, 114u8, 74u8, 112u8, 97u8, 110u8, 72u8, 101u8,
+                    98u8, 114u8, 68u8, 101u8, 118u8, 97u8, 71u8, 101u8, 111u8, 114u8, 67u8,
+                    121u8, 114u8, 108u8, 75u8, 97u8, 119u8, 105u8, 67u8, 121u8, 114u8, 108u8,
+                    65u8, 114u8, 97u8, 98u8, 84u8, 104u8, 97u8, 105u8, 68u8, 101u8, 118u8,
+                    97u8, 68u8, 101u8, 118u8, 97u8, 84u8, 97u8, 108u8, 117u8, 68u8, 101u8,
+                    118u8, 97u8, 77u8, 121u8, 109u8, 114u8, 65u8, 114u8, 97u8, 98u8, 76u8,
+                    97u8, 111u8, 111u8, 67u8, 121u8, 114u8, 108u8, 75u8, 104u8, 109u8, 114u8,
+                    75u8, 110u8, 100u8, 97u8, 75u8, 111u8, 114u8, 101u8, 67u8, 121u8, 114u8,
+                    108u8, 68u8, 101u8, 118u8, 97u8, 69u8, 116u8, 104u8, 105u8, 67u8, 121u8,
+                    114u8, 108u8, 68u8, 101u8, 118u8, 97u8, 65u8, 114u8, 97u8, 98u8, 69u8,
+                    116u8, 104u8, 105u8, 67u8, 121u8, 114u8, 108u8, 67u8, 121u8, 114u8, 108u8,
+                    65u8, 114u8, 97u8, 98u8, 69u8, 116u8, 104u8, 105u8, 68u8, 101u8, 118u8,
+                    97u8, 84u8, 104u8, 97u8, 105u8, 65u8, 114u8, 97u8, 98u8, 67u8, 121u8,
+                    114u8, 108u8, 65u8, 114u8, 97u8, 98u8, 76u8, 105u8, 110u8, 97u8, 72u8,
+                    101u8, 98u8, 114u8, 65u8, 114u8, 97u8, 98u8, 67u8, 121u8, 114u8, 108u8,
+                    84u8, 104u8, 97u8, 105u8, 76u8, 101u8, 112u8, 99u8, 67u8, 121u8, 114u8,
+                    108u8, 68u8, 101u8, 118u8, 97u8, 76u8, 105u8, 115u8, 117u8, 65u8, 114u8,
+                    97u8, 98u8, 84u8, 101u8, 108u8, 117u8, 76u8, 97u8, 111u8, 111u8, 65u8,
+                    114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 84u8, 104u8, 97u8, 105u8, 72u8,
+                    97u8, 110u8, 115u8, 68u8, 101u8, 118u8, 97u8, 68u8, 101u8, 118u8, 97u8,
+                    65u8, 114u8, 97u8, 98u8, 67u8, 121u8, 114u8, 108u8, 69u8, 116u8, 104u8,
+                    105u8, 65u8, 114u8, 97u8, 98u8, 68u8, 101u8, 118u8, 97u8, 67u8, 121u8,
+                    114u8, 108u8, 65u8, 114u8, 97u8, 98u8, 77u8, 108u8, 121u8, 109u8, 67u8,
+                    121u8, 114u8, 108u8, 66u8, 101u8, 110u8, 103u8, 77u8, 121u8, 109u8, 114u8,
+                    68u8, 101u8, 118u8, 97u8, 68u8, 101u8, 118u8, 97u8, 67u8, 121u8, 114u8,
+                    108u8, 77u8, 114u8, 111u8, 111u8, 68u8, 101u8, 118u8, 97u8, 65u8, 114u8,
+                    97u8, 98u8, 68u8, 101u8, 118u8, 97u8, 72u8, 109u8, 110u8, 112u8, 77u8,
+                    121u8, 109u8, 114u8, 69u8, 116u8, 104u8, 105u8, 67u8, 121u8, 114u8, 108u8,
+                    77u8, 97u8, 110u8, 100u8, 65u8, 114u8, 97u8, 98u8, 72u8, 97u8, 110u8,
+                    115u8, 68u8, 101u8, 118u8, 97u8, 68u8, 101u8, 118u8, 97u8, 87u8, 99u8,
+                    104u8, 111u8, 76u8, 97u8, 110u8, 97u8, 68u8, 101u8, 118u8, 97u8, 82u8,
+                    117u8, 110u8, 114u8, 78u8, 107u8, 111u8, 111u8, 67u8, 97u8, 110u8, 115u8,
+                    84u8, 110u8, 115u8, 97u8, 67u8, 97u8, 110u8, 115u8, 67u8, 97u8, 110u8,
+                    115u8, 79u8, 114u8, 121u8, 97u8, 65u8, 114u8, 97u8, 98u8, 67u8, 121u8,
+                    114u8, 108u8, 79u8, 115u8, 103u8, 101u8, 65u8, 114u8, 97u8, 98u8, 79u8,
+                    114u8, 107u8, 104u8, 79u8, 117u8, 103u8, 114u8, 71u8, 117u8, 114u8, 117u8,
+                    80u8, 104u8, 108u8, 105u8, 88u8, 112u8, 101u8, 111u8, 65u8, 114u8, 97u8,
+                    98u8, 80u8, 104u8, 110u8, 120u8, 66u8, 114u8, 97u8, 104u8, 71u8, 114u8,
+                    101u8, 107u8, 68u8, 101u8, 118u8, 97u8, 75u8, 104u8, 97u8, 114u8, 65u8,
+                    114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 68u8, 101u8, 118u8, 97u8, 82u8,
+                    111u8, 104u8, 103u8, 84u8, 102u8, 110u8, 103u8, 68u8, 101u8, 118u8, 97u8,
+                    66u8, 101u8, 110u8, 103u8, 65u8, 114u8, 97u8, 98u8, 67u8, 121u8, 114u8,
+                    108u8, 67u8, 121u8, 114u8, 108u8, 75u8, 97u8, 110u8, 97u8, 68u8, 101u8,
+                    118u8, 97u8, 67u8, 121u8, 114u8, 108u8, 79u8, 108u8, 99u8, 107u8, 83u8,
+                    97u8, 117u8, 114u8, 68u8, 101u8, 118u8, 97u8, 65u8, 114u8, 97u8, 98u8,
+                    65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 79u8, 103u8, 97u8, 109u8,
+                    69u8, 116u8, 104u8, 105u8, 84u8, 102u8, 110u8, 103u8, 77u8, 121u8, 109u8,
+                    114u8, 65u8, 114u8, 97u8, 98u8, 83u8, 105u8, 110u8, 104u8, 65u8, 114u8,
+                    97u8, 98u8, 83u8, 97u8, 109u8, 114u8, 83u8, 111u8, 103u8, 100u8, 84u8,
+                    104u8, 97u8, 105u8, 67u8, 121u8, 114u8, 108u8, 83u8, 111u8, 114u8, 97u8,
+                    68u8, 101u8, 118u8, 97u8, 65u8, 114u8, 97u8, 98u8, 68u8, 101u8, 118u8,
+                    97u8, 66u8, 101u8, 110u8, 103u8, 83u8, 121u8, 114u8, 99u8, 84u8, 97u8,
+                    109u8, 108u8, 68u8, 101u8, 118u8, 97u8, 75u8, 110u8, 100u8, 97u8, 84u8,
+                    97u8, 108u8, 101u8, 68u8, 101u8, 118u8, 97u8, 68u8, 101u8, 118u8, 97u8,
+                    84u8, 101u8, 108u8, 117u8, 67u8, 121u8, 114u8, 108u8, 84u8, 104u8, 97u8,
+                    105u8, 68u8, 101u8, 118u8, 97u8, 68u8, 101u8, 118u8, 97u8, 68u8, 101u8,
+                    118u8, 97u8, 69u8, 116u8, 104u8, 105u8, 69u8, 116u8, 104u8, 105u8, 68u8,
+                    101u8, 118u8, 97u8, 65u8, 114u8, 97u8, 98u8, 71u8, 114u8, 101u8, 107u8,
+                    68u8, 101u8, 118u8, 97u8, 84u8, 105u8, 98u8, 116u8, 67u8, 121u8, 114u8,
+                    108u8, 84u8, 104u8, 97u8, 105u8, 84u8, 97u8, 110u8, 103u8, 84u8, 111u8,
+                    116u8, 111u8, 67u8, 121u8, 114u8, 108u8, 65u8, 103u8, 104u8, 98u8, 67u8,
+                    121u8, 114u8, 108u8, 65u8, 114u8, 97u8, 98u8, 85u8, 103u8, 97u8, 114u8,
+                    67u8, 121u8, 114u8, 108u8, 66u8, 101u8, 110u8, 103u8, 66u8, 101u8, 110u8,
+                    103u8, 65u8, 114u8, 97u8, 98u8, 86u8, 97u8, 105u8, 105u8, 69u8, 116u8,
+                    104u8, 105u8, 84u8, 101u8, 108u8, 117u8, 68u8, 101u8, 118u8, 97u8, 65u8,
+                    114u8, 97u8, 98u8, 71u8, 111u8, 110u8, 103u8, 68u8, 101u8, 118u8, 97u8,
+                    72u8, 97u8, 110u8, 115u8, 67u8, 104u8, 114u8, 115u8, 67u8, 97u8, 114u8,
+                    105u8, 76u8, 121u8, 99u8, 105u8, 76u8, 121u8, 100u8, 105u8, 71u8, 101u8,
+                    111u8, 114u8, 77u8, 97u8, 110u8, 105u8, 77u8, 101u8, 114u8, 99u8, 78u8,
+                    97u8, 114u8, 98u8, 68u8, 101u8, 118u8, 97u8, 80u8, 114u8, 116u8, 105u8,
+                    83u8, 97u8, 114u8, 98u8, 68u8, 101u8, 118u8, 97u8, 72u8, 101u8, 98u8,
+                    114u8, 72u8, 97u8, 110u8, 116u8, 65u8, 114u8, 97u8, 98u8, 84u8, 102u8,
+                    110u8, 103u8, 72u8, 97u8, 110u8, 115u8, 78u8, 115u8, 104u8, 117u8, 75u8,
+                    105u8, 116u8, 115u8,
+                ])
+            },
+        )
+    },
+    lr2s: unsafe {
+        #[allow(unused_unsafe)]
+        ::zerovec::ZeroMap2d::from_parts_unchecked(
+            unsafe {
+                ::zerovec::ZeroVec::from_bytes_unchecked(&[
+                    97u8, 122u8, 0u8, 104u8, 97u8, 0u8, 107u8, 107u8, 0u8, 107u8, 117u8, 0u8,
+                    107u8, 121u8, 0u8, 109u8, 97u8, 110u8, 109u8, 110u8, 0u8, 109u8, 115u8,
+                    0u8, 112u8, 97u8, 0u8, 114u8, 105u8, 102u8, 115u8, 100u8, 0u8, 115u8,
+                    114u8, 0u8, 116u8, 103u8, 0u8, 117u8, 103u8, 0u8, 117u8, 110u8, 114u8,
+                    117u8, 122u8, 0u8, 121u8, 117u8, 101u8, 122u8, 104u8, 0u8,
+                ])
+            },
+            unsafe {
+                ::zerovec::ZeroVec::from_bytes_unchecked(&[
+                    3u8, 0u8, 0u8, 0u8, 5u8, 0u8, 0u8, 0u8, 9u8, 0u8, 0u8, 0u8, 10u8, 0u8, 0u8,
+                    0u8, 12u8, 0u8, 0u8, 0u8, 13u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 15u8,
+                    0u8, 0u8, 0u8, 16u8, 0u8, 0u8, 0u8, 17u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8,
+                    0u8, 22u8, 0u8, 0u8, 0u8, 23u8, 0u8, 0u8, 0u8, 25u8, 0u8, 0u8, 0u8, 26u8,
+                    0u8, 0u8, 0u8, 28u8, 0u8, 0u8, 0u8, 29u8, 0u8, 0u8, 0u8, 44u8, 0u8, 0u8,
+                    0u8,
+                ])
+            },
+            unsafe {
+                ::zerovec::ZeroVec::from_bytes_unchecked(&[
+                    73u8, 81u8, 0u8, 73u8, 82u8, 0u8, 82u8, 85u8, 0u8, 67u8, 77u8, 0u8, 83u8,
+                    68u8, 0u8, 65u8, 70u8, 0u8, 67u8, 78u8, 0u8, 73u8, 82u8, 0u8, 77u8, 78u8,
+                    0u8, 76u8, 66u8, 0u8, 67u8, 78u8, 0u8, 84u8, 82u8, 0u8, 71u8, 78u8, 0u8,
+                    67u8, 78u8, 0u8, 67u8, 67u8, 0u8, 80u8, 75u8, 0u8, 78u8, 76u8, 0u8, 73u8,
+                    78u8, 0u8, 77u8, 69u8, 0u8, 82u8, 79u8, 0u8, 82u8, 85u8, 0u8, 84u8, 82u8,
+                    0u8, 80u8, 75u8, 0u8, 75u8, 90u8, 0u8, 77u8, 78u8, 0u8, 78u8, 80u8, 0u8,
+                    65u8, 70u8, 0u8, 67u8, 78u8, 0u8, 67u8, 78u8, 0u8, 65u8, 85u8, 0u8, 66u8,
+                    78u8, 0u8, 71u8, 66u8, 0u8, 71u8, 70u8, 0u8, 72u8, 75u8, 0u8, 73u8, 68u8,
+                    0u8, 77u8, 79u8, 0u8, 80u8, 65u8, 0u8, 80u8, 70u8, 0u8, 80u8, 72u8, 0u8,
+                    83u8, 82u8, 0u8, 84u8, 72u8, 0u8, 84u8, 87u8, 0u8, 85u8, 83u8, 0u8, 86u8,
+                    78u8, 0u8,
+                ])
+            },
+            unsafe {
+                ::zerovec::ZeroVec::from_bytes_unchecked(&[
+                    65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 67u8, 121u8, 114u8,
+                    108u8, 65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8,
+                    98u8, 65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8,
+                    98u8, 65u8, 114u8, 97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 76u8, 97u8, 116u8,
+                    110u8, 78u8, 107u8, 111u8, 111u8, 77u8, 111u8, 110u8, 103u8, 65u8, 114u8,
+                    97u8, 98u8, 65u8, 114u8, 97u8, 98u8, 76u8, 97u8, 116u8, 110u8, 68u8, 101u8,
+                    118u8, 97u8, 76u8, 97u8, 116u8, 110u8, 76u8, 97u8, 116u8, 110u8, 76u8,
+                    97u8, 116u8, 110u8, 76u8, 97u8, 116u8, 110u8, 65u8, 114u8, 97u8, 98u8,
+                    67u8, 121u8, 114u8, 108u8, 67u8, 121u8, 114u8, 108u8, 68u8, 101u8, 118u8,
+                    97u8, 65u8, 114u8, 97u8, 98u8, 67u8, 121u8, 114u8, 108u8, 72u8, 97u8,
+                    110u8, 115u8, 72u8, 97u8, 110u8, 116u8, 72u8, 97u8, 110u8, 116u8, 72u8,
+                    97u8, 110u8, 116u8, 72u8, 97u8, 110u8, 116u8, 72u8, 97u8, 110u8, 116u8,
+                    72u8, 97u8, 110u8, 116u8, 72u8, 97u8, 110u8, 116u8, 72u8, 97u8, 110u8,
+                    116u8, 72u8, 97u8, 110u8, 116u8, 72u8, 97u8, 110u8, 116u8, 72u8, 97u8,
+                    110u8, 116u8, 72u8, 97u8, 110u8, 116u8, 72u8, 97u8, 110u8, 116u8, 72u8,
+                    97u8, 110u8, 116u8, 72u8, 97u8, 110u8, 116u8,
+                ])
+            },
+        )
+    },
+    l2r: unsafe {
+        #[allow(unused_unsafe)]
+        ::zerovec::ZeroMap::from_parts_unchecked(
+            unsafe {
+                ::zerovec::ZeroVec::from_bytes_unchecked(&[
+                    97u8, 97u8, 0u8, 97u8, 98u8, 0u8, 97u8, 98u8, 114u8, 97u8, 99u8, 101u8,
+                    97u8, 99u8, 104u8, 97u8, 100u8, 97u8, 97u8, 100u8, 112u8, 97u8, 100u8,
+                    121u8, 97u8, 101u8, 0u8, 97u8, 101u8, 98u8, 97u8, 102u8, 0u8, 97u8, 103u8,
+                    113u8, 97u8, 104u8, 111u8, 97u8, 106u8, 116u8, 97u8, 107u8, 0u8, 97u8,
+                    107u8, 107u8, 97u8, 108u8, 110u8, 97u8, 108u8, 116u8, 97u8, 109u8, 0u8,
+                    97u8, 109u8, 111u8, 97u8, 110u8, 0u8, 97u8, 110u8, 110u8, 97u8, 111u8,
+                    122u8, 97u8, 112u8, 100u8, 97u8, 114u8, 0u8, 97u8, 114u8, 99u8, 97u8,
+                    114u8, 110u8, 97u8, 114u8, 111u8, 97u8, 114u8, 113u8, 97u8, 114u8, 115u8,
+                    97u8, 114u8, 121u8, 97u8, 114u8, 122u8, 97u8, 115u8, 0u8, 97u8, 115u8,
+                    97u8, 97u8, 115u8, 101u8, 97u8, 115u8, 116u8, 97u8, 116u8, 106u8, 97u8,
+                    118u8, 0u8, 97u8, 119u8, 97u8, 97u8, 121u8, 0u8, 97u8, 122u8, 0u8, 98u8,
+                    97u8, 0u8, 98u8, 97u8, 108u8, 98u8, 97u8, 110u8, 98u8, 97u8, 112u8, 98u8,
+                    97u8, 114u8, 98u8, 97u8, 115u8, 98u8, 97u8, 120u8, 98u8, 98u8, 99u8, 98u8,
+                    98u8, 106u8, 98u8, 99u8, 105u8, 98u8, 101u8, 0u8, 98u8, 101u8, 106u8, 98u8,
+                    101u8, 109u8, 98u8, 101u8, 119u8, 98u8, 101u8, 122u8, 98u8, 102u8, 100u8,
+                    98u8, 102u8, 113u8, 98u8, 102u8, 116u8, 98u8, 102u8, 121u8, 98u8, 103u8,
+                    0u8, 98u8, 103u8, 99u8, 98u8, 103u8, 110u8, 98u8, 103u8, 120u8, 98u8,
+                    104u8, 98u8, 98u8, 104u8, 105u8, 98u8, 104u8, 111u8, 98u8, 105u8, 0u8,
+                    98u8, 105u8, 107u8, 98u8, 105u8, 110u8, 98u8, 106u8, 106u8, 98u8, 106u8,
+                    110u8, 98u8, 106u8, 116u8, 98u8, 107u8, 109u8, 98u8, 107u8, 117u8, 98u8,
+                    108u8, 97u8, 98u8, 108u8, 103u8, 98u8, 108u8, 116u8, 98u8, 109u8, 0u8,
+                    98u8, 109u8, 113u8, 98u8, 110u8, 0u8, 98u8, 111u8, 0u8, 98u8, 112u8, 121u8,
+                    98u8, 113u8, 105u8, 98u8, 113u8, 118u8, 98u8, 114u8, 0u8, 98u8, 114u8,
+                    97u8, 98u8, 114u8, 104u8, 98u8, 114u8, 120u8, 98u8, 115u8, 0u8, 98u8,
+                    115u8, 113u8, 98u8, 115u8, 115u8, 98u8, 116u8, 111u8, 98u8, 116u8, 118u8,
+                    98u8, 117u8, 97u8, 98u8, 117u8, 99u8, 98u8, 117u8, 103u8, 98u8, 117u8,
+                    109u8, 98u8, 118u8, 98u8, 98u8, 121u8, 110u8, 98u8, 121u8, 118u8, 98u8,
+                    122u8, 101u8, 99u8, 97u8, 0u8, 99u8, 97u8, 100u8, 99u8, 99u8, 104u8, 99u8,
+                    99u8, 112u8, 99u8, 101u8, 0u8, 99u8, 101u8, 98u8, 99u8, 103u8, 103u8, 99u8,
+                    104u8, 0u8, 99u8, 104u8, 107u8, 99u8, 104u8, 109u8, 99u8, 104u8, 111u8,
+                    99u8, 104u8, 112u8, 99u8, 104u8, 114u8, 99u8, 105u8, 99u8, 99u8, 106u8,
+                    97u8, 99u8, 106u8, 109u8, 99u8, 107u8, 98u8, 99u8, 108u8, 99u8, 99u8,
+                    109u8, 103u8, 99u8, 111u8, 0u8, 99u8, 111u8, 112u8, 99u8, 112u8, 115u8,
+                    99u8, 114u8, 0u8, 99u8, 114u8, 103u8, 99u8, 114u8, 104u8, 99u8, 114u8,
+                    107u8, 99u8, 114u8, 108u8, 99u8, 114u8, 115u8, 99u8, 115u8, 0u8, 99u8,
+                    115u8, 98u8, 99u8, 115u8, 119u8, 99u8, 116u8, 100u8, 99u8, 117u8, 0u8,
+                    99u8, 118u8, 0u8, 99u8, 121u8, 0u8, 100u8, 97u8, 0u8, 100u8, 97u8, 102u8,
+                    100u8, 97u8, 107u8, 100u8, 97u8, 114u8, 100u8, 97u8, 118u8, 100u8, 99u8,
+                    99u8, 100u8, 101u8, 0u8, 100u8, 101u8, 110u8, 100u8, 103u8, 114u8, 100u8,
+                    106u8, 101u8, 100u8, 109u8, 102u8, 100u8, 110u8, 106u8, 100u8, 111u8,
+                    105u8, 100u8, 114u8, 104u8, 100u8, 115u8, 98u8, 100u8, 116u8, 109u8, 100u8,
+                    116u8, 112u8, 100u8, 116u8, 121u8, 100u8, 117u8, 97u8, 100u8, 118u8, 0u8,
+                    100u8, 121u8, 111u8, 100u8, 121u8, 117u8, 100u8, 122u8, 0u8, 101u8, 98u8,
+                    117u8, 101u8, 101u8, 0u8, 101u8, 102u8, 105u8, 101u8, 103u8, 108u8, 101u8,
+                    103u8, 121u8, 101u8, 107u8, 121u8, 101u8, 108u8, 0u8, 101u8, 110u8, 0u8,
+                    101u8, 111u8, 0u8, 101u8, 115u8, 0u8, 101u8, 115u8, 103u8, 101u8, 115u8,
+                    117u8, 101u8, 116u8, 0u8, 101u8, 116u8, 116u8, 101u8, 117u8, 0u8, 101u8,
+                    119u8, 111u8, 101u8, 120u8, 116u8, 102u8, 97u8, 0u8, 102u8, 97u8, 110u8,
+                    102u8, 102u8, 0u8, 102u8, 102u8, 109u8, 102u8, 105u8, 0u8, 102u8, 105u8,
+                    97u8, 102u8, 105u8, 108u8, 102u8, 105u8, 116u8, 102u8, 106u8, 0u8, 102u8,
+                    111u8, 0u8, 102u8, 111u8, 110u8, 102u8, 114u8, 0u8, 102u8, 114u8, 99u8,
+                    102u8, 114u8, 112u8, 102u8, 114u8, 114u8, 102u8, 114u8, 115u8, 102u8,
+                    117u8, 98u8, 102u8, 117u8, 100u8, 102u8, 117u8, 102u8, 102u8, 117u8, 113u8,
+                    102u8, 117u8, 114u8, 102u8, 117u8, 118u8, 102u8, 118u8, 114u8, 102u8,
+                    121u8, 0u8, 103u8, 97u8, 0u8, 103u8, 97u8, 97u8, 103u8, 97u8, 103u8, 103u8,
+                    97u8, 110u8, 103u8, 97u8, 121u8, 103u8, 98u8, 109u8, 103u8, 98u8, 122u8,
+                    103u8, 99u8, 114u8, 103u8, 100u8, 0u8, 103u8, 101u8, 122u8, 103u8, 103u8,
+                    110u8, 103u8, 105u8, 108u8, 103u8, 106u8, 107u8, 103u8, 106u8, 117u8,
+                    103u8, 108u8, 0u8, 103u8, 108u8, 107u8, 103u8, 110u8, 0u8, 103u8, 111u8,
+                    109u8, 103u8, 111u8, 110u8, 103u8, 111u8, 114u8, 103u8, 111u8, 115u8,
+                    103u8, 111u8, 116u8, 103u8, 114u8, 99u8, 103u8, 114u8, 116u8, 103u8, 115u8,
+                    119u8, 103u8, 117u8, 0u8, 103u8, 117u8, 98u8, 103u8, 117u8, 99u8, 103u8,
+                    117u8, 114u8, 103u8, 117u8, 122u8, 103u8, 118u8, 0u8, 103u8, 118u8, 114u8,
+                    103u8, 119u8, 105u8, 104u8, 97u8, 0u8, 104u8, 97u8, 107u8, 104u8, 97u8,
+                    119u8, 104u8, 97u8, 122u8, 104u8, 101u8, 0u8, 104u8, 105u8, 0u8, 104u8,
+                    105u8, 102u8, 104u8, 105u8, 108u8, 104u8, 108u8, 117u8, 104u8, 109u8,
+                    100u8, 104u8, 110u8, 100u8, 104u8, 110u8, 101u8, 104u8, 110u8, 106u8,
+                    104u8, 110u8, 110u8, 104u8, 110u8, 111u8, 104u8, 111u8, 0u8, 104u8, 111u8,
+                    99u8, 104u8, 111u8, 106u8, 104u8, 114u8, 0u8, 104u8, 115u8, 98u8, 104u8,
+                    115u8, 110u8, 104u8, 116u8, 0u8, 104u8, 117u8, 0u8, 104u8, 117u8, 114u8,
+                    104u8, 121u8, 0u8, 104u8, 122u8, 0u8, 105u8, 97u8, 0u8, 105u8, 98u8, 97u8,
+                    105u8, 98u8, 98u8, 105u8, 100u8, 0u8, 105u8, 102u8, 101u8, 105u8, 103u8,
+                    0u8, 105u8, 105u8, 0u8, 105u8, 107u8, 0u8, 105u8, 108u8, 111u8, 105u8,
+                    110u8, 0u8, 105u8, 110u8, 104u8, 105u8, 111u8, 0u8, 105u8, 115u8, 0u8,
+                    105u8, 116u8, 0u8, 105u8, 117u8, 0u8, 105u8, 119u8, 0u8, 105u8, 122u8,
+                    104u8, 106u8, 97u8, 0u8, 106u8, 97u8, 109u8, 106u8, 98u8, 111u8, 106u8,
+                    103u8, 111u8, 106u8, 105u8, 0u8, 106u8, 109u8, 99u8, 106u8, 109u8, 108u8,
+                    106u8, 117u8, 116u8, 106u8, 118u8, 0u8, 106u8, 119u8, 0u8, 107u8, 97u8,
+                    0u8, 107u8, 97u8, 97u8, 107u8, 97u8, 98u8, 107u8, 97u8, 99u8, 107u8, 97u8,
+                    106u8, 107u8, 97u8, 109u8, 107u8, 97u8, 111u8, 107u8, 97u8, 119u8, 107u8,
+                    98u8, 100u8, 107u8, 98u8, 121u8, 107u8, 99u8, 103u8, 107u8, 99u8, 107u8,
+                    107u8, 100u8, 101u8, 107u8, 100u8, 104u8, 107u8, 100u8, 116u8, 107u8,
+                    101u8, 97u8, 107u8, 101u8, 110u8, 107u8, 102u8, 111u8, 107u8, 102u8, 114u8,
+                    107u8, 102u8, 121u8, 107u8, 103u8, 0u8, 107u8, 103u8, 101u8, 107u8, 103u8,
+                    112u8, 107u8, 104u8, 97u8, 107u8, 104u8, 98u8, 107u8, 104u8, 110u8, 107u8,
+                    104u8, 113u8, 107u8, 104u8, 116u8, 107u8, 104u8, 119u8, 107u8, 105u8, 0u8,
+                    107u8, 105u8, 117u8, 107u8, 106u8, 0u8, 107u8, 106u8, 103u8, 107u8, 107u8,
+                    0u8, 107u8, 107u8, 106u8, 107u8, 108u8, 0u8, 107u8, 108u8, 110u8, 107u8,
+                    109u8, 0u8, 107u8, 109u8, 98u8, 107u8, 110u8, 0u8, 107u8, 110u8, 102u8,
+                    107u8, 111u8, 0u8, 107u8, 111u8, 105u8, 107u8, 111u8, 107u8, 107u8, 111u8,
+                    115u8, 107u8, 112u8, 101u8, 107u8, 114u8, 99u8, 107u8, 114u8, 105u8, 107u8,
+                    114u8, 106u8, 107u8, 114u8, 108u8, 107u8, 114u8, 117u8, 107u8, 115u8, 0u8,
+                    107u8, 115u8, 98u8, 107u8, 115u8, 102u8, 107u8, 115u8, 104u8, 107u8, 116u8,
+                    114u8, 107u8, 117u8, 0u8, 107u8, 117u8, 109u8, 107u8, 118u8, 0u8, 107u8,
+                    118u8, 114u8, 107u8, 118u8, 120u8, 107u8, 119u8, 0u8, 107u8, 119u8, 107u8,
+                    107u8, 120u8, 108u8, 107u8, 120u8, 109u8, 107u8, 120u8, 112u8, 107u8,
+                    121u8, 0u8, 107u8, 122u8, 106u8, 107u8, 122u8, 116u8, 108u8, 97u8, 0u8,
+                    108u8, 97u8, 98u8, 108u8, 97u8, 100u8, 108u8, 97u8, 103u8, 108u8, 97u8,
+                    104u8, 108u8, 97u8, 106u8, 108u8, 98u8, 0u8, 108u8, 98u8, 101u8, 108u8,
+                    98u8, 119u8, 108u8, 99u8, 112u8, 108u8, 101u8, 112u8, 108u8, 101u8, 122u8,
+                    108u8, 103u8, 0u8, 108u8, 105u8, 0u8, 108u8, 105u8, 102u8, 108u8, 105u8,
+                    106u8, 108u8, 105u8, 108u8, 108u8, 105u8, 115u8, 108u8, 106u8, 112u8,
+                    108u8, 107u8, 105u8, 108u8, 107u8, 116u8, 108u8, 109u8, 110u8, 108u8,
+                    109u8, 111u8, 108u8, 110u8, 0u8, 108u8, 111u8, 0u8, 108u8, 111u8, 108u8,
+                    108u8, 111u8, 122u8, 108u8, 114u8, 99u8, 108u8, 116u8, 0u8, 108u8, 116u8,
+                    103u8, 108u8, 117u8, 0u8, 108u8, 117u8, 97u8, 108u8, 117u8, 111u8, 108u8,
+                    117u8, 121u8, 108u8, 117u8, 122u8, 108u8, 118u8, 0u8, 108u8, 119u8, 108u8,
+                    108u8, 122u8, 104u8, 108u8, 122u8, 122u8, 109u8, 97u8, 100u8, 109u8, 97u8,
+                    102u8, 109u8, 97u8, 103u8, 109u8, 97u8, 105u8, 109u8, 97u8, 107u8, 109u8,
+                    97u8, 110u8, 109u8, 97u8, 115u8, 109u8, 97u8, 122u8, 109u8, 100u8, 102u8,
+                    109u8, 100u8, 104u8, 109u8, 100u8, 114u8, 109u8, 101u8, 110u8, 109u8,
+                    101u8, 114u8, 109u8, 102u8, 97u8, 109u8, 102u8, 101u8, 109u8, 103u8, 0u8,
+                    109u8, 103u8, 104u8, 109u8, 103u8, 111u8, 109u8, 103u8, 112u8, 109u8,
+                    103u8, 121u8, 109u8, 104u8, 0u8, 109u8, 105u8, 0u8, 109u8, 105u8, 99u8,
+                    109u8, 105u8, 110u8, 109u8, 107u8, 0u8, 109u8, 108u8, 0u8, 109u8, 108u8,
+                    115u8, 109u8, 110u8, 0u8, 109u8, 110u8, 105u8, 109u8, 110u8, 119u8, 109u8,
+                    111u8, 0u8, 109u8, 111u8, 101u8, 109u8, 111u8, 104u8, 109u8, 111u8, 115u8,
+                    109u8, 114u8, 0u8, 109u8, 114u8, 100u8, 109u8, 114u8, 106u8, 109u8, 114u8,
+                    111u8, 109u8, 115u8, 0u8, 109u8, 116u8, 0u8, 109u8, 116u8, 114u8, 109u8,
+                    117u8, 97u8, 109u8, 117u8, 115u8, 109u8, 118u8, 121u8, 109u8, 119u8, 107u8,
+                    109u8, 119u8, 114u8, 109u8, 119u8, 118u8, 109u8, 119u8, 119u8, 109u8,
+                    120u8, 99u8, 109u8, 121u8, 0u8, 109u8, 121u8, 118u8, 109u8, 121u8, 120u8,
+                    109u8, 121u8, 122u8, 109u8, 122u8, 110u8, 110u8, 97u8, 0u8, 110u8, 97u8,
+                    110u8, 110u8, 97u8, 112u8, 110u8, 97u8, 113u8, 110u8, 98u8, 0u8, 110u8,
+                    99u8, 104u8, 110u8, 100u8, 0u8, 110u8, 100u8, 99u8, 110u8, 100u8, 115u8,
+                    110u8, 101u8, 0u8, 110u8, 101u8, 119u8, 110u8, 103u8, 0u8, 110u8, 103u8,
+                    108u8, 110u8, 104u8, 101u8, 110u8, 104u8, 119u8, 110u8, 105u8, 106u8,
+                    110u8, 105u8, 117u8, 110u8, 106u8, 111u8, 110u8, 108u8, 0u8, 110u8, 109u8,
+                    103u8, 110u8, 110u8, 0u8, 110u8, 110u8, 104u8, 110u8, 110u8, 112u8, 110u8,
+                    111u8, 0u8, 110u8, 111u8, 100u8, 110u8, 111u8, 101u8, 110u8, 111u8, 110u8,
+                    110u8, 113u8, 111u8, 110u8, 114u8, 0u8, 110u8, 115u8, 107u8, 110u8, 115u8,
+                    111u8, 110u8, 115u8, 116u8, 110u8, 117u8, 115u8, 110u8, 118u8, 0u8, 110u8,
+                    120u8, 113u8, 110u8, 121u8, 0u8, 110u8, 121u8, 109u8, 110u8, 121u8, 110u8,
+                    110u8, 122u8, 105u8, 111u8, 99u8, 0u8, 111u8, 106u8, 0u8, 111u8, 106u8,
+                    115u8, 111u8, 107u8, 97u8, 111u8, 109u8, 0u8, 111u8, 114u8, 0u8, 111u8,
+                    115u8, 0u8, 111u8, 115u8, 97u8, 111u8, 116u8, 107u8, 111u8, 117u8, 105u8,
+                    112u8, 97u8, 0u8, 112u8, 97u8, 103u8, 112u8, 97u8, 108u8, 112u8, 97u8,
+                    109u8, 112u8, 97u8, 112u8, 112u8, 97u8, 117u8, 112u8, 99u8, 100u8, 112u8,
+                    99u8, 109u8, 112u8, 100u8, 99u8, 112u8, 100u8, 116u8, 112u8, 101u8, 111u8,
+                    112u8, 102u8, 108u8, 112u8, 104u8, 110u8, 112u8, 105u8, 115u8, 112u8,
+                    107u8, 97u8, 112u8, 107u8, 111u8, 112u8, 108u8, 0u8, 112u8, 109u8, 115u8,
+                    112u8, 110u8, 116u8, 112u8, 111u8, 110u8, 112u8, 112u8, 97u8, 112u8, 113u8,
+                    109u8, 112u8, 114u8, 97u8, 112u8, 114u8, 100u8, 112u8, 114u8, 103u8, 112u8,
+                    115u8, 0u8, 112u8, 116u8, 0u8, 112u8, 117u8, 117u8, 113u8, 117u8, 0u8,
+                    113u8, 117u8, 99u8, 113u8, 117u8, 103u8, 114u8, 97u8, 106u8, 114u8, 99u8,
+                    102u8, 114u8, 101u8, 106u8, 114u8, 103u8, 110u8, 114u8, 104u8, 103u8,
+                    114u8, 105u8, 97u8, 114u8, 105u8, 102u8, 114u8, 106u8, 115u8, 114u8, 107u8,
+                    116u8, 114u8, 109u8, 0u8, 114u8, 109u8, 102u8, 114u8, 109u8, 111u8, 114u8,
+                    109u8, 116u8, 114u8, 109u8, 117u8, 114u8, 110u8, 0u8, 114u8, 110u8, 103u8,
+                    114u8, 111u8, 0u8, 114u8, 111u8, 98u8, 114u8, 111u8, 102u8, 114u8, 116u8,
+                    109u8, 114u8, 117u8, 0u8, 114u8, 117u8, 101u8, 114u8, 117u8, 103u8, 114u8,
+                    119u8, 0u8, 114u8, 119u8, 107u8, 114u8, 121u8, 117u8, 115u8, 97u8, 0u8,
+                    115u8, 97u8, 102u8, 115u8, 97u8, 104u8, 115u8, 97u8, 113u8, 115u8, 97u8,
+                    115u8, 115u8, 97u8, 116u8, 115u8, 97u8, 118u8, 115u8, 97u8, 122u8, 115u8,
+                    98u8, 112u8, 115u8, 99u8, 0u8, 115u8, 99u8, 107u8, 115u8, 99u8, 110u8,
+                    115u8, 99u8, 111u8, 115u8, 100u8, 0u8, 115u8, 100u8, 99u8, 115u8, 100u8,
+                    104u8, 115u8, 101u8, 0u8, 115u8, 101u8, 102u8, 115u8, 101u8, 104u8, 115u8,
+                    101u8, 105u8, 115u8, 101u8, 115u8, 115u8, 103u8, 0u8, 115u8, 103u8, 97u8,
+                    115u8, 103u8, 115u8, 115u8, 104u8, 105u8, 115u8, 104u8, 110u8, 115u8,
+                    105u8, 0u8, 115u8, 105u8, 100u8, 115u8, 107u8, 0u8, 115u8, 107u8, 114u8,
+                    115u8, 108u8, 0u8, 115u8, 108u8, 105u8, 115u8, 108u8, 121u8, 115u8, 109u8,
+                    0u8, 115u8, 109u8, 97u8, 115u8, 109u8, 100u8, 115u8, 109u8, 106u8, 115u8,
+                    109u8, 110u8, 115u8, 109u8, 112u8, 115u8, 109u8, 115u8, 115u8, 110u8, 0u8,
+                    115u8, 110u8, 98u8, 115u8, 110u8, 107u8, 115u8, 111u8, 0u8, 115u8, 111u8,
+                    103u8, 115u8, 111u8, 117u8, 115u8, 113u8, 0u8, 115u8, 114u8, 0u8, 115u8,
+                    114u8, 98u8, 115u8, 114u8, 110u8, 115u8, 114u8, 114u8, 115u8, 114u8, 120u8,
+                    115u8, 115u8, 0u8, 115u8, 115u8, 121u8, 115u8, 116u8, 0u8, 115u8, 116u8,
+                    113u8, 115u8, 117u8, 0u8, 115u8, 117u8, 107u8, 115u8, 117u8, 115u8, 115u8,
+                    118u8, 0u8, 115u8, 119u8, 0u8, 115u8, 119u8, 98u8, 115u8, 119u8, 99u8,
+                    115u8, 119u8, 103u8, 115u8, 119u8, 118u8, 115u8, 120u8, 110u8, 115u8,
+                    121u8, 108u8, 115u8, 121u8, 114u8, 115u8, 122u8, 108u8, 116u8, 97u8, 0u8,
+                    116u8, 97u8, 106u8, 116u8, 98u8, 119u8, 116u8, 99u8, 121u8, 116u8, 100u8,
+                    100u8, 116u8, 100u8, 103u8, 116u8, 100u8, 104u8, 116u8, 100u8, 117u8,
+                    116u8, 101u8, 0u8, 116u8, 101u8, 109u8, 116u8, 101u8, 111u8, 116u8, 101u8,
+                    116u8, 116u8, 103u8, 0u8, 116u8, 104u8, 0u8, 116u8, 104u8, 108u8, 116u8,
+                    104u8, 113u8, 116u8, 104u8, 114u8, 116u8, 105u8, 0u8, 116u8, 105u8, 103u8,
+                    116u8, 105u8, 118u8, 116u8, 107u8, 0u8, 116u8, 107u8, 108u8, 116u8, 107u8,
+                    114u8, 116u8, 107u8, 116u8, 116u8, 108u8, 0u8, 116u8, 108u8, 121u8, 116u8,
+                    109u8, 104u8, 116u8, 110u8, 0u8, 116u8, 111u8, 0u8, 116u8, 111u8, 103u8,
+                    116u8, 111u8, 107u8, 116u8, 112u8, 105u8, 116u8, 114u8, 0u8, 116u8, 114u8,
+                    117u8, 116u8, 114u8, 118u8, 116u8, 114u8, 119u8, 116u8, 115u8, 0u8, 116u8,
+                    115u8, 100u8, 116u8, 115u8, 102u8, 116u8, 115u8, 103u8, 116u8, 115u8,
+                    106u8, 116u8, 116u8, 0u8, 116u8, 116u8, 106u8, 116u8, 116u8, 115u8, 116u8,
+                    116u8, 116u8, 116u8, 117u8, 109u8, 116u8, 118u8, 108u8, 116u8, 119u8,
+                    113u8, 116u8, 120u8, 103u8, 116u8, 120u8, 111u8, 116u8, 121u8, 0u8, 116u8,
+                    121u8, 118u8, 116u8, 122u8, 109u8, 117u8, 100u8, 105u8, 117u8, 100u8,
+                    109u8, 117u8, 103u8, 0u8, 117u8, 103u8, 97u8, 117u8, 107u8, 0u8, 117u8,
+                    108u8, 105u8, 117u8, 109u8, 98u8, 117u8, 110u8, 114u8, 117u8, 110u8, 120u8,
+                    117u8, 114u8, 0u8, 117u8, 122u8, 0u8, 118u8, 97u8, 105u8, 118u8, 101u8,
+                    0u8, 118u8, 101u8, 99u8, 118u8, 101u8, 112u8, 118u8, 105u8, 0u8, 118u8,
+                    105u8, 99u8, 118u8, 108u8, 115u8, 118u8, 109u8, 102u8, 118u8, 109u8, 119u8,
+                    118u8, 111u8, 0u8, 118u8, 111u8, 116u8, 118u8, 114u8, 111u8, 118u8, 117u8,
+                    110u8, 119u8, 97u8, 0u8, 119u8, 97u8, 101u8, 119u8, 97u8, 108u8, 119u8,
+                    97u8, 114u8, 119u8, 98u8, 112u8, 119u8, 98u8, 113u8, 119u8, 98u8, 114u8,
+                    119u8, 108u8, 115u8, 119u8, 110u8, 105u8, 119u8, 111u8, 0u8, 119u8, 115u8,
+                    103u8, 119u8, 116u8, 109u8, 119u8, 117u8, 117u8, 120u8, 97u8, 118u8, 120u8,
+                    99u8, 111u8, 120u8, 99u8, 114u8, 120u8, 104u8, 0u8, 120u8, 108u8, 99u8,
+                    120u8, 108u8, 100u8, 120u8, 109u8, 102u8, 120u8, 109u8, 110u8, 120u8,
+                    109u8, 114u8, 120u8, 110u8, 97u8, 120u8, 110u8, 114u8, 120u8, 111u8, 103u8,
+                    120u8, 112u8, 114u8, 120u8, 115u8, 97u8, 120u8, 115u8, 114u8, 121u8, 97u8,
+                    111u8, 121u8, 97u8, 112u8, 121u8, 97u8, 118u8, 121u8, 98u8, 98u8, 121u8,
+                    105u8, 0u8, 121u8, 111u8, 0u8, 121u8, 114u8, 108u8, 121u8, 117u8, 97u8,
+                    121u8, 117u8, 101u8, 122u8, 97u8, 0u8, 122u8, 97u8, 103u8, 122u8, 100u8,
+                    106u8, 122u8, 101u8, 97u8, 122u8, 103u8, 104u8, 122u8, 104u8, 0u8, 122u8,
+                    104u8, 120u8, 122u8, 107u8, 116u8, 122u8, 108u8, 109u8, 122u8, 109u8,
+                    105u8, 122u8, 117u8, 0u8, 122u8, 122u8, 97u8,
+                ])
+            },
+            unsafe {
+                ::zerovec::ZeroVec::from_bytes_unchecked(&[
+                    69u8, 84u8, 0u8, 71u8, 69u8, 0u8, 71u8, 72u8, 0u8, 73u8, 68u8, 0u8, 85u8,
+                    71u8, 0u8, 71u8, 72u8, 0u8, 66u8, 84u8, 0u8, 82u8, 85u8, 0u8, 73u8, 82u8,
+                    0u8, 84u8, 78u8, 0u8, 90u8, 65u8, 0u8, 67u8, 77u8, 0u8, 73u8, 78u8, 0u8,
+                    84u8, 78u8, 0u8, 71u8, 72u8, 0u8, 73u8, 81u8, 0u8, 88u8, 75u8, 0u8, 82u8,
+                    85u8, 0u8, 69u8, 84u8, 0u8, 78u8, 71u8, 0u8, 69u8, 83u8, 0u8, 78u8, 71u8,
+                    0u8, 73u8, 68u8, 0u8, 84u8, 71u8, 0u8, 69u8, 71u8, 0u8, 73u8, 82u8, 0u8,
+                    67u8, 76u8, 0u8, 66u8, 79u8, 0u8, 68u8, 90u8, 0u8, 83u8, 65u8, 0u8, 77u8,
+                    65u8, 0u8, 69u8, 71u8, 0u8, 73u8, 78u8, 0u8, 84u8, 90u8, 0u8, 85u8, 83u8,
+                    0u8, 69u8, 83u8, 0u8, 67u8, 65u8, 0u8, 82u8, 85u8, 0u8, 73u8, 78u8, 0u8,
+                    66u8, 79u8, 0u8, 65u8, 90u8, 0u8, 82u8, 85u8, 0u8, 80u8, 75u8, 0u8, 73u8,
+                    68u8, 0u8, 78u8, 80u8, 0u8, 65u8, 84u8, 0u8, 67u8, 77u8, 0u8, 67u8, 77u8,
+                    0u8, 73u8, 68u8, 0u8, 67u8, 77u8, 0u8, 67u8, 73u8, 0u8, 66u8, 89u8, 0u8,
+                    83u8, 68u8, 0u8, 90u8, 77u8, 0u8, 73u8, 68u8, 0u8, 84u8, 90u8, 0u8, 67u8,
+                    77u8, 0u8, 73u8, 78u8, 0u8, 80u8, 75u8, 0u8, 73u8, 78u8, 0u8, 66u8, 71u8,
+                    0u8, 73u8, 78u8, 0u8, 80u8, 75u8, 0u8, 84u8, 82u8, 0u8, 73u8, 78u8, 0u8,
+                    73u8, 78u8, 0u8, 73u8, 78u8, 0u8, 86u8, 85u8, 0u8, 80u8, 72u8, 0u8, 78u8,
+                    71u8, 0u8, 73u8, 78u8, 0u8, 73u8, 68u8, 0u8, 83u8, 78u8, 0u8, 67u8, 77u8,
+                    0u8, 80u8, 72u8, 0u8, 67u8, 65u8, 0u8, 77u8, 89u8, 0u8, 86u8, 78u8, 0u8,
+                    77u8, 76u8, 0u8, 77u8, 76u8, 0u8, 66u8, 68u8, 0u8, 67u8, 78u8, 0u8, 73u8,
+                    78u8, 0u8, 73u8, 82u8, 0u8, 67u8, 73u8, 0u8, 70u8, 82u8, 0u8, 73u8, 78u8,
+                    0u8, 80u8, 75u8, 0u8, 73u8, 78u8, 0u8, 66u8, 65u8, 0u8, 76u8, 82u8, 0u8,
+                    67u8, 77u8, 0u8, 80u8, 72u8, 0u8, 80u8, 75u8, 0u8, 82u8, 85u8, 0u8, 89u8,
+                    84u8, 0u8, 73u8, 68u8, 0u8, 67u8, 77u8, 0u8, 71u8, 81u8, 0u8, 69u8, 82u8,
+                    0u8, 67u8, 77u8, 0u8, 77u8, 76u8, 0u8, 69u8, 83u8, 0u8, 85u8, 83u8, 0u8,
+                    78u8, 71u8, 0u8, 66u8, 68u8, 0u8, 82u8, 85u8, 0u8, 80u8, 72u8, 0u8, 85u8,
+                    71u8, 0u8, 71u8, 85u8, 0u8, 70u8, 77u8, 0u8, 82u8, 85u8, 0u8, 85u8, 83u8,
+                    0u8, 67u8, 65u8, 0u8, 85u8, 83u8, 0u8, 85u8, 83u8, 0u8, 75u8, 72u8, 0u8,
+                    86u8, 78u8, 0u8, 73u8, 81u8, 0u8, 67u8, 65u8, 0u8, 77u8, 78u8, 0u8, 70u8,
+                    82u8, 0u8, 69u8, 71u8, 0u8, 80u8, 72u8, 0u8, 67u8, 65u8, 0u8, 67u8, 65u8,
+                    0u8, 85u8, 65u8, 0u8, 67u8, 65u8, 0u8, 67u8, 65u8, 0u8, 83u8, 67u8, 0u8,
+                    67u8, 90u8, 0u8, 80u8, 76u8, 0u8, 67u8, 65u8, 0u8, 77u8, 77u8, 0u8, 82u8,
+                    85u8, 0u8, 82u8, 85u8, 0u8, 71u8, 66u8, 0u8, 68u8, 75u8, 0u8, 67u8, 73u8,
+                    0u8, 85u8, 83u8, 0u8, 82u8, 85u8, 0u8, 75u8, 69u8, 0u8, 73u8, 78u8, 0u8,
+                    68u8, 69u8, 0u8, 67u8, 65u8, 0u8, 67u8, 65u8, 0u8, 78u8, 69u8, 0u8, 78u8,
+                    71u8, 0u8, 67u8, 73u8, 0u8, 73u8, 78u8, 0u8, 67u8, 78u8, 0u8, 68u8, 69u8,
+                    0u8, 77u8, 76u8, 0u8, 77u8, 89u8, 0u8, 78u8, 80u8, 0u8, 67u8, 77u8, 0u8,
+                    77u8, 86u8, 0u8, 83u8, 78u8, 0u8, 66u8, 70u8, 0u8, 66u8, 84u8, 0u8, 75u8,
+                    69u8, 0u8, 71u8, 72u8, 0u8, 78u8, 71u8, 0u8, 73u8, 84u8, 0u8, 69u8, 71u8,
+                    0u8, 77u8, 77u8, 0u8, 71u8, 82u8, 0u8, 85u8, 83u8, 0u8, 48u8, 48u8, 49u8,
+                    69u8, 83u8, 0u8, 73u8, 78u8, 0u8, 85u8, 83u8, 0u8, 69u8, 69u8, 0u8, 73u8,
+                    84u8, 0u8, 69u8, 83u8, 0u8, 67u8, 77u8, 0u8, 69u8, 83u8, 0u8, 73u8, 82u8,
+                    0u8, 71u8, 81u8, 0u8, 83u8, 78u8, 0u8, 77u8, 76u8, 0u8, 70u8, 73u8, 0u8,
+                    83u8, 68u8, 0u8, 80u8, 72u8, 0u8, 83u8, 69u8, 0u8, 70u8, 74u8, 0u8, 70u8,
+                    79u8, 0u8, 66u8, 74u8, 0u8, 70u8, 82u8, 0u8, 85u8, 83u8, 0u8, 70u8, 82u8,
+                    0u8, 68u8, 69u8, 0u8, 68u8, 69u8, 0u8, 67u8, 77u8, 0u8, 87u8, 70u8, 0u8,
+                    71u8, 78u8, 0u8, 78u8, 69u8, 0u8, 73u8, 84u8, 0u8, 78u8, 71u8, 0u8, 83u8,
+                    68u8, 0u8, 78u8, 76u8, 0u8, 73u8, 69u8, 0u8, 71u8, 72u8, 0u8, 77u8, 68u8,
+                    0u8, 67u8, 78u8, 0u8, 73u8, 68u8, 0u8, 73u8, 78u8, 0u8, 73u8, 82u8, 0u8,
+                    71u8, 70u8, 0u8, 71u8, 66u8, 0u8, 69u8, 84u8, 0u8, 78u8, 80u8, 0u8, 75u8,
+                    73u8, 0u8, 80u8, 75u8, 0u8, 80u8, 75u8, 0u8, 69u8, 83u8, 0u8, 73u8, 82u8,
+                    0u8, 80u8, 89u8, 0u8, 73u8, 78u8, 0u8, 73u8, 78u8, 0u8, 73u8, 68u8, 0u8,
+                    78u8, 76u8, 0u8, 85u8, 65u8, 0u8, 67u8, 89u8, 0u8, 73u8, 78u8, 0u8, 67u8,
+                    72u8, 0u8, 73u8, 78u8, 0u8, 66u8, 82u8, 0u8, 67u8, 79u8, 0u8, 71u8, 72u8,
+                    0u8, 75u8, 69u8, 0u8, 73u8, 77u8, 0u8, 78u8, 80u8, 0u8, 67u8, 65u8, 0u8,
+                    78u8, 71u8, 0u8, 67u8, 78u8, 0u8, 85u8, 83u8, 0u8, 65u8, 70u8, 0u8, 73u8,
+                    76u8, 0u8, 73u8, 78u8, 0u8, 70u8, 74u8, 0u8, 80u8, 72u8, 0u8, 84u8, 82u8,
+                    0u8, 67u8, 78u8, 0u8, 80u8, 75u8, 0u8, 73u8, 78u8, 0u8, 85u8, 83u8, 0u8,
+                    80u8, 72u8, 0u8, 80u8, 75u8, 0u8, 80u8, 71u8, 0u8, 73u8, 78u8, 0u8, 73u8,
+                    78u8, 0u8, 72u8, 82u8, 0u8, 68u8, 69u8, 0u8, 67u8, 78u8, 0u8, 72u8, 84u8,
+                    0u8, 72u8, 85u8, 0u8, 67u8, 65u8, 0u8, 65u8, 77u8, 0u8, 78u8, 65u8, 0u8,
+                    48u8, 48u8, 49u8, 77u8, 89u8, 0u8, 78u8, 71u8, 0u8, 73u8, 68u8, 0u8, 84u8,
+                    71u8, 0u8, 78u8, 71u8, 0u8, 67u8, 78u8, 0u8, 85u8, 83u8, 0u8, 80u8, 72u8,
+                    0u8, 73u8, 68u8, 0u8, 82u8, 85u8, 0u8, 48u8, 48u8, 49u8, 73u8, 83u8, 0u8,
+                    73u8, 84u8, 0u8, 67u8, 65u8, 0u8, 73u8, 76u8, 0u8, 82u8, 85u8, 0u8, 74u8,
+                    80u8, 0u8, 74u8, 77u8, 0u8, 48u8, 48u8, 49u8, 67u8, 77u8, 0u8, 85u8, 65u8,
+                    0u8, 84u8, 90u8, 0u8, 78u8, 80u8, 0u8, 68u8, 75u8, 0u8, 73u8, 68u8, 0u8,
+                    73u8, 68u8, 0u8, 71u8, 69u8, 0u8, 85u8, 90u8, 0u8, 68u8, 90u8, 0u8, 77u8,
+                    77u8, 0u8, 78u8, 71u8, 0u8, 75u8, 69u8, 0u8, 77u8, 76u8, 0u8, 73u8, 68u8,
+                    0u8, 82u8, 85u8, 0u8, 78u8, 69u8, 0u8, 78u8, 71u8, 0u8, 90u8, 87u8, 0u8,
+                    84u8, 90u8, 0u8, 84u8, 71u8, 0u8, 84u8, 72u8, 0u8, 67u8, 86u8, 0u8, 67u8,
+                    77u8, 0u8, 67u8, 73u8, 0u8, 73u8, 78u8, 0u8, 73u8, 78u8, 0u8, 67u8, 68u8,
+                    0u8, 73u8, 68u8, 0u8, 66u8, 82u8, 0u8, 73u8, 78u8, 0u8, 67u8, 78u8, 0u8,
+                    73u8, 78u8, 0u8, 77u8, 76u8, 0u8, 73u8, 78u8, 0u8, 80u8, 75u8, 0u8, 75u8,
+                    69u8, 0u8, 84u8, 82u8, 0u8, 78u8, 65u8, 0u8, 76u8, 65u8, 0u8, 75u8, 90u8,
+                    0u8, 67u8, 77u8, 0u8, 71u8, 76u8, 0u8, 75u8, 69u8, 0u8, 75u8, 72u8, 0u8,
+                    65u8, 79u8, 0u8, 73u8, 78u8, 0u8, 71u8, 87u8, 0u8, 75u8, 82u8, 0u8, 82u8,
+                    85u8, 0u8, 73u8, 78u8, 0u8, 70u8, 77u8, 0u8, 76u8, 82u8, 0u8, 82u8, 85u8,
+                    0u8, 83u8, 76u8, 0u8, 80u8, 72u8, 0u8, 82u8, 85u8, 0u8, 73u8, 78u8, 0u8,
+                    73u8, 78u8, 0u8, 84u8, 90u8, 0u8, 67u8, 77u8, 0u8, 68u8, 69u8, 0u8, 77u8,
+                    89u8, 0u8, 84u8, 82u8, 0u8, 82u8, 85u8, 0u8, 82u8, 85u8, 0u8, 73u8, 68u8,
+                    0u8, 80u8, 75u8, 0u8, 71u8, 66u8, 0u8, 67u8, 65u8, 0u8, 73u8, 78u8, 0u8,
+                    84u8, 72u8, 0u8, 80u8, 75u8, 0u8, 75u8, 71u8, 0u8, 77u8, 89u8, 0u8, 77u8,
+                    89u8, 0u8, 86u8, 65u8, 0u8, 71u8, 82u8, 0u8, 73u8, 76u8, 0u8, 84u8, 90u8,
+                    0u8, 80u8, 75u8, 0u8, 85u8, 71u8, 0u8, 76u8, 85u8, 0u8, 82u8, 85u8, 0u8,
+                    73u8, 68u8, 0u8, 67u8, 78u8, 0u8, 73u8, 78u8, 0u8, 82u8, 85u8, 0u8, 85u8,
+                    71u8, 0u8, 78u8, 76u8, 0u8, 78u8, 80u8, 0u8, 73u8, 84u8, 0u8, 67u8, 65u8,
+                    0u8, 67u8, 78u8, 0u8, 73u8, 68u8, 0u8, 73u8, 82u8, 0u8, 85u8, 83u8, 0u8,
+                    73u8, 78u8, 0u8, 73u8, 84u8, 0u8, 67u8, 68u8, 0u8, 76u8, 65u8, 0u8, 67u8,
+                    68u8, 0u8, 90u8, 77u8, 0u8, 73u8, 82u8, 0u8, 76u8, 84u8, 0u8, 76u8, 86u8,
+                    0u8, 67u8, 68u8, 0u8, 67u8, 68u8, 0u8, 75u8, 69u8, 0u8, 75u8, 69u8, 0u8,
+                    73u8, 82u8, 0u8, 76u8, 86u8, 0u8, 84u8, 72u8, 0u8, 67u8, 78u8, 0u8, 84u8,
+                    82u8, 0u8, 73u8, 68u8, 0u8, 67u8, 77u8, 0u8, 73u8, 78u8, 0u8, 73u8, 78u8,
+                    0u8, 73u8, 68u8, 0u8, 71u8, 77u8, 0u8, 75u8, 69u8, 0u8, 77u8, 88u8, 0u8,
+                    82u8, 85u8, 0u8, 80u8, 72u8, 0u8, 73u8, 68u8, 0u8, 83u8, 76u8, 0u8, 75u8,
+                    69u8, 0u8, 84u8, 72u8, 0u8, 77u8, 85u8, 0u8, 77u8, 71u8, 0u8, 77u8, 90u8,
+                    0u8, 67u8, 77u8, 0u8, 78u8, 80u8, 0u8, 84u8, 90u8, 0u8, 77u8, 72u8, 0u8,
+                    78u8, 90u8, 0u8, 67u8, 65u8, 0u8, 73u8, 68u8, 0u8, 77u8, 75u8, 0u8, 73u8,
+                    78u8, 0u8, 83u8, 68u8, 0u8, 77u8, 78u8, 0u8, 73u8, 78u8, 0u8, 77u8, 77u8,
+                    0u8, 82u8, 79u8, 0u8, 67u8, 65u8, 0u8, 67u8, 65u8, 0u8, 66u8, 70u8, 0u8,
+                    73u8, 78u8, 0u8, 78u8, 80u8, 0u8, 82u8, 85u8, 0u8, 66u8, 68u8, 0u8, 77u8,
+                    89u8, 0u8, 77u8, 84u8, 0u8, 73u8, 78u8, 0u8, 67u8, 77u8, 0u8, 85u8, 83u8,
+                    0u8, 80u8, 75u8, 0u8, 77u8, 76u8, 0u8, 73u8, 78u8, 0u8, 73u8, 68u8, 0u8,
+                    85u8, 83u8, 0u8, 90u8, 87u8, 0u8, 77u8, 77u8, 0u8, 82u8, 85u8, 0u8, 85u8,
+                    71u8, 0u8, 73u8, 82u8, 0u8, 73u8, 82u8, 0u8, 78u8, 82u8, 0u8, 67u8, 78u8,
+                    0u8, 73u8, 84u8, 0u8, 78u8, 65u8, 0u8, 78u8, 79u8, 0u8, 77u8, 88u8, 0u8,
+                    90u8, 87u8, 0u8, 77u8, 90u8, 0u8, 68u8, 69u8, 0u8, 78u8, 80u8, 0u8, 78u8,
+                    80u8, 0u8, 78u8, 65u8, 0u8, 77u8, 90u8, 0u8, 77u8, 88u8, 0u8, 77u8, 88u8,
+                    0u8, 73u8, 68u8, 0u8, 78u8, 85u8, 0u8, 73u8, 78u8, 0u8, 78u8, 76u8, 0u8,
+                    67u8, 77u8, 0u8, 78u8, 79u8, 0u8, 67u8, 77u8, 0u8, 73u8, 78u8, 0u8, 78u8,
+                    79u8, 0u8, 84u8, 72u8, 0u8, 73u8, 78u8, 0u8, 83u8, 69u8, 0u8, 71u8, 78u8,
+                    0u8, 90u8, 65u8, 0u8, 67u8, 65u8, 0u8, 90u8, 65u8, 0u8, 73u8, 78u8, 0u8,
+                    83u8, 83u8, 0u8, 85u8, 83u8, 0u8, 67u8, 78u8, 0u8, 77u8, 87u8, 0u8, 84u8,
+                    90u8, 0u8, 85u8, 71u8, 0u8, 71u8, 72u8, 0u8, 70u8, 82u8, 0u8, 67u8, 65u8,
+                    0u8, 67u8, 65u8, 0u8, 67u8, 65u8, 0u8, 69u8, 84u8, 0u8, 73u8, 78u8, 0u8,
+                    71u8, 69u8, 0u8, 85u8, 83u8, 0u8, 77u8, 78u8, 0u8, 49u8, 52u8, 51u8, 73u8,
+                    78u8, 0u8, 80u8, 72u8, 0u8, 73u8, 82u8, 0u8, 80u8, 72u8, 0u8, 65u8, 87u8,
+                    0u8, 80u8, 87u8, 0u8, 70u8, 82u8, 0u8, 78u8, 71u8, 0u8, 85u8, 83u8, 0u8,
+                    67u8, 65u8, 0u8, 73u8, 82u8, 0u8, 68u8, 69u8, 0u8, 76u8, 66u8, 0u8, 83u8,
+                    66u8, 0u8, 73u8, 78u8, 0u8, 75u8, 69u8, 0u8, 80u8, 76u8, 0u8, 73u8, 84u8,
+                    0u8, 71u8, 82u8, 0u8, 70u8, 77u8, 0u8, 73u8, 78u8, 0u8, 67u8, 65u8, 0u8,
+                    80u8, 75u8, 0u8, 73u8, 82u8, 0u8, 48u8, 48u8, 49u8, 65u8, 70u8, 0u8, 66u8,
+                    82u8, 0u8, 71u8, 65u8, 0u8, 80u8, 69u8, 0u8, 71u8, 84u8, 0u8, 69u8, 67u8,
+                    0u8, 73u8, 78u8, 0u8, 82u8, 69u8, 0u8, 73u8, 68u8, 0u8, 73u8, 84u8, 0u8,
+                    77u8, 77u8, 0u8, 73u8, 78u8, 0u8, 77u8, 65u8, 0u8, 78u8, 80u8, 0u8, 66u8,
+                    68u8, 0u8, 67u8, 72u8, 0u8, 70u8, 73u8, 0u8, 67u8, 72u8, 0u8, 73u8, 82u8,
+                    0u8, 83u8, 69u8, 0u8, 66u8, 73u8, 0u8, 77u8, 90u8, 0u8, 82u8, 79u8, 0u8,
+                    73u8, 68u8, 0u8, 84u8, 90u8, 0u8, 70u8, 74u8, 0u8, 82u8, 85u8, 0u8, 85u8,
+                    65u8, 0u8, 83u8, 66u8, 0u8, 82u8, 87u8, 0u8, 84u8, 90u8, 0u8, 74u8, 80u8,
+                    0u8, 73u8, 78u8, 0u8, 71u8, 72u8, 0u8, 82u8, 85u8, 0u8, 75u8, 69u8, 0u8,
+                    73u8, 68u8, 0u8, 73u8, 78u8, 0u8, 83u8, 78u8, 0u8, 73u8, 78u8, 0u8, 84u8,
+                    90u8, 0u8, 73u8, 84u8, 0u8, 73u8, 78u8, 0u8, 73u8, 84u8, 0u8, 71u8, 66u8,
+                    0u8, 80u8, 75u8, 0u8, 73u8, 84u8, 0u8, 73u8, 82u8, 0u8, 78u8, 79u8, 0u8,
+                    67u8, 73u8, 0u8, 77u8, 90u8, 0u8, 77u8, 88u8, 0u8, 77u8, 76u8, 0u8, 67u8,
+                    70u8, 0u8, 73u8, 69u8, 0u8, 76u8, 84u8, 0u8, 77u8, 65u8, 0u8, 77u8, 77u8,
+                    0u8, 76u8, 75u8, 0u8, 69u8, 84u8, 0u8, 83u8, 75u8, 0u8, 80u8, 75u8, 0u8,
+                    83u8, 73u8, 0u8, 80u8, 76u8, 0u8, 73u8, 68u8, 0u8, 87u8, 83u8, 0u8, 83u8,
+                    69u8, 0u8, 65u8, 79u8, 0u8, 83u8, 69u8, 0u8, 70u8, 73u8, 0u8, 73u8, 76u8,
+                    0u8, 70u8, 73u8, 0u8, 90u8, 87u8, 0u8, 77u8, 89u8, 0u8, 77u8, 76u8, 0u8,
+                    83u8, 79u8, 0u8, 85u8, 90u8, 0u8, 84u8, 72u8, 0u8, 65u8, 76u8, 0u8, 82u8,
+                    83u8, 0u8, 73u8, 78u8, 0u8, 83u8, 82u8, 0u8, 83u8, 78u8, 0u8, 73u8, 78u8,
+                    0u8, 90u8, 65u8, 0u8, 69u8, 82u8, 0u8, 90u8, 65u8, 0u8, 68u8, 69u8, 0u8,
+                    73u8, 68u8, 0u8, 84u8, 90u8, 0u8, 71u8, 78u8, 0u8, 83u8, 69u8, 0u8, 84u8,
+                    90u8, 0u8, 89u8, 84u8, 0u8, 67u8, 68u8, 0u8, 68u8, 69u8, 0u8, 73u8, 78u8,
+                    0u8, 73u8, 68u8, 0u8, 66u8, 68u8, 0u8, 73u8, 81u8, 0u8, 80u8, 76u8, 0u8,
+                    73u8, 78u8, 0u8, 78u8, 80u8, 0u8, 80u8, 72u8, 0u8, 73u8, 78u8, 0u8, 67u8,
+                    78u8, 0u8, 78u8, 80u8, 0u8, 78u8, 80u8, 0u8, 77u8, 89u8, 0u8, 73u8, 78u8,
+                    0u8, 83u8, 76u8, 0u8, 85u8, 71u8, 0u8, 84u8, 76u8, 0u8, 84u8, 74u8, 0u8,
+                    84u8, 72u8, 0u8, 78u8, 80u8, 0u8, 78u8, 80u8, 0u8, 78u8, 80u8, 0u8, 69u8,
+                    84u8, 0u8, 69u8, 82u8, 0u8, 78u8, 71u8, 0u8, 84u8, 77u8, 0u8, 84u8, 75u8,
+                    0u8, 65u8, 90u8, 0u8, 78u8, 80u8, 0u8, 80u8, 72u8, 0u8, 65u8, 90u8, 0u8,
+                    78u8, 69u8, 0u8, 90u8, 65u8, 0u8, 84u8, 79u8, 0u8, 77u8, 87u8, 0u8, 48u8,
+                    48u8, 49u8, 80u8, 71u8, 0u8, 84u8, 82u8, 0u8, 84u8, 82u8, 0u8, 84u8, 87u8,
+                    0u8, 80u8, 75u8, 0u8, 90u8, 65u8, 0u8, 71u8, 82u8, 0u8, 78u8, 80u8, 0u8,
+                    80u8, 72u8, 0u8, 66u8, 84u8, 0u8, 82u8, 85u8, 0u8, 85u8, 71u8, 0u8, 84u8,
+                    72u8, 0u8, 65u8, 90u8, 0u8, 77u8, 87u8, 0u8, 84u8, 86u8, 0u8, 78u8, 69u8,
+                    0u8, 67u8, 78u8, 0u8, 73u8, 78u8, 0u8, 80u8, 70u8, 0u8, 82u8, 85u8, 0u8,
+                    77u8, 65u8, 0u8, 82u8, 85u8, 0u8, 82u8, 85u8, 0u8, 67u8, 78u8, 0u8, 83u8,
+                    89u8, 0u8, 85u8, 65u8, 0u8, 70u8, 77u8, 0u8, 65u8, 79u8, 0u8, 73u8, 78u8,
+                    0u8, 73u8, 78u8, 0u8, 80u8, 75u8, 0u8, 85u8, 90u8, 0u8, 76u8, 82u8, 0u8,
+                    90u8, 65u8, 0u8, 73u8, 84u8, 0u8, 82u8, 85u8, 0u8, 86u8, 78u8, 0u8, 83u8,
+                    88u8, 0u8, 66u8, 69u8, 0u8, 68u8, 69u8, 0u8, 77u8, 90u8, 0u8, 48u8, 48u8,
+                    49u8, 82u8, 85u8, 0u8, 69u8, 69u8, 0u8, 84u8, 90u8, 0u8, 66u8, 69u8, 0u8,
+                    67u8, 72u8, 0u8, 69u8, 84u8, 0u8, 80u8, 72u8, 0u8, 65u8, 85u8, 0u8, 73u8,
+                    78u8, 0u8, 73u8, 78u8, 0u8, 87u8, 70u8, 0u8, 75u8, 77u8, 0u8, 83u8, 78u8,
+                    0u8, 73u8, 78u8, 0u8, 73u8, 78u8, 0u8, 67u8, 78u8, 0u8, 66u8, 82u8, 0u8,
+                    85u8, 90u8, 0u8, 84u8, 82u8, 0u8, 90u8, 65u8, 0u8, 84u8, 82u8, 0u8, 84u8,
+                    82u8, 0u8, 71u8, 69u8, 0u8, 67u8, 78u8, 0u8, 83u8, 68u8, 0u8, 83u8, 65u8,
+                    0u8, 73u8, 78u8, 0u8, 85u8, 71u8, 0u8, 73u8, 82u8, 0u8, 89u8, 69u8, 0u8,
+                    78u8, 80u8, 0u8, 77u8, 90u8, 0u8, 70u8, 77u8, 0u8, 67u8, 77u8, 0u8, 67u8,
+                    77u8, 0u8, 48u8, 48u8, 49u8, 78u8, 71u8, 0u8, 66u8, 82u8, 0u8, 77u8, 88u8,
+                    0u8, 72u8, 75u8, 0u8, 67u8, 78u8, 0u8, 83u8, 68u8, 0u8, 75u8, 77u8, 0u8,
+                    78u8, 76u8, 0u8, 77u8, 65u8, 0u8, 67u8, 78u8, 0u8, 67u8, 78u8, 0u8, 67u8,
+                    78u8, 0u8, 84u8, 71u8, 0u8, 77u8, 89u8, 0u8, 90u8, 65u8, 0u8, 84u8, 82u8,
+                    0u8,
+                ])
+            },
+        )
+    },
+    ls2r: unsafe {
+        #[allow(unused_unsafe)]
+        ::zerovec::ZeroMap2d::from_parts_unchecked(
+            unsafe {
+                ::zerovec::ZeroVec::from_bytes_unchecked(&[
+                    97u8, 114u8, 99u8, 97u8, 122u8, 0u8, 99u8, 117u8, 0u8, 101u8, 110u8, 0u8,
+                    102u8, 102u8, 0u8, 103u8, 114u8, 99u8, 107u8, 107u8, 0u8, 107u8, 117u8,
+                    0u8, 107u8, 121u8, 0u8, 108u8, 105u8, 102u8, 109u8, 97u8, 110u8, 109u8,
+                    110u8, 0u8, 112u8, 97u8, 0u8, 112u8, 97u8, 108u8, 115u8, 100u8, 0u8, 116u8,
+                    103u8, 0u8, 117u8, 103u8, 0u8, 117u8, 110u8, 114u8, 117u8, 122u8, 0u8,
+                    121u8, 117u8, 101u8, 122u8, 104u8, 0u8,
+                ])
+            },
+            unsafe {
+                ::zerovec::ZeroVec::from_bytes_unchecked(&[
+                    2u8, 0u8, 0u8, 0u8, 3u8, 0u8, 0u8, 0u8, 4u8, 0u8, 0u8, 0u8, 5u8, 0u8, 0u8,
+                    0u8, 6u8, 0u8, 0u8, 0u8, 7u8, 0u8, 0u8, 0u8, 8u8, 0u8, 0u8, 0u8, 10u8, 0u8,
+                    0u8, 0u8, 12u8, 0u8, 0u8, 0u8, 13u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8,
+                    15u8, 0u8, 0u8, 0u8, 16u8, 0u8, 0u8, 0u8, 17u8, 0u8, 0u8, 0u8, 20u8, 0u8,
+                    0u8, 0u8, 21u8, 0u8, 0u8, 0u8, 22u8, 0u8, 0u8, 0u8, 23u8, 0u8, 0u8, 0u8,
+                    24u8, 0u8, 0u8, 0u8, 25u8, 0u8, 0u8, 0u8, 28u8, 0u8, 0u8, 0u8,
+                ])
+            },
+            unsafe {
+                ::zerovec::ZeroVec::from_bytes_unchecked(&[
+                    78u8, 98u8, 97u8, 116u8, 80u8, 97u8, 108u8, 109u8, 65u8, 114u8, 97u8, 98u8,
+                    71u8, 108u8, 97u8, 103u8, 83u8, 104u8, 97u8, 119u8, 65u8, 100u8, 108u8,
+                    109u8, 76u8, 105u8, 110u8, 98u8, 65u8, 114u8, 97u8, 98u8, 65u8, 114u8,
+                    97u8, 98u8, 89u8, 101u8, 122u8, 105u8, 65u8, 114u8, 97u8, 98u8, 76u8, 97u8,
+                    116u8, 110u8, 76u8, 105u8, 109u8, 98u8, 78u8, 107u8, 111u8, 111u8, 77u8,
+                    111u8, 110u8, 103u8, 65u8, 114u8, 97u8, 98u8, 80u8, 104u8, 108u8, 112u8,
+                    68u8, 101u8, 118u8, 97u8, 75u8, 104u8, 111u8, 106u8, 83u8, 105u8, 110u8,
+                    100u8, 65u8, 114u8, 97u8, 98u8, 67u8, 121u8, 114u8, 108u8, 68u8, 101u8,
+                    118u8, 97u8, 65u8, 114u8, 97u8, 98u8, 72u8, 97u8, 110u8, 115u8, 66u8,
+                    111u8, 112u8, 111u8, 72u8, 97u8, 110u8, 98u8, 72u8, 97u8, 110u8, 116u8,
+                ])
+            },
+            unsafe {
+                ::zerovec::ZeroVec::from_bytes_unchecked(&[
+                    74u8, 79u8, 0u8, 83u8, 89u8, 0u8, 73u8, 82u8, 0u8, 66u8, 71u8, 0u8, 71u8,
+                    66u8, 0u8, 71u8, 78u8, 0u8, 71u8, 82u8, 0u8, 67u8, 78u8, 0u8, 73u8, 81u8,
+                    0u8, 71u8, 69u8, 0u8, 67u8, 78u8, 0u8, 84u8, 82u8, 0u8, 73u8, 78u8, 0u8,
+                    71u8, 78u8, 0u8, 67u8, 78u8, 0u8, 80u8, 75u8, 0u8, 67u8, 78u8, 0u8, 73u8,
+                    78u8, 0u8, 73u8, 78u8, 0u8, 73u8, 78u8, 0u8, 80u8, 75u8, 0u8, 75u8, 90u8,
+                    0u8, 78u8, 80u8, 0u8, 65u8, 70u8, 0u8, 67u8, 78u8, 0u8, 84u8, 87u8, 0u8,
+                    84u8, 87u8, 0u8, 84u8, 87u8, 0u8,
+                ])
+            },
+        )
+    },
+}
diff --git a/compiler/rustc_baked_icu_data/src/data/fallback/parents_v1.rs b/compiler/rustc_baked_icu_data/src/data/fallback/parents_v1.rs
deleted file mode 100644
index f07b4b806..000000000
--- a/compiler/rustc_baked_icu_data/src/data/fallback/parents_v1.rs
+++ /dev/null
@@ -1,207 +0,0 @@
-// @generated
-type DataStruct = < :: icu_provider_adapters :: fallback :: provider :: LocaleFallbackParentsV1Marker as :: icu_provider :: DataMarker > :: Yokeable ;
-pub static DATA: litemap::LiteMap<&str, &DataStruct, &[(&str, &DataStruct)]> =
-    litemap::LiteMap::from_sorted_store_unchecked(&[("und", UND)]);
-static UND: &DataStruct = &::icu_provider_adapters::fallback::provider::LocaleFallbackParentsV1 {
-    parents: unsafe {
-        #[allow(unused_unsafe)]
-        ::zerovec::ZeroMap::from_parts_unchecked(
-            unsafe {
-                ::zerovec::VarZeroVec::from_bytes_unchecked(&[
-                    131u8, 0u8, 0u8, 0u8, 0u8, 0u8, 6u8, 0u8, 11u8, 0u8, 16u8, 0u8, 21u8, 0u8,
-                    26u8, 0u8, 31u8, 0u8, 36u8, 0u8, 41u8, 0u8, 46u8, 0u8, 51u8, 0u8, 56u8, 0u8,
-                    61u8, 0u8, 66u8, 0u8, 71u8, 0u8, 76u8, 0u8, 81u8, 0u8, 86u8, 0u8, 91u8, 0u8,
-                    96u8, 0u8, 101u8, 0u8, 106u8, 0u8, 111u8, 0u8, 116u8, 0u8, 121u8, 0u8, 126u8,
-                    0u8, 131u8, 0u8, 136u8, 0u8, 141u8, 0u8, 146u8, 0u8, 151u8, 0u8, 156u8, 0u8,
-                    161u8, 0u8, 166u8, 0u8, 171u8, 0u8, 176u8, 0u8, 181u8, 0u8, 186u8, 0u8, 191u8,
-                    0u8, 196u8, 0u8, 201u8, 0u8, 206u8, 0u8, 211u8, 0u8, 216u8, 0u8, 221u8, 0u8,
-                    226u8, 0u8, 231u8, 0u8, 236u8, 0u8, 241u8, 0u8, 246u8, 0u8, 251u8, 0u8, 0u8,
-                    1u8, 5u8, 1u8, 10u8, 1u8, 15u8, 1u8, 20u8, 1u8, 25u8, 1u8, 30u8, 1u8, 35u8,
-                    1u8, 40u8, 1u8, 45u8, 1u8, 50u8, 1u8, 55u8, 1u8, 60u8, 1u8, 65u8, 1u8, 70u8,
-                    1u8, 75u8, 1u8, 80u8, 1u8, 85u8, 1u8, 90u8, 1u8, 95u8, 1u8, 100u8, 1u8, 105u8,
-                    1u8, 110u8, 1u8, 115u8, 1u8, 120u8, 1u8, 125u8, 1u8, 130u8, 1u8, 135u8, 1u8,
-                    140u8, 1u8, 145u8, 1u8, 150u8, 1u8, 155u8, 1u8, 160u8, 1u8, 165u8, 1u8, 170u8,
-                    1u8, 175u8, 1u8, 180u8, 1u8, 185u8, 1u8, 190u8, 1u8, 195u8, 1u8, 200u8, 1u8,
-                    205u8, 1u8, 210u8, 1u8, 215u8, 1u8, 220u8, 1u8, 225u8, 1u8, 230u8, 1u8, 235u8,
-                    1u8, 240u8, 1u8, 245u8, 1u8, 250u8, 1u8, 255u8, 1u8, 4u8, 2u8, 9u8, 2u8, 14u8,
-                    2u8, 19u8, 2u8, 24u8, 2u8, 29u8, 2u8, 34u8, 2u8, 39u8, 2u8, 44u8, 2u8, 49u8,
-                    2u8, 54u8, 2u8, 59u8, 2u8, 64u8, 2u8, 71u8, 2u8, 73u8, 2u8, 75u8, 2u8, 77u8,
-                    2u8, 82u8, 2u8, 87u8, 2u8, 92u8, 2u8, 97u8, 2u8, 102u8, 2u8, 107u8, 2u8, 112u8,
-                    2u8, 117u8, 2u8, 122u8, 2u8, 127u8, 2u8, 132u8, 2u8, 101u8, 110u8, 45u8, 49u8,
-                    53u8, 48u8, 101u8, 110u8, 45u8, 65u8, 71u8, 101u8, 110u8, 45u8, 65u8, 73u8,
-                    101u8, 110u8, 45u8, 65u8, 84u8, 101u8, 110u8, 45u8, 65u8, 85u8, 101u8, 110u8,
-                    45u8, 66u8, 66u8, 101u8, 110u8, 45u8, 66u8, 69u8, 101u8, 110u8, 45u8, 66u8,
-                    77u8, 101u8, 110u8, 45u8, 66u8, 83u8, 101u8, 110u8, 45u8, 66u8, 87u8, 101u8,
-                    110u8, 45u8, 66u8, 90u8, 101u8, 110u8, 45u8, 67u8, 67u8, 101u8, 110u8, 45u8,
-                    67u8, 72u8, 101u8, 110u8, 45u8, 67u8, 75u8, 101u8, 110u8, 45u8, 67u8, 77u8,
-                    101u8, 110u8, 45u8, 67u8, 88u8, 101u8, 110u8, 45u8, 67u8, 89u8, 101u8, 110u8,
-                    45u8, 68u8, 69u8, 101u8, 110u8, 45u8, 68u8, 71u8, 101u8, 110u8, 45u8, 68u8,
-                    75u8, 101u8, 110u8, 45u8, 68u8, 77u8, 101u8, 110u8, 45u8, 69u8, 82u8, 101u8,
-                    110u8, 45u8, 70u8, 73u8, 101u8, 110u8, 45u8, 70u8, 74u8, 101u8, 110u8, 45u8,
-                    70u8, 75u8, 101u8, 110u8, 45u8, 70u8, 77u8, 101u8, 110u8, 45u8, 71u8, 66u8,
-                    101u8, 110u8, 45u8, 71u8, 68u8, 101u8, 110u8, 45u8, 71u8, 71u8, 101u8, 110u8,
-                    45u8, 71u8, 72u8, 101u8, 110u8, 45u8, 71u8, 73u8, 101u8, 110u8, 45u8, 71u8,
-                    77u8, 101u8, 110u8, 45u8, 71u8, 89u8, 101u8, 110u8, 45u8, 72u8, 75u8, 101u8,
-                    110u8, 45u8, 73u8, 69u8, 101u8, 110u8, 45u8, 73u8, 76u8, 101u8, 110u8, 45u8,
-                    73u8, 77u8, 101u8, 110u8, 45u8, 73u8, 78u8, 101u8, 110u8, 45u8, 73u8, 79u8,
-                    101u8, 110u8, 45u8, 74u8, 69u8, 101u8, 110u8, 45u8, 74u8, 77u8, 101u8, 110u8,
-                    45u8, 75u8, 69u8, 101u8, 110u8, 45u8, 75u8, 73u8, 101u8, 110u8, 45u8, 75u8,
-                    78u8, 101u8, 110u8, 45u8, 75u8, 89u8, 101u8, 110u8, 45u8, 76u8, 67u8, 101u8,
-                    110u8, 45u8, 76u8, 82u8, 101u8, 110u8, 45u8, 76u8, 83u8, 101u8, 110u8, 45u8,
-                    77u8, 71u8, 101u8, 110u8, 45u8, 77u8, 79u8, 101u8, 110u8, 45u8, 77u8, 83u8,
-                    101u8, 110u8, 45u8, 77u8, 84u8, 101u8, 110u8, 45u8, 77u8, 85u8, 101u8, 110u8,
-                    45u8, 77u8, 86u8, 101u8, 110u8, 45u8, 77u8, 87u8, 101u8, 110u8, 45u8, 77u8,
-                    89u8, 101u8, 110u8, 45u8, 78u8, 65u8, 101u8, 110u8, 45u8, 78u8, 70u8, 101u8,
-                    110u8, 45u8, 78u8, 71u8, 101u8, 110u8, 45u8, 78u8, 76u8, 101u8, 110u8, 45u8,
-                    78u8, 82u8, 101u8, 110u8, 45u8, 78u8, 85u8, 101u8, 110u8, 45u8, 78u8, 90u8,
-                    101u8, 110u8, 45u8, 80u8, 71u8, 101u8, 110u8, 45u8, 80u8, 75u8, 101u8, 110u8,
-                    45u8, 80u8, 78u8, 101u8, 110u8, 45u8, 80u8, 87u8, 101u8, 110u8, 45u8, 82u8,
-                    87u8, 101u8, 110u8, 45u8, 83u8, 66u8, 101u8, 110u8, 45u8, 83u8, 67u8, 101u8,
-                    110u8, 45u8, 83u8, 68u8, 101u8, 110u8, 45u8, 83u8, 69u8, 101u8, 110u8, 45u8,
-                    83u8, 71u8, 101u8, 110u8, 45u8, 83u8, 72u8, 101u8, 110u8, 45u8, 83u8, 73u8,
-                    101u8, 110u8, 45u8, 83u8, 76u8, 101u8, 110u8, 45u8, 83u8, 83u8, 101u8, 110u8,
-                    45u8, 83u8, 88u8, 101u8, 110u8, 45u8, 83u8, 90u8, 101u8, 110u8, 45u8, 84u8,
-                    67u8, 101u8, 110u8, 45u8, 84u8, 75u8, 101u8, 110u8, 45u8, 84u8, 79u8, 101u8,
-                    110u8, 45u8, 84u8, 84u8, 101u8, 110u8, 45u8, 84u8, 86u8, 101u8, 110u8, 45u8,
-                    84u8, 90u8, 101u8, 110u8, 45u8, 85u8, 71u8, 101u8, 110u8, 45u8, 86u8, 67u8,
-                    101u8, 110u8, 45u8, 86u8, 71u8, 101u8, 110u8, 45u8, 86u8, 85u8, 101u8, 110u8,
-                    45u8, 87u8, 83u8, 101u8, 110u8, 45u8, 90u8, 65u8, 101u8, 110u8, 45u8, 90u8,
-                    77u8, 101u8, 110u8, 45u8, 90u8, 87u8, 101u8, 115u8, 45u8, 65u8, 82u8, 101u8,
-                    115u8, 45u8, 66u8, 79u8, 101u8, 115u8, 45u8, 66u8, 82u8, 101u8, 115u8, 45u8,
-                    66u8, 90u8, 101u8, 115u8, 45u8, 67u8, 76u8, 101u8, 115u8, 45u8, 67u8, 79u8,
-                    101u8, 115u8, 45u8, 67u8, 82u8, 101u8, 115u8, 45u8, 67u8, 85u8, 101u8, 115u8,
-                    45u8, 68u8, 79u8, 101u8, 115u8, 45u8, 69u8, 67u8, 101u8, 115u8, 45u8, 71u8,
-                    84u8, 101u8, 115u8, 45u8, 72u8, 78u8, 101u8, 115u8, 45u8, 77u8, 88u8, 101u8,
-                    115u8, 45u8, 78u8, 73u8, 101u8, 115u8, 45u8, 80u8, 65u8, 101u8, 115u8, 45u8,
-                    80u8, 69u8, 101u8, 115u8, 45u8, 80u8, 82u8, 101u8, 115u8, 45u8, 80u8, 89u8,
-                    101u8, 115u8, 45u8, 83u8, 86u8, 101u8, 115u8, 45u8, 85u8, 83u8, 101u8, 115u8,
-                    45u8, 85u8, 89u8, 101u8, 115u8, 45u8, 86u8, 69u8, 104u8, 105u8, 45u8, 76u8,
-                    97u8, 116u8, 110u8, 104u8, 116u8, 110u8, 98u8, 110u8, 110u8, 112u8, 116u8,
-                    45u8, 65u8, 79u8, 112u8, 116u8, 45u8, 67u8, 72u8, 112u8, 116u8, 45u8, 67u8,
-                    86u8, 112u8, 116u8, 45u8, 70u8, 82u8, 112u8, 116u8, 45u8, 71u8, 81u8, 112u8,
-                    116u8, 45u8, 71u8, 87u8, 112u8, 116u8, 45u8, 76u8, 85u8, 112u8, 116u8, 45u8,
-                    77u8, 79u8, 112u8, 116u8, 45u8, 77u8, 90u8, 112u8, 116u8, 45u8, 83u8, 84u8,
-                    112u8, 116u8, 45u8, 84u8, 76u8, 122u8, 104u8, 45u8, 72u8, 97u8, 110u8, 116u8,
-                    45u8, 77u8, 79u8,
-                ])
-            },
-            unsafe {
-                ::zerovec::ZeroVec::from_bytes_unchecked(&[
-                    101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
-                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 49u8, 53u8, 48u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
-                    48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 49u8, 53u8,
-                    48u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
-                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
-                    49u8, 53u8, 48u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8,
-                    49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
-                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 49u8, 53u8, 48u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
-                    49u8, 53u8, 48u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8,
-                    49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
-                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 49u8, 53u8, 48u8, 101u8, 110u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
-                    48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8,
-                    49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
-                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
-                    48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8,
-                    49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
-                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
-                    48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8,
-                    49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
-                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
-                    48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8,
-                    49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
-                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
-                    48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8,
-                    49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
-                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 49u8, 53u8, 48u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
-                    48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8,
-                    49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
-                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
-                    48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8,
-                    49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
-                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 49u8, 53u8, 48u8, 101u8, 110u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 1u8, 49u8, 53u8, 48u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
-                    48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8,
-                    49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
-                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
-                    48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8,
-                    49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
-                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
-                    48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8,
-                    49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
-                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 115u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
-                    52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8,
-                    57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8,
-                    115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
-                    52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8,
-                    57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8,
-                    115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
-                    52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8,
-                    57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8,
-                    115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 110u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 73u8, 78u8, 0u8, 102u8, 114u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 1u8, 72u8, 84u8, 0u8, 110u8, 111u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 110u8, 111u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    112u8, 116u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 80u8, 84u8, 0u8, 112u8, 116u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 80u8, 84u8, 0u8, 112u8, 116u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 1u8, 80u8, 84u8, 0u8, 112u8, 116u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 1u8, 80u8, 84u8, 0u8, 112u8, 116u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    1u8, 80u8, 84u8, 0u8, 112u8, 116u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 80u8,
-                    84u8, 0u8, 112u8, 116u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 80u8, 84u8, 0u8,
-                    112u8, 116u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 80u8, 84u8, 0u8, 112u8, 116u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 80u8, 84u8, 0u8, 112u8, 116u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 1u8, 80u8, 84u8, 0u8, 112u8, 116u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 1u8, 80u8, 84u8, 0u8, 122u8, 104u8, 0u8, 1u8, 72u8, 97u8, 110u8,
-                    116u8, 1u8, 72u8, 75u8, 0u8,
-                ])
-            },
-        )
-    },
-};
diff --git a/compiler/rustc_baked_icu_data/src/data/fallback/parents_v1/mod.rs b/compiler/rustc_baked_icu_data/src/data/fallback/parents_v1/mod.rs
new file mode 100644
index 000000000..ce04af868
--- /dev/null
+++ b/compiler/rustc_baked_icu_data/src/data/fallback/parents_v1/mod.rs
@@ -0,0 +1,6 @@
+// @generated
+type DataStruct = < :: icu_provider_adapters :: fallback :: provider :: LocaleFallbackParentsV1Marker as :: icu_provider :: DataMarker > :: Yokeable ;
+pub fn lookup(locale: &icu_provider::DataLocale) -> Option<&'static DataStruct> {
+    locale.is_empty().then(|| &UND)
+}
+static UND: DataStruct = include!("und.rs.data");
diff --git a/compiler/rustc_baked_icu_data/src/data/fallback/parents_v1/und.rs.data b/compiler/rustc_baked_icu_data/src/data/fallback/parents_v1/und.rs.data
new file mode 100644
index 000000000..5ead95908
--- /dev/null
+++ b/compiler/rustc_baked_icu_data/src/data/fallback/parents_v1/und.rs.data
@@ -0,0 +1,216 @@
+::icu_provider_adapters::fallback::provider::LocaleFallbackParentsV1 {
+    parents: unsafe {
+        #[allow(unused_unsafe)]
+        ::zerovec::ZeroMap::from_parts_unchecked(
+            unsafe {
+                ::zerovec::VarZeroVec::from_bytes_unchecked(&[
+                    131u8, 0u8, 0u8, 0u8, 0u8, 0u8, 6u8, 0u8, 11u8, 0u8, 16u8, 0u8, 21u8, 0u8,
+                    26u8, 0u8, 31u8, 0u8, 36u8, 0u8, 41u8, 0u8, 46u8, 0u8, 51u8, 0u8, 56u8,
+                    0u8, 61u8, 0u8, 66u8, 0u8, 71u8, 0u8, 76u8, 0u8, 81u8, 0u8, 86u8, 0u8,
+                    91u8, 0u8, 96u8, 0u8, 101u8, 0u8, 106u8, 0u8, 111u8, 0u8, 116u8, 0u8,
+                    121u8, 0u8, 126u8, 0u8, 131u8, 0u8, 136u8, 0u8, 141u8, 0u8, 146u8, 0u8,
+                    151u8, 0u8, 156u8, 0u8, 161u8, 0u8, 166u8, 0u8, 171u8, 0u8, 176u8, 0u8,
+                    181u8, 0u8, 186u8, 0u8, 191u8, 0u8, 196u8, 0u8, 201u8, 0u8, 206u8, 0u8,
+                    211u8, 0u8, 216u8, 0u8, 221u8, 0u8, 226u8, 0u8, 231u8, 0u8, 236u8, 0u8,
+                    241u8, 0u8, 246u8, 0u8, 251u8, 0u8, 0u8, 1u8, 5u8, 1u8, 10u8, 1u8, 15u8,
+                    1u8, 20u8, 1u8, 25u8, 1u8, 30u8, 1u8, 35u8, 1u8, 40u8, 1u8, 45u8, 1u8,
+                    50u8, 1u8, 55u8, 1u8, 60u8, 1u8, 65u8, 1u8, 70u8, 1u8, 75u8, 1u8, 80u8,
+                    1u8, 85u8, 1u8, 90u8, 1u8, 95u8, 1u8, 100u8, 1u8, 105u8, 1u8, 110u8, 1u8,
+                    115u8, 1u8, 120u8, 1u8, 125u8, 1u8, 130u8, 1u8, 135u8, 1u8, 140u8, 1u8,
+                    145u8, 1u8, 150u8, 1u8, 155u8, 1u8, 160u8, 1u8, 165u8, 1u8, 170u8, 1u8,
+                    175u8, 1u8, 180u8, 1u8, 185u8, 1u8, 190u8, 1u8, 195u8, 1u8, 200u8, 1u8,
+                    205u8, 1u8, 210u8, 1u8, 215u8, 1u8, 220u8, 1u8, 225u8, 1u8, 230u8, 1u8,
+                    235u8, 1u8, 240u8, 1u8, 245u8, 1u8, 250u8, 1u8, 255u8, 1u8, 4u8, 2u8, 9u8,
+                    2u8, 14u8, 2u8, 19u8, 2u8, 24u8, 2u8, 29u8, 2u8, 34u8, 2u8, 39u8, 2u8,
+                    44u8, 2u8, 49u8, 2u8, 54u8, 2u8, 59u8, 2u8, 64u8, 2u8, 71u8, 2u8, 73u8,
+                    2u8, 75u8, 2u8, 77u8, 2u8, 82u8, 2u8, 87u8, 2u8, 92u8, 2u8, 97u8, 2u8,
+                    102u8, 2u8, 107u8, 2u8, 112u8, 2u8, 117u8, 2u8, 122u8, 2u8, 127u8, 2u8,
+                    132u8, 2u8, 101u8, 110u8, 45u8, 49u8, 53u8, 48u8, 101u8, 110u8, 45u8, 65u8,
+                    71u8, 101u8, 110u8, 45u8, 65u8, 73u8, 101u8, 110u8, 45u8, 65u8, 84u8,
+                    101u8, 110u8, 45u8, 65u8, 85u8, 101u8, 110u8, 45u8, 66u8, 66u8, 101u8,
+                    110u8, 45u8, 66u8, 69u8, 101u8, 110u8, 45u8, 66u8, 77u8, 101u8, 110u8,
+                    45u8, 66u8, 83u8, 101u8, 110u8, 45u8, 66u8, 87u8, 101u8, 110u8, 45u8, 66u8,
+                    90u8, 101u8, 110u8, 45u8, 67u8, 67u8, 101u8, 110u8, 45u8, 67u8, 72u8,
+                    101u8, 110u8, 45u8, 67u8, 75u8, 101u8, 110u8, 45u8, 67u8, 77u8, 101u8,
+                    110u8, 45u8, 67u8, 88u8, 101u8, 110u8, 45u8, 67u8, 89u8, 101u8, 110u8,
+                    45u8, 68u8, 69u8, 101u8, 110u8, 45u8, 68u8, 71u8, 101u8, 110u8, 45u8, 68u8,
+                    75u8, 101u8, 110u8, 45u8, 68u8, 77u8, 101u8, 110u8, 45u8, 69u8, 82u8,
+                    101u8, 110u8, 45u8, 70u8, 73u8, 101u8, 110u8, 45u8, 70u8, 74u8, 101u8,
+                    110u8, 45u8, 70u8, 75u8, 101u8, 110u8, 45u8, 70u8, 77u8, 101u8, 110u8,
+                    45u8, 71u8, 66u8, 101u8, 110u8, 45u8, 71u8, 68u8, 101u8, 110u8, 45u8, 71u8,
+                    71u8, 101u8, 110u8, 45u8, 71u8, 72u8, 101u8, 110u8, 45u8, 71u8, 73u8,
+                    101u8, 110u8, 45u8, 71u8, 77u8, 101u8, 110u8, 45u8, 71u8, 89u8, 101u8,
+                    110u8, 45u8, 72u8, 75u8, 101u8, 110u8, 45u8, 73u8, 69u8, 101u8, 110u8,
+                    45u8, 73u8, 76u8, 101u8, 110u8, 45u8, 73u8, 77u8, 101u8, 110u8, 45u8, 73u8,
+                    78u8, 101u8, 110u8, 45u8, 73u8, 79u8, 101u8, 110u8, 45u8, 74u8, 69u8,
+                    101u8, 110u8, 45u8, 74u8, 77u8, 101u8, 110u8, 45u8, 75u8, 69u8, 101u8,
+                    110u8, 45u8, 75u8, 73u8, 101u8, 110u8, 45u8, 75u8, 78u8, 101u8, 110u8,
+                    45u8, 75u8, 89u8, 101u8, 110u8, 45u8, 76u8, 67u8, 101u8, 110u8, 45u8, 76u8,
+                    82u8, 101u8, 110u8, 45u8, 76u8, 83u8, 101u8, 110u8, 45u8, 77u8, 71u8,
+                    101u8, 110u8, 45u8, 77u8, 79u8, 101u8, 110u8, 45u8, 77u8, 83u8, 101u8,
+                    110u8, 45u8, 77u8, 84u8, 101u8, 110u8, 45u8, 77u8, 85u8, 101u8, 110u8,
+                    45u8, 77u8, 86u8, 101u8, 110u8, 45u8, 77u8, 87u8, 101u8, 110u8, 45u8, 77u8,
+                    89u8, 101u8, 110u8, 45u8, 78u8, 65u8, 101u8, 110u8, 45u8, 78u8, 70u8,
+                    101u8, 110u8, 45u8, 78u8, 71u8, 101u8, 110u8, 45u8, 78u8, 76u8, 101u8,
+                    110u8, 45u8, 78u8, 82u8, 101u8, 110u8, 45u8, 78u8, 85u8, 101u8, 110u8,
+                    45u8, 78u8, 90u8, 101u8, 110u8, 45u8, 80u8, 71u8, 101u8, 110u8, 45u8, 80u8,
+                    75u8, 101u8, 110u8, 45u8, 80u8, 78u8, 101u8, 110u8, 45u8, 80u8, 87u8,
+                    101u8, 110u8, 45u8, 82u8, 87u8, 101u8, 110u8, 45u8, 83u8, 66u8, 101u8,
+                    110u8, 45u8, 83u8, 67u8, 101u8, 110u8, 45u8, 83u8, 68u8, 101u8, 110u8,
+                    45u8, 83u8, 69u8, 101u8, 110u8, 45u8, 83u8, 71u8, 101u8, 110u8, 45u8, 83u8,
+                    72u8, 101u8, 110u8, 45u8, 83u8, 73u8, 101u8, 110u8, 45u8, 83u8, 76u8,
+                    101u8, 110u8, 45u8, 83u8, 83u8, 101u8, 110u8, 45u8, 83u8, 88u8, 101u8,
+                    110u8, 45u8, 83u8, 90u8, 101u8, 110u8, 45u8, 84u8, 67u8, 101u8, 110u8,
+                    45u8, 84u8, 75u8, 101u8, 110u8, 45u8, 84u8, 79u8, 101u8, 110u8, 45u8, 84u8,
+                    84u8, 101u8, 110u8, 45u8, 84u8, 86u8, 101u8, 110u8, 45u8, 84u8, 90u8,
+                    101u8, 110u8, 45u8, 85u8, 71u8, 101u8, 110u8, 45u8, 86u8, 67u8, 101u8,
+                    110u8, 45u8, 86u8, 71u8, 101u8, 110u8, 45u8, 86u8, 85u8, 101u8, 110u8,
+                    45u8, 87u8, 83u8, 101u8, 110u8, 45u8, 90u8, 65u8, 101u8, 110u8, 45u8, 90u8,
+                    77u8, 101u8, 110u8, 45u8, 90u8, 87u8, 101u8, 115u8, 45u8, 65u8, 82u8,
+                    101u8, 115u8, 45u8, 66u8, 79u8, 101u8, 115u8, 45u8, 66u8, 82u8, 101u8,
+                    115u8, 45u8, 66u8, 90u8, 101u8, 115u8, 45u8, 67u8, 76u8, 101u8, 115u8,
+                    45u8, 67u8, 79u8, 101u8, 115u8, 45u8, 67u8, 82u8, 101u8, 115u8, 45u8, 67u8,
+                    85u8, 101u8, 115u8, 45u8, 68u8, 79u8, 101u8, 115u8, 45u8, 69u8, 67u8,
+                    101u8, 115u8, 45u8, 71u8, 84u8, 101u8, 115u8, 45u8, 72u8, 78u8, 101u8,
+                    115u8, 45u8, 77u8, 88u8, 101u8, 115u8, 45u8, 78u8, 73u8, 101u8, 115u8,
+                    45u8, 80u8, 65u8, 101u8, 115u8, 45u8, 80u8, 69u8, 101u8, 115u8, 45u8, 80u8,
+                    82u8, 101u8, 115u8, 45u8, 80u8, 89u8, 101u8, 115u8, 45u8, 83u8, 86u8,
+                    101u8, 115u8, 45u8, 85u8, 83u8, 101u8, 115u8, 45u8, 85u8, 89u8, 101u8,
+                    115u8, 45u8, 86u8, 69u8, 104u8, 105u8, 45u8, 76u8, 97u8, 116u8, 110u8,
+                    104u8, 116u8, 110u8, 98u8, 110u8, 110u8, 112u8, 116u8, 45u8, 65u8, 79u8,
+                    112u8, 116u8, 45u8, 67u8, 72u8, 112u8, 116u8, 45u8, 67u8, 86u8, 112u8,
+                    116u8, 45u8, 70u8, 82u8, 112u8, 116u8, 45u8, 71u8, 81u8, 112u8, 116u8,
+                    45u8, 71u8, 87u8, 112u8, 116u8, 45u8, 76u8, 85u8, 112u8, 116u8, 45u8, 77u8,
+                    79u8, 112u8, 116u8, 45u8, 77u8, 90u8, 112u8, 116u8, 45u8, 83u8, 84u8,
+                    112u8, 116u8, 45u8, 84u8, 76u8, 122u8, 104u8, 45u8, 72u8, 97u8, 110u8,
+                    116u8, 45u8, 77u8, 79u8,
+                ])
+            },
+            unsafe {
+                ::zerovec::ZeroVec::from_bytes_unchecked(&[
+                    101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
+                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8,
+                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8,
+                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 49u8, 53u8, 48u8, 101u8, 110u8, 0u8, 0u8,
+                    0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8,
+                    0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8,
+                    0u8, 0u8, 1u8, 49u8, 53u8, 48u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                    0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                    1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
+                    48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8,
+                    48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8,
+                    49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 49u8, 53u8, 48u8,
+                    101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
+                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8,
+                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8,
+                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8,
+                    0u8, 0u8, 0u8, 0u8, 1u8, 49u8, 53u8, 48u8, 101u8, 110u8, 0u8, 0u8, 0u8,
+                    0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8,
+                    0u8, 0u8, 1u8, 49u8, 53u8, 48u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                    0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                    1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
+                    49u8, 53u8, 48u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8,
+                    48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8,
+                    49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8,
+                    101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
+                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8,
+                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8,
+                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8,
+                    0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8,
+                    0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8,
+                    0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                    0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                    1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
+                    48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8,
+                    48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8,
+                    49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8,
+                    101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
+                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8,
+                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8,
+                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8,
+                    0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8,
+                    0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8,
+                    0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                    0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                    1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
+                    48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8,
+                    48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8,
+                    49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8,
+                    101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
+                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8,
+                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8,
+                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8,
+                    0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8,
+                    0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8,
+                    0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                    0u8, 1u8, 49u8, 53u8, 48u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                    1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
+                    48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8,
+                    48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8,
+                    49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8,
+                    101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
+                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8,
+                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8,
+                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8,
+                    0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8,
+                    0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8,
+                    0u8, 0u8, 1u8, 49u8, 53u8, 48u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                    0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                    1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
+                    49u8, 53u8, 48u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8,
+                    48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8,
+                    49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8,
+                    101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
+                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8,
+                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8,
+                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8,
+                    0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8,
+                    0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8,
+                    0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                    0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                    1u8, 48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
+                    48u8, 48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8,
+                    48u8, 49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8,
+                    49u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8,
+                    101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8,
+                    110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 48u8, 48u8, 49u8, 101u8, 115u8,
+                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8,
+                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8,
+                    0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8,
+                    0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8,
+                    0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                    0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                    1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
+                    52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8,
+                    49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8,
+                    57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8,
+                    101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8,
+                    115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8,
+                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8,
+                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8,
+                    0u8, 0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8,
+                    0u8, 0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8,
+                    0u8, 0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                    0u8, 1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                    1u8, 52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
+                    52u8, 49u8, 57u8, 101u8, 115u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 52u8,
+                    49u8, 57u8, 101u8, 110u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 73u8, 78u8,
+                    0u8, 102u8, 114u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 72u8, 84u8, 0u8,
+                    110u8, 111u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 110u8,
+                    111u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 112u8, 116u8, 0u8,
+                    0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 80u8, 84u8, 0u8, 112u8, 116u8, 0u8, 0u8, 0u8,
+                    0u8, 0u8, 0u8, 1u8, 80u8, 84u8, 0u8, 112u8, 116u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                    0u8, 1u8, 80u8, 84u8, 0u8, 112u8, 116u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8,
+                    80u8, 84u8, 0u8, 112u8, 116u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 80u8,
+                    84u8, 0u8, 112u8, 116u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 80u8, 84u8,
+                    0u8, 112u8, 116u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 80u8, 84u8, 0u8,
+                    112u8, 116u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 80u8, 84u8, 0u8, 112u8,
+                    116u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 80u8, 84u8, 0u8, 112u8, 116u8,
+                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 80u8, 84u8, 0u8, 112u8, 116u8, 0u8, 0u8,
+                    0u8, 0u8, 0u8, 0u8, 1u8, 80u8, 84u8, 0u8, 122u8, 104u8, 0u8, 1u8, 72u8,
+                    97u8, 110u8, 116u8, 1u8, 72u8, 75u8, 0u8,
+                ])
+            },
+        )
+    },
+}
diff --git a/compiler/rustc_baked_icu_data/src/data/fallback/supplement/co_v1.rs b/compiler/rustc_baked_icu_data/src/data/fallback/supplement/co_v1.rs
deleted file mode 100644
index 7df33c12e..000000000
--- a/compiler/rustc_baked_icu_data/src/data/fallback/supplement/co_v1.rs
+++ /dev/null
@@ -1,41 +0,0 @@
-// @generated
-type DataStruct = < :: icu_provider_adapters :: fallback :: provider :: CollationFallbackSupplementV1Marker as :: icu_provider :: DataMarker > :: Yokeable ;
-pub static DATA: litemap::LiteMap<&str, &DataStruct, &[(&str, &DataStruct)]> =
-    litemap::LiteMap::from_sorted_store_unchecked(&[("und", UND)]);
-static UND: &DataStruct =
-    &::icu_provider_adapters::fallback::provider::LocaleFallbackSupplementV1 {
-        parents: unsafe {
-            #[allow(unused_unsafe)]
-            ::zerovec::ZeroMap::from_parts_unchecked(
-                unsafe {
-                    ::zerovec::VarZeroVec::from_bytes_unchecked(&[
-                        1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 121u8, 117u8, 101u8,
-                    ])
-                },
-                unsafe {
-                    ::zerovec::ZeroVec::from_bytes_unchecked(&[
-                        122u8, 104u8, 0u8, 1u8, 72u8, 97u8, 110u8, 116u8, 0u8, 0u8, 0u8, 0u8,
-                    ])
-                },
-            )
-        },
-        unicode_extension_defaults: unsafe {
-            #[allow(unused_unsafe)]
-            ::zerovec::ZeroMap2d::from_parts_unchecked(
-                unsafe { ::zerovec::ZeroVec::from_bytes_unchecked(&[99u8, 111u8]) },
-                unsafe { ::zerovec::ZeroVec::from_bytes_unchecked(&[2u8, 0u8, 0u8, 0u8]) },
-                unsafe {
-                    ::zerovec::VarZeroVec::from_bytes_unchecked(&[
-                        2u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 122u8, 104u8, 122u8, 104u8, 45u8,
-                        72u8, 97u8, 110u8, 116u8,
-                    ])
-                },
-                unsafe {
-                    ::zerovec::VarZeroVec::from_bytes_unchecked(&[
-                        2u8, 0u8, 0u8, 0u8, 0u8, 0u8, 6u8, 0u8, 112u8, 105u8, 110u8, 121u8, 105u8,
-                        110u8, 115u8, 116u8, 114u8, 111u8, 107u8, 101u8,
-                    ])
-                },
-            )
-        },
-    };
diff --git a/compiler/rustc_baked_icu_data/src/data/fallback/supplement/co_v1/mod.rs b/compiler/rustc_baked_icu_data/src/data/fallback/supplement/co_v1/mod.rs
new file mode 100644
index 000000000..902364713
--- /dev/null
+++ b/compiler/rustc_baked_icu_data/src/data/fallback/supplement/co_v1/mod.rs
@@ -0,0 +1,6 @@
+// @generated
+type DataStruct = < :: icu_provider_adapters :: fallback :: provider :: CollationFallbackSupplementV1Marker as :: icu_provider :: DataMarker > :: Yokeable ;
+pub fn lookup(locale: &icu_provider::DataLocale) -> Option<&'static DataStruct> {
+    locale.is_empty().then(|| &UND)
+}
+static UND: DataStruct = include!("und.rs.data");
diff --git a/compiler/rustc_baked_icu_data/src/data/fallback/supplement/co_v1/und.rs.data b/compiler/rustc_baked_icu_data/src/data/fallback/supplement/co_v1/und.rs.data
new file mode 100644
index 000000000..7d70e78c3
--- /dev/null
+++ b/compiler/rustc_baked_icu_data/src/data/fallback/supplement/co_v1/und.rs.data
@@ -0,0 +1,36 @@
+::icu_provider_adapters::fallback::provider::LocaleFallbackSupplementV1 {
+    parents: unsafe {
+        #[allow(unused_unsafe)]
+        ::zerovec::ZeroMap::from_parts_unchecked(
+            unsafe {
+                ::zerovec::VarZeroVec::from_bytes_unchecked(&[
+                    1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 121u8, 117u8, 101u8,
+                ])
+            },
+            unsafe {
+                ::zerovec::ZeroVec::from_bytes_unchecked(&[
+                    122u8, 104u8, 0u8, 1u8, 72u8, 97u8, 110u8, 116u8, 0u8, 0u8, 0u8, 0u8,
+                ])
+            },
+        )
+    },
+    unicode_extension_defaults: unsafe {
+        #[allow(unused_unsafe)]
+        ::zerovec::ZeroMap2d::from_parts_unchecked(
+            unsafe { ::zerovec::ZeroVec::from_bytes_unchecked(&[99u8, 111u8]) },
+            unsafe { ::zerovec::ZeroVec::from_bytes_unchecked(&[2u8, 0u8, 0u8, 0u8]) },
+            unsafe {
+                ::zerovec::VarZeroVec::from_bytes_unchecked(&[
+                    2u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 122u8, 104u8, 122u8, 104u8, 45u8,
+                    72u8, 97u8, 110u8, 116u8,
+                ])
+            },
+            unsafe {
+                ::zerovec::VarZeroVec::from_bytes_unchecked(&[
+                    2u8, 0u8, 0u8, 0u8, 0u8, 0u8, 6u8, 0u8, 112u8, 105u8, 110u8, 121u8, 105u8,
+                    110u8, 115u8, 116u8, 114u8, 111u8, 107u8, 101u8,
+                ])
+            },
+        )
+    },
+}
diff --git a/compiler/rustc_baked_icu_data/src/data/list/and_v1.rs b/compiler/rustc_baked_icu_data/src/data/list/and_v1.rs
deleted file mode 100644
index 9cae549e1..000000000
--- a/compiler/rustc_baked_icu_data/src/data/list/and_v1.rs
+++ /dev/null
@@ -1,1161 +0,0 @@
-// @generated
-type DataStruct = <::icu_list::provider::AndListV1Marker as ::icu_provider::DataMarker>::Yokeable;
-pub static DATA: litemap::LiteMap<&str, &DataStruct, &[(&str, &DataStruct)]> =
-    litemap::LiteMap::from_sorted_store_unchecked(&[
-        ("en", EN),
-        ("es", ES),
-        ("fr", FR),
-        ("it", IT),
-        ("ja", JA),
-        ("pt", PT),
-        ("ru", RU),
-        ("tr", TR),
-        ("und", UND),
-        ("zh", ZH_ZH_HANS),
-        ("zh-Hans", ZH_ZH_HANS),
-        ("zh-Hant", ZH_HANT),
-    ]);
-static EN: &DataStruct = &::icu_list::provider::ListFormatterPatternsV1([
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", and ", 6u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" and ", 5u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", & ", 4u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" & ", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-]);
-static ES: &DataStruct = &::icu_list::provider::ListFormatterPatternsV1([
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" y ", 3u8)
-        },
-        special_case: Some(::icu_list::provider::SpecialCasePattern {
-            condition: unsafe {
-                ::icu_list::provider::StringMatcher::from_dfa_bytes_unchecked(&[
-                    114u8, 117u8, 115u8, 116u8, 45u8, 114u8, 101u8, 103u8, 101u8, 120u8, 45u8,
-                    97u8, 117u8, 116u8, 111u8, 109u8, 97u8, 116u8, 97u8, 45u8, 100u8, 102u8, 97u8,
-                    45u8, 115u8, 112u8, 97u8, 114u8, 115u8, 101u8, 0u8, 0u8, 255u8, 254u8, 0u8,
-                    0u8, 2u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 1u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 2u8, 2u8, 2u8, 3u8, 4u8, 4u8, 5u8, 6u8, 7u8, 7u8, 7u8, 7u8,
-                    7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
-                    7u8, 7u8, 7u8, 8u8, 9u8, 9u8, 9u8, 10u8, 11u8, 11u8, 12u8, 13u8, 14u8, 14u8,
-                    14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
-                    14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8,
-                    15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 16u8, 16u8, 16u8,
-                    16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8,
-                    17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
-                    17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
-                    17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 18u8, 18u8, 19u8, 19u8, 19u8, 19u8, 19u8,
-                    19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
-                    19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 20u8,
-                    21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 22u8,
-                    23u8, 23u8, 24u8, 25u8, 25u8, 25u8, 26u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8,
-                    27u8, 27u8, 27u8, 27u8, 27u8, 40u8, 1u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 128u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 5u8,
-                    0u8, 5u8, 5u8, 6u8, 6u8, 12u8, 12u8, 13u8, 13u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8,
-                    68u8, 0u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 2u8, 0u8, 0u8, 27u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8,
-                    0u8, 0u8, 3u8, 0u8, 6u8, 6u8, 13u8, 13u8, 0u8, 0u8, 104u8, 0u8, 0u8, 0u8,
-                    104u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 2u8, 2u8,
-                    4u8, 7u8, 9u8, 9u8, 11u8, 14u8, 19u8, 19u8, 20u8, 20u8, 21u8, 21u8, 22u8, 22u8,
-                    23u8, 23u8, 24u8, 24u8, 25u8, 25u8, 26u8, 26u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
-                    68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8,
-                    0u8, 191u8, 0u8, 0u8, 0u8, 206u8, 0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8, 236u8,
-                    0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8, 251u8, 0u8, 0u8, 0u8, 10u8, 1u8, 0u8, 0u8,
-                    25u8, 1u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8,
-                    68u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 17u8, 17u8, 0u8, 0u8,
-                    191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8,
-                    191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 16u8, 0u8, 0u8,
-                    191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 16u8, 17u8, 0u8, 0u8,
-                    221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8,
-                    221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 15u8, 0u8, 0u8,
-                    221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 4u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8,
-                    0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 9u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8,
-                    18u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8,
-                    0u8, 35u8, 0u8, 0u8, 0u8,
-                ])
-            },
-            pattern: unsafe {
-                ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
-            },
-        }),
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" y ", 3u8)
-        },
-        special_case: Some(::icu_list::provider::SpecialCasePattern {
-            condition: unsafe {
-                ::icu_list::provider::StringMatcher::from_dfa_bytes_unchecked(&[
-                    114u8, 117u8, 115u8, 116u8, 45u8, 114u8, 101u8, 103u8, 101u8, 120u8, 45u8,
-                    97u8, 117u8, 116u8, 111u8, 109u8, 97u8, 116u8, 97u8, 45u8, 100u8, 102u8, 97u8,
-                    45u8, 115u8, 112u8, 97u8, 114u8, 115u8, 101u8, 0u8, 0u8, 255u8, 254u8, 0u8,
-                    0u8, 2u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 1u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 2u8, 2u8, 2u8, 3u8, 4u8, 4u8, 5u8, 6u8, 7u8, 7u8, 7u8, 7u8,
-                    7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
-                    7u8, 7u8, 7u8, 8u8, 9u8, 9u8, 9u8, 10u8, 11u8, 11u8, 12u8, 13u8, 14u8, 14u8,
-                    14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
-                    14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8,
-                    15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 16u8, 16u8, 16u8,
-                    16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8,
-                    17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
-                    17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
-                    17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 18u8, 18u8, 19u8, 19u8, 19u8, 19u8, 19u8,
-                    19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
-                    19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 20u8,
-                    21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 22u8,
-                    23u8, 23u8, 24u8, 25u8, 25u8, 25u8, 26u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8,
-                    27u8, 27u8, 27u8, 27u8, 27u8, 40u8, 1u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 128u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 5u8,
-                    0u8, 5u8, 5u8, 6u8, 6u8, 12u8, 12u8, 13u8, 13u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8,
-                    68u8, 0u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 2u8, 0u8, 0u8, 27u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8,
-                    0u8, 0u8, 3u8, 0u8, 6u8, 6u8, 13u8, 13u8, 0u8, 0u8, 104u8, 0u8, 0u8, 0u8,
-                    104u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 2u8, 2u8,
-                    4u8, 7u8, 9u8, 9u8, 11u8, 14u8, 19u8, 19u8, 20u8, 20u8, 21u8, 21u8, 22u8, 22u8,
-                    23u8, 23u8, 24u8, 24u8, 25u8, 25u8, 26u8, 26u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
-                    68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8,
-                    0u8, 191u8, 0u8, 0u8, 0u8, 206u8, 0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8, 236u8,
-                    0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8, 251u8, 0u8, 0u8, 0u8, 10u8, 1u8, 0u8, 0u8,
-                    25u8, 1u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8,
-                    68u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 17u8, 17u8, 0u8, 0u8,
-                    191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8,
-                    191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 16u8, 0u8, 0u8,
-                    191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 16u8, 17u8, 0u8, 0u8,
-                    221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8,
-                    221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 15u8, 0u8, 0u8,
-                    221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 4u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8,
-                    0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 9u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8,
-                    18u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8,
-                    0u8, 35u8, 0u8, 0u8, 0u8,
-                ])
-            },
-            pattern: unsafe {
-                ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
-            },
-        }),
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" y ", 3u8)
-        },
-        special_case: Some(::icu_list::provider::SpecialCasePattern {
-            condition: unsafe {
-                ::icu_list::provider::StringMatcher::from_dfa_bytes_unchecked(&[
-                    114u8, 117u8, 115u8, 116u8, 45u8, 114u8, 101u8, 103u8, 101u8, 120u8, 45u8,
-                    97u8, 117u8, 116u8, 111u8, 109u8, 97u8, 116u8, 97u8, 45u8, 100u8, 102u8, 97u8,
-                    45u8, 115u8, 112u8, 97u8, 114u8, 115u8, 101u8, 0u8, 0u8, 255u8, 254u8, 0u8,
-                    0u8, 2u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 1u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 2u8, 2u8, 2u8, 3u8, 4u8, 4u8, 5u8, 6u8, 7u8, 7u8, 7u8, 7u8,
-                    7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
-                    7u8, 7u8, 7u8, 8u8, 9u8, 9u8, 9u8, 10u8, 11u8, 11u8, 12u8, 13u8, 14u8, 14u8,
-                    14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
-                    14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8,
-                    15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 16u8, 16u8, 16u8,
-                    16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8,
-                    17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
-                    17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
-                    17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 18u8, 18u8, 19u8, 19u8, 19u8, 19u8, 19u8,
-                    19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
-                    19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 20u8,
-                    21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 22u8,
-                    23u8, 23u8, 24u8, 25u8, 25u8, 25u8, 26u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8,
-                    27u8, 27u8, 27u8, 27u8, 27u8, 40u8, 1u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 128u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 5u8,
-                    0u8, 5u8, 5u8, 6u8, 6u8, 12u8, 12u8, 13u8, 13u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8,
-                    68u8, 0u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 2u8, 0u8, 0u8, 27u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8,
-                    0u8, 0u8, 3u8, 0u8, 6u8, 6u8, 13u8, 13u8, 0u8, 0u8, 104u8, 0u8, 0u8, 0u8,
-                    104u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 2u8, 2u8,
-                    4u8, 7u8, 9u8, 9u8, 11u8, 14u8, 19u8, 19u8, 20u8, 20u8, 21u8, 21u8, 22u8, 22u8,
-                    23u8, 23u8, 24u8, 24u8, 25u8, 25u8, 26u8, 26u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
-                    68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8,
-                    0u8, 191u8, 0u8, 0u8, 0u8, 206u8, 0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8, 236u8,
-                    0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8, 251u8, 0u8, 0u8, 0u8, 10u8, 1u8, 0u8, 0u8,
-                    25u8, 1u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8,
-                    68u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 17u8, 17u8, 0u8, 0u8,
-                    191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8,
-                    191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 16u8, 0u8, 0u8,
-                    191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 16u8, 17u8, 0u8, 0u8,
-                    221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8,
-                    221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 15u8, 0u8, 0u8,
-                    221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 4u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8,
-                    0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 9u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8,
-                    18u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8,
-                    0u8, 35u8, 0u8, 0u8, 0u8,
-                ])
-            },
-            pattern: unsafe {
-                ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
-            },
-        }),
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" y ", 3u8)
-        },
-        special_case: Some(::icu_list::provider::SpecialCasePattern {
-            condition: unsafe {
-                ::icu_list::provider::StringMatcher::from_dfa_bytes_unchecked(&[
-                    114u8, 117u8, 115u8, 116u8, 45u8, 114u8, 101u8, 103u8, 101u8, 120u8, 45u8,
-                    97u8, 117u8, 116u8, 111u8, 109u8, 97u8, 116u8, 97u8, 45u8, 100u8, 102u8, 97u8,
-                    45u8, 115u8, 112u8, 97u8, 114u8, 115u8, 101u8, 0u8, 0u8, 255u8, 254u8, 0u8,
-                    0u8, 2u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 1u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 2u8, 2u8, 2u8, 3u8, 4u8, 4u8, 5u8, 6u8, 7u8, 7u8, 7u8, 7u8,
-                    7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
-                    7u8, 7u8, 7u8, 8u8, 9u8, 9u8, 9u8, 10u8, 11u8, 11u8, 12u8, 13u8, 14u8, 14u8,
-                    14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
-                    14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8,
-                    15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 16u8, 16u8, 16u8,
-                    16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8,
-                    17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
-                    17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
-                    17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 18u8, 18u8, 19u8, 19u8, 19u8, 19u8, 19u8,
-                    19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
-                    19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 20u8,
-                    21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 22u8,
-                    23u8, 23u8, 24u8, 25u8, 25u8, 25u8, 26u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8,
-                    27u8, 27u8, 27u8, 27u8, 27u8, 40u8, 1u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 128u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 5u8,
-                    0u8, 5u8, 5u8, 6u8, 6u8, 12u8, 12u8, 13u8, 13u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8,
-                    68u8, 0u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 2u8, 0u8, 0u8, 27u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8,
-                    0u8, 0u8, 3u8, 0u8, 6u8, 6u8, 13u8, 13u8, 0u8, 0u8, 104u8, 0u8, 0u8, 0u8,
-                    104u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 2u8, 2u8,
-                    4u8, 7u8, 9u8, 9u8, 11u8, 14u8, 19u8, 19u8, 20u8, 20u8, 21u8, 21u8, 22u8, 22u8,
-                    23u8, 23u8, 24u8, 24u8, 25u8, 25u8, 26u8, 26u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
-                    68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8,
-                    0u8, 191u8, 0u8, 0u8, 0u8, 206u8, 0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8, 236u8,
-                    0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8, 251u8, 0u8, 0u8, 0u8, 10u8, 1u8, 0u8, 0u8,
-                    25u8, 1u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8,
-                    68u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 17u8, 17u8, 0u8, 0u8,
-                    191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8,
-                    191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 16u8, 0u8, 0u8,
-                    191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 16u8, 17u8, 0u8, 0u8,
-                    221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8,
-                    221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 15u8, 0u8, 0u8,
-                    221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 4u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8,
-                    0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 9u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8,
-                    18u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8,
-                    0u8, 35u8, 0u8, 0u8, 0u8,
-                ])
-            },
-            pattern: unsafe {
-                ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
-            },
-        }),
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" y ", 3u8)
-        },
-        special_case: Some(::icu_list::provider::SpecialCasePattern {
-            condition: unsafe {
-                ::icu_list::provider::StringMatcher::from_dfa_bytes_unchecked(&[
-                    114u8, 117u8, 115u8, 116u8, 45u8, 114u8, 101u8, 103u8, 101u8, 120u8, 45u8,
-                    97u8, 117u8, 116u8, 111u8, 109u8, 97u8, 116u8, 97u8, 45u8, 100u8, 102u8, 97u8,
-                    45u8, 115u8, 112u8, 97u8, 114u8, 115u8, 101u8, 0u8, 0u8, 255u8, 254u8, 0u8,
-                    0u8, 2u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 1u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 2u8, 2u8, 2u8, 3u8, 4u8, 4u8, 5u8, 6u8, 7u8, 7u8, 7u8, 7u8,
-                    7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
-                    7u8, 7u8, 7u8, 8u8, 9u8, 9u8, 9u8, 10u8, 11u8, 11u8, 12u8, 13u8, 14u8, 14u8,
-                    14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
-                    14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8,
-                    15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 16u8, 16u8, 16u8,
-                    16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8,
-                    17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
-                    17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
-                    17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 18u8, 18u8, 19u8, 19u8, 19u8, 19u8, 19u8,
-                    19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
-                    19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 20u8,
-                    21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 22u8,
-                    23u8, 23u8, 24u8, 25u8, 25u8, 25u8, 26u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8,
-                    27u8, 27u8, 27u8, 27u8, 27u8, 40u8, 1u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 128u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 5u8,
-                    0u8, 5u8, 5u8, 6u8, 6u8, 12u8, 12u8, 13u8, 13u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8,
-                    68u8, 0u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 2u8, 0u8, 0u8, 27u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8,
-                    0u8, 0u8, 3u8, 0u8, 6u8, 6u8, 13u8, 13u8, 0u8, 0u8, 104u8, 0u8, 0u8, 0u8,
-                    104u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 2u8, 2u8,
-                    4u8, 7u8, 9u8, 9u8, 11u8, 14u8, 19u8, 19u8, 20u8, 20u8, 21u8, 21u8, 22u8, 22u8,
-                    23u8, 23u8, 24u8, 24u8, 25u8, 25u8, 26u8, 26u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
-                    68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8,
-                    0u8, 191u8, 0u8, 0u8, 0u8, 206u8, 0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8, 236u8,
-                    0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8, 251u8, 0u8, 0u8, 0u8, 10u8, 1u8, 0u8, 0u8,
-                    25u8, 1u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8,
-                    68u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 17u8, 17u8, 0u8, 0u8,
-                    191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8,
-                    191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 16u8, 0u8, 0u8,
-                    191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 16u8, 17u8, 0u8, 0u8,
-                    221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8,
-                    221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 15u8, 0u8, 0u8,
-                    221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 4u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8,
-                    0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 9u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8,
-                    18u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8,
-                    0u8, 35u8, 0u8, 0u8, 0u8,
-                ])
-            },
-            pattern: unsafe {
-                ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
-            },
-        }),
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" y ", 3u8)
-        },
-        special_case: Some(::icu_list::provider::SpecialCasePattern {
-            condition: unsafe {
-                ::icu_list::provider::StringMatcher::from_dfa_bytes_unchecked(&[
-                    114u8, 117u8, 115u8, 116u8, 45u8, 114u8, 101u8, 103u8, 101u8, 120u8, 45u8,
-                    97u8, 117u8, 116u8, 111u8, 109u8, 97u8, 116u8, 97u8, 45u8, 100u8, 102u8, 97u8,
-                    45u8, 115u8, 112u8, 97u8, 114u8, 115u8, 101u8, 0u8, 0u8, 255u8, 254u8, 0u8,
-                    0u8, 2u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 1u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 1u8, 2u8, 2u8, 2u8, 3u8, 4u8, 4u8, 5u8, 6u8, 7u8, 7u8, 7u8, 7u8,
-                    7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
-                    7u8, 7u8, 7u8, 8u8, 9u8, 9u8, 9u8, 10u8, 11u8, 11u8, 12u8, 13u8, 14u8, 14u8,
-                    14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
-                    14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8,
-                    15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 16u8, 16u8, 16u8,
-                    16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8,
-                    17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
-                    17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
-                    17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 18u8, 18u8, 19u8, 19u8, 19u8, 19u8, 19u8,
-                    19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
-                    19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 20u8,
-                    21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 22u8,
-                    23u8, 23u8, 24u8, 25u8, 25u8, 25u8, 26u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8,
-                    27u8, 27u8, 27u8, 27u8, 27u8, 40u8, 1u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 128u8,
-                    0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 5u8,
-                    0u8, 5u8, 5u8, 6u8, 6u8, 12u8, 12u8, 13u8, 13u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8,
-                    68u8, 0u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 2u8, 0u8, 0u8, 27u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8,
-                    0u8, 0u8, 3u8, 0u8, 6u8, 6u8, 13u8, 13u8, 0u8, 0u8, 104u8, 0u8, 0u8, 0u8,
-                    104u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 2u8, 2u8,
-                    4u8, 7u8, 9u8, 9u8, 11u8, 14u8, 19u8, 19u8, 20u8, 20u8, 21u8, 21u8, 22u8, 22u8,
-                    23u8, 23u8, 24u8, 24u8, 25u8, 25u8, 26u8, 26u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
-                    68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8,
-                    0u8, 191u8, 0u8, 0u8, 0u8, 206u8, 0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8, 236u8,
-                    0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8, 251u8, 0u8, 0u8, 0u8, 10u8, 1u8, 0u8, 0u8,
-                    25u8, 1u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8,
-                    68u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 17u8, 17u8, 0u8, 0u8,
-                    191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8,
-                    191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 16u8, 0u8, 0u8,
-                    191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 16u8, 17u8, 0u8, 0u8,
-                    221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8,
-                    221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 15u8, 0u8, 0u8,
-                    221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 4u8, 0u8, 0u8, 0u8, 0u8, 0u8,
-                    0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8,
-                    0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 9u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8,
-                    18u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8,
-                    0u8, 35u8, 0u8, 0u8, 0u8,
-                ])
-            },
-            pattern: unsafe {
-                ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
-            },
-        }),
-    },
-]);
-static FR: &DataStruct = &::icu_list::provider::ListFormatterPatternsV1([
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" et ", 4u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" et ", 4u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" et ", 4u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" et ", 4u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-]);
-static IT: &DataStruct = &::icu_list::provider::ListFormatterPatternsV1([
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
-        },
-        special_case: None,
-    },
-]);
-static JA: &DataStruct = &::icu_list::provider::ListFormatterPatternsV1([
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-]);
-static PT: &DataStruct = &::icu_list::provider::ListFormatterPatternsV1([
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-]);
-static RU: &DataStruct = &::icu_list::provider::ListFormatterPatternsV1([
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" и ", 4u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" и ", 4u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" и ", 4u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" и ", 4u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-]);
-static TR: &DataStruct = &::icu_list::provider::ListFormatterPatternsV1([
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" ve ", 4u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" ve ", 4u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" ve ", 4u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" ve ", 4u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-]);
-static UND: &DataStruct = &::icu_list::provider::ListFormatterPatternsV1([
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
-        },
-        special_case: None,
-    },
-]);
-static ZH_HANT: &DataStruct = &::icu_list::provider::ListFormatterPatternsV1([
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("和", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("和", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("和", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("和", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("和", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("和", 3u8)
-        },
-        special_case: None,
-    },
-]);
-static ZH_ZH_HANS: &DataStruct = &::icu_list::provider::ListFormatterPatternsV1([
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("和", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("和", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("和", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("和", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-    ::icu_list::provider::ConditionalListJoinerPattern {
-        default: unsafe {
-            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
-        },
-        special_case: None,
-    },
-]);
diff --git a/compiler/rustc_baked_icu_data/src/data/list/and_v1/en.rs.data b/compiler/rustc_baked_icu_data/src/data/list/and_v1/en.rs.data
new file mode 100644
index 000000000..cb5cbfa87
--- /dev/null
+++ b/compiler/rustc_baked_icu_data/src/data/list/and_v1/en.rs.data
@@ -0,0 +1,74 @@
+::icu_list::provider::ListFormatterPatternsV1([
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", and ", 6u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" and ", 5u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", & ", 4u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" & ", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+])
diff --git a/compiler/rustc_baked_icu_data/src/data/list/and_v1/es.rs.data b/compiler/rustc_baked_icu_data/src/data/list/and_v1/es.rs.data
new file mode 100644
index 000000000..51f910975
--- /dev/null
+++ b/compiler/rustc_baked_icu_data/src/data/list/and_v1/es.rs.data
@@ -0,0 +1,836 @@
+::icu_list::provider::ListFormatterPatternsV1([
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" y ", 3u8)
+        },
+        special_case: Some(::icu_list::provider::SpecialCasePattern {
+            condition: unsafe {
+                ::icu_list::provider::SerdeDFA::from_dfa_bytes_unchecked(
+                    if cfg!(target_endian = "little") {
+                        &[
+                            114u8, 117u8, 115u8, 116u8, 45u8, 114u8, 101u8, 103u8, 101u8,
+                            120u8, 45u8, 97u8, 117u8, 116u8, 111u8, 109u8, 97u8, 116u8, 97u8,
+                            45u8, 100u8, 102u8, 97u8, 45u8, 115u8, 112u8, 97u8, 114u8, 115u8,
+                            101u8, 0u8, 0u8, 255u8, 254u8, 0u8, 0u8, 2u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 2u8,
+                            2u8, 2u8, 3u8, 4u8, 4u8, 5u8, 6u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 8u8, 9u8, 9u8, 9u8, 10u8, 11u8, 11u8, 12u8,
+                            13u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8,
+                            15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 18u8,
+                            18u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 20u8, 21u8,
+                            21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8,
+                            22u8, 23u8, 23u8, 24u8, 25u8, 25u8, 25u8, 26u8, 27u8, 27u8, 27u8,
+                            27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 40u8, 1u8, 0u8,
+                            0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 128u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 5u8, 0u8, 5u8,
+                            5u8, 6u8, 6u8, 12u8, 12u8, 13u8, 13u8, 0u8, 0u8, 83u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 0u8, 27u8, 0u8, 0u8, 18u8, 0u8,
+                            0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 3u8, 0u8, 6u8, 6u8, 13u8, 13u8,
+                            0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 2u8, 2u8, 4u8, 7u8, 9u8, 9u8,
+                            11u8, 14u8, 19u8, 19u8, 20u8, 20u8, 21u8, 21u8, 22u8, 22u8, 23u8,
+                            23u8, 24u8, 24u8, 25u8, 25u8, 26u8, 26u8, 0u8, 0u8, 68u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            68u8, 0u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 206u8, 0u8, 0u8, 0u8,
+                            221u8, 0u8, 0u8, 0u8, 236u8, 0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            251u8, 0u8, 0u8, 0u8, 10u8, 1u8, 0u8, 0u8, 25u8, 1u8, 0u8, 0u8,
+                            18u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 68u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 17u8, 17u8, 0u8,
+                            0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8,
+                            17u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8,
+                            0u8, 15u8, 16u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 2u8, 0u8, 16u8, 17u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 221u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 15u8, 0u8,
+                            0u8, 221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 4u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8,
+                            35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 9u8,
+                            0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8,
+                            0u8,
+                        ]
+                    } else {
+                        &[
+                            114u8, 117u8, 115u8, 116u8, 45u8, 114u8, 101u8, 103u8, 101u8,
+                            120u8, 45u8, 97u8, 117u8, 116u8, 111u8, 109u8, 97u8, 116u8, 97u8,
+                            45u8, 100u8, 102u8, 97u8, 45u8, 115u8, 112u8, 97u8, 114u8, 115u8,
+                            101u8, 0u8, 0u8, 0u8, 0u8, 254u8, 255u8, 0u8, 0u8, 0u8, 2u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 2u8,
+                            2u8, 2u8, 3u8, 4u8, 4u8, 5u8, 6u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 8u8, 9u8, 9u8, 9u8, 10u8, 11u8, 11u8, 12u8,
+                            13u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8,
+                            15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 18u8,
+                            18u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 20u8, 21u8,
+                            21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8,
+                            22u8, 23u8, 23u8, 24u8, 25u8, 25u8, 25u8, 26u8, 27u8, 27u8, 27u8,
+                            27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 0u8, 0u8, 1u8,
+                            40u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 128u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 5u8, 0u8, 5u8,
+                            5u8, 6u8, 6u8, 12u8, 12u8, 13u8, 13u8, 0u8, 0u8, 83u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 0u8, 27u8, 0u8, 0u8, 18u8, 0u8,
+                            0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 3u8, 0u8, 6u8, 6u8, 13u8, 13u8,
+                            0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 2u8, 2u8, 4u8, 7u8, 9u8, 9u8,
+                            11u8, 14u8, 19u8, 19u8, 20u8, 20u8, 21u8, 21u8, 22u8, 22u8, 23u8,
+                            23u8, 24u8, 24u8, 25u8, 25u8, 26u8, 26u8, 0u8, 0u8, 68u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            68u8, 0u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 206u8, 0u8, 0u8, 0u8,
+                            221u8, 0u8, 0u8, 0u8, 236u8, 0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            251u8, 0u8, 0u8, 0u8, 10u8, 1u8, 0u8, 0u8, 25u8, 1u8, 0u8, 0u8,
+                            18u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 68u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 17u8, 17u8, 0u8,
+                            0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8,
+                            17u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8,
+                            0u8, 15u8, 16u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 2u8, 0u8, 16u8, 17u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 221u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 15u8, 0u8,
+                            0u8, 221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            4u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8,
+                            35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8,
+                            0u8, 0u8, 9u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8,
+                            35u8,
+                        ]
+                    },
+                )
+            },
+            pattern: unsafe {
+                ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
+            },
+        }),
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" y ", 3u8)
+        },
+        special_case: Some(::icu_list::provider::SpecialCasePattern {
+            condition: unsafe {
+                ::icu_list::provider::SerdeDFA::from_dfa_bytes_unchecked(
+                    if cfg!(target_endian = "little") {
+                        &[
+                            114u8, 117u8, 115u8, 116u8, 45u8, 114u8, 101u8, 103u8, 101u8,
+                            120u8, 45u8, 97u8, 117u8, 116u8, 111u8, 109u8, 97u8, 116u8, 97u8,
+                            45u8, 100u8, 102u8, 97u8, 45u8, 115u8, 112u8, 97u8, 114u8, 115u8,
+                            101u8, 0u8, 0u8, 255u8, 254u8, 0u8, 0u8, 2u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 2u8,
+                            2u8, 2u8, 3u8, 4u8, 4u8, 5u8, 6u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 8u8, 9u8, 9u8, 9u8, 10u8, 11u8, 11u8, 12u8,
+                            13u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8,
+                            15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 18u8,
+                            18u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 20u8, 21u8,
+                            21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8,
+                            22u8, 23u8, 23u8, 24u8, 25u8, 25u8, 25u8, 26u8, 27u8, 27u8, 27u8,
+                            27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 40u8, 1u8, 0u8,
+                            0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 128u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 5u8, 0u8, 5u8,
+                            5u8, 6u8, 6u8, 12u8, 12u8, 13u8, 13u8, 0u8, 0u8, 83u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 0u8, 27u8, 0u8, 0u8, 18u8, 0u8,
+                            0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 3u8, 0u8, 6u8, 6u8, 13u8, 13u8,
+                            0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 2u8, 2u8, 4u8, 7u8, 9u8, 9u8,
+                            11u8, 14u8, 19u8, 19u8, 20u8, 20u8, 21u8, 21u8, 22u8, 22u8, 23u8,
+                            23u8, 24u8, 24u8, 25u8, 25u8, 26u8, 26u8, 0u8, 0u8, 68u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            68u8, 0u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 206u8, 0u8, 0u8, 0u8,
+                            221u8, 0u8, 0u8, 0u8, 236u8, 0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            251u8, 0u8, 0u8, 0u8, 10u8, 1u8, 0u8, 0u8, 25u8, 1u8, 0u8, 0u8,
+                            18u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 68u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 17u8, 17u8, 0u8,
+                            0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8,
+                            17u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8,
+                            0u8, 15u8, 16u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 2u8, 0u8, 16u8, 17u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 221u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 15u8, 0u8,
+                            0u8, 221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 4u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8,
+                            35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 9u8,
+                            0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8,
+                            0u8,
+                        ]
+                    } else {
+                        &[
+                            114u8, 117u8, 115u8, 116u8, 45u8, 114u8, 101u8, 103u8, 101u8,
+                            120u8, 45u8, 97u8, 117u8, 116u8, 111u8, 109u8, 97u8, 116u8, 97u8,
+                            45u8, 100u8, 102u8, 97u8, 45u8, 115u8, 112u8, 97u8, 114u8, 115u8,
+                            101u8, 0u8, 0u8, 0u8, 0u8, 254u8, 255u8, 0u8, 0u8, 0u8, 2u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 2u8,
+                            2u8, 2u8, 3u8, 4u8, 4u8, 5u8, 6u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 8u8, 9u8, 9u8, 9u8, 10u8, 11u8, 11u8, 12u8,
+                            13u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8,
+                            15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 18u8,
+                            18u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 20u8, 21u8,
+                            21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8,
+                            22u8, 23u8, 23u8, 24u8, 25u8, 25u8, 25u8, 26u8, 27u8, 27u8, 27u8,
+                            27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 0u8, 0u8, 1u8,
+                            40u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 128u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 5u8, 0u8, 5u8,
+                            5u8, 6u8, 6u8, 12u8, 12u8, 13u8, 13u8, 0u8, 0u8, 83u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 0u8, 27u8, 0u8, 0u8, 18u8, 0u8,
+                            0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 3u8, 0u8, 6u8, 6u8, 13u8, 13u8,
+                            0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 2u8, 2u8, 4u8, 7u8, 9u8, 9u8,
+                            11u8, 14u8, 19u8, 19u8, 20u8, 20u8, 21u8, 21u8, 22u8, 22u8, 23u8,
+                            23u8, 24u8, 24u8, 25u8, 25u8, 26u8, 26u8, 0u8, 0u8, 68u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            68u8, 0u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 206u8, 0u8, 0u8, 0u8,
+                            221u8, 0u8, 0u8, 0u8, 236u8, 0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            251u8, 0u8, 0u8, 0u8, 10u8, 1u8, 0u8, 0u8, 25u8, 1u8, 0u8, 0u8,
+                            18u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 68u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 17u8, 17u8, 0u8,
+                            0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8,
+                            17u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8,
+                            0u8, 15u8, 16u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 2u8, 0u8, 16u8, 17u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 221u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 15u8, 0u8,
+                            0u8, 221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            4u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8,
+                            35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8,
+                            0u8, 0u8, 9u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8,
+                            35u8,
+                        ]
+                    },
+                )
+            },
+            pattern: unsafe {
+                ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
+            },
+        }),
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" y ", 3u8)
+        },
+        special_case: Some(::icu_list::provider::SpecialCasePattern {
+            condition: unsafe {
+                ::icu_list::provider::SerdeDFA::from_dfa_bytes_unchecked(
+                    if cfg!(target_endian = "little") {
+                        &[
+                            114u8, 117u8, 115u8, 116u8, 45u8, 114u8, 101u8, 103u8, 101u8,
+                            120u8, 45u8, 97u8, 117u8, 116u8, 111u8, 109u8, 97u8, 116u8, 97u8,
+                            45u8, 100u8, 102u8, 97u8, 45u8, 115u8, 112u8, 97u8, 114u8, 115u8,
+                            101u8, 0u8, 0u8, 255u8, 254u8, 0u8, 0u8, 2u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 2u8,
+                            2u8, 2u8, 3u8, 4u8, 4u8, 5u8, 6u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 8u8, 9u8, 9u8, 9u8, 10u8, 11u8, 11u8, 12u8,
+                            13u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8,
+                            15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 18u8,
+                            18u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 20u8, 21u8,
+                            21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8,
+                            22u8, 23u8, 23u8, 24u8, 25u8, 25u8, 25u8, 26u8, 27u8, 27u8, 27u8,
+                            27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 40u8, 1u8, 0u8,
+                            0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 128u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 5u8, 0u8, 5u8,
+                            5u8, 6u8, 6u8, 12u8, 12u8, 13u8, 13u8, 0u8, 0u8, 83u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 0u8, 27u8, 0u8, 0u8, 18u8, 0u8,
+                            0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 3u8, 0u8, 6u8, 6u8, 13u8, 13u8,
+                            0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 2u8, 2u8, 4u8, 7u8, 9u8, 9u8,
+                            11u8, 14u8, 19u8, 19u8, 20u8, 20u8, 21u8, 21u8, 22u8, 22u8, 23u8,
+                            23u8, 24u8, 24u8, 25u8, 25u8, 26u8, 26u8, 0u8, 0u8, 68u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            68u8, 0u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 206u8, 0u8, 0u8, 0u8,
+                            221u8, 0u8, 0u8, 0u8, 236u8, 0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            251u8, 0u8, 0u8, 0u8, 10u8, 1u8, 0u8, 0u8, 25u8, 1u8, 0u8, 0u8,
+                            18u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 68u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 17u8, 17u8, 0u8,
+                            0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8,
+                            17u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8,
+                            0u8, 15u8, 16u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 2u8, 0u8, 16u8, 17u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 221u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 15u8, 0u8,
+                            0u8, 221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 4u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8,
+                            35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 9u8,
+                            0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8,
+                            0u8,
+                        ]
+                    } else {
+                        &[
+                            114u8, 117u8, 115u8, 116u8, 45u8, 114u8, 101u8, 103u8, 101u8,
+                            120u8, 45u8, 97u8, 117u8, 116u8, 111u8, 109u8, 97u8, 116u8, 97u8,
+                            45u8, 100u8, 102u8, 97u8, 45u8, 115u8, 112u8, 97u8, 114u8, 115u8,
+                            101u8, 0u8, 0u8, 0u8, 0u8, 254u8, 255u8, 0u8, 0u8, 0u8, 2u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 2u8,
+                            2u8, 2u8, 3u8, 4u8, 4u8, 5u8, 6u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 8u8, 9u8, 9u8, 9u8, 10u8, 11u8, 11u8, 12u8,
+                            13u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8,
+                            15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 18u8,
+                            18u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 20u8, 21u8,
+                            21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8,
+                            22u8, 23u8, 23u8, 24u8, 25u8, 25u8, 25u8, 26u8, 27u8, 27u8, 27u8,
+                            27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 0u8, 0u8, 1u8,
+                            40u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 128u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 5u8, 0u8, 5u8,
+                            5u8, 6u8, 6u8, 12u8, 12u8, 13u8, 13u8, 0u8, 0u8, 83u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 0u8, 27u8, 0u8, 0u8, 18u8, 0u8,
+                            0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 3u8, 0u8, 6u8, 6u8, 13u8, 13u8,
+                            0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 2u8, 2u8, 4u8, 7u8, 9u8, 9u8,
+                            11u8, 14u8, 19u8, 19u8, 20u8, 20u8, 21u8, 21u8, 22u8, 22u8, 23u8,
+                            23u8, 24u8, 24u8, 25u8, 25u8, 26u8, 26u8, 0u8, 0u8, 68u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            68u8, 0u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 206u8, 0u8, 0u8, 0u8,
+                            221u8, 0u8, 0u8, 0u8, 236u8, 0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            251u8, 0u8, 0u8, 0u8, 10u8, 1u8, 0u8, 0u8, 25u8, 1u8, 0u8, 0u8,
+                            18u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 68u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 17u8, 17u8, 0u8,
+                            0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8,
+                            17u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8,
+                            0u8, 15u8, 16u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 2u8, 0u8, 16u8, 17u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 221u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 15u8, 0u8,
+                            0u8, 221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            4u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8,
+                            35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8,
+                            0u8, 0u8, 9u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8,
+                            35u8,
+                        ]
+                    },
+                )
+            },
+            pattern: unsafe {
+                ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
+            },
+        }),
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" y ", 3u8)
+        },
+        special_case: Some(::icu_list::provider::SpecialCasePattern {
+            condition: unsafe {
+                ::icu_list::provider::SerdeDFA::from_dfa_bytes_unchecked(
+                    if cfg!(target_endian = "little") {
+                        &[
+                            114u8, 117u8, 115u8, 116u8, 45u8, 114u8, 101u8, 103u8, 101u8,
+                            120u8, 45u8, 97u8, 117u8, 116u8, 111u8, 109u8, 97u8, 116u8, 97u8,
+                            45u8, 100u8, 102u8, 97u8, 45u8, 115u8, 112u8, 97u8, 114u8, 115u8,
+                            101u8, 0u8, 0u8, 255u8, 254u8, 0u8, 0u8, 2u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 2u8,
+                            2u8, 2u8, 3u8, 4u8, 4u8, 5u8, 6u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 8u8, 9u8, 9u8, 9u8, 10u8, 11u8, 11u8, 12u8,
+                            13u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8,
+                            15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 18u8,
+                            18u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 20u8, 21u8,
+                            21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8,
+                            22u8, 23u8, 23u8, 24u8, 25u8, 25u8, 25u8, 26u8, 27u8, 27u8, 27u8,
+                            27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 40u8, 1u8, 0u8,
+                            0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 128u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 5u8, 0u8, 5u8,
+                            5u8, 6u8, 6u8, 12u8, 12u8, 13u8, 13u8, 0u8, 0u8, 83u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 0u8, 27u8, 0u8, 0u8, 18u8, 0u8,
+                            0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 3u8, 0u8, 6u8, 6u8, 13u8, 13u8,
+                            0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 2u8, 2u8, 4u8, 7u8, 9u8, 9u8,
+                            11u8, 14u8, 19u8, 19u8, 20u8, 20u8, 21u8, 21u8, 22u8, 22u8, 23u8,
+                            23u8, 24u8, 24u8, 25u8, 25u8, 26u8, 26u8, 0u8, 0u8, 68u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            68u8, 0u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 206u8, 0u8, 0u8, 0u8,
+                            221u8, 0u8, 0u8, 0u8, 236u8, 0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            251u8, 0u8, 0u8, 0u8, 10u8, 1u8, 0u8, 0u8, 25u8, 1u8, 0u8, 0u8,
+                            18u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 68u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 17u8, 17u8, 0u8,
+                            0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8,
+                            17u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8,
+                            0u8, 15u8, 16u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 2u8, 0u8, 16u8, 17u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 221u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 15u8, 0u8,
+                            0u8, 221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 4u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8,
+                            35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 9u8,
+                            0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8,
+                            0u8,
+                        ]
+                    } else {
+                        &[
+                            114u8, 117u8, 115u8, 116u8, 45u8, 114u8, 101u8, 103u8, 101u8,
+                            120u8, 45u8, 97u8, 117u8, 116u8, 111u8, 109u8, 97u8, 116u8, 97u8,
+                            45u8, 100u8, 102u8, 97u8, 45u8, 115u8, 112u8, 97u8, 114u8, 115u8,
+                            101u8, 0u8, 0u8, 0u8, 0u8, 254u8, 255u8, 0u8, 0u8, 0u8, 2u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 2u8,
+                            2u8, 2u8, 3u8, 4u8, 4u8, 5u8, 6u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 8u8, 9u8, 9u8, 9u8, 10u8, 11u8, 11u8, 12u8,
+                            13u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8,
+                            15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 18u8,
+                            18u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 20u8, 21u8,
+                            21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8,
+                            22u8, 23u8, 23u8, 24u8, 25u8, 25u8, 25u8, 26u8, 27u8, 27u8, 27u8,
+                            27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 0u8, 0u8, 1u8,
+                            40u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 128u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 5u8, 0u8, 5u8,
+                            5u8, 6u8, 6u8, 12u8, 12u8, 13u8, 13u8, 0u8, 0u8, 83u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 0u8, 27u8, 0u8, 0u8, 18u8, 0u8,
+                            0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 3u8, 0u8, 6u8, 6u8, 13u8, 13u8,
+                            0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 2u8, 2u8, 4u8, 7u8, 9u8, 9u8,
+                            11u8, 14u8, 19u8, 19u8, 20u8, 20u8, 21u8, 21u8, 22u8, 22u8, 23u8,
+                            23u8, 24u8, 24u8, 25u8, 25u8, 26u8, 26u8, 0u8, 0u8, 68u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            68u8, 0u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 206u8, 0u8, 0u8, 0u8,
+                            221u8, 0u8, 0u8, 0u8, 236u8, 0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            251u8, 0u8, 0u8, 0u8, 10u8, 1u8, 0u8, 0u8, 25u8, 1u8, 0u8, 0u8,
+                            18u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 68u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 17u8, 17u8, 0u8,
+                            0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8,
+                            17u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8,
+                            0u8, 15u8, 16u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 2u8, 0u8, 16u8, 17u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 221u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 15u8, 0u8,
+                            0u8, 221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            4u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8,
+                            35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8,
+                            0u8, 0u8, 9u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8,
+                            35u8,
+                        ]
+                    },
+                )
+            },
+            pattern: unsafe {
+                ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
+            },
+        }),
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" y ", 3u8)
+        },
+        special_case: Some(::icu_list::provider::SpecialCasePattern {
+            condition: unsafe {
+                ::icu_list::provider::SerdeDFA::from_dfa_bytes_unchecked(
+                    if cfg!(target_endian = "little") {
+                        &[
+                            114u8, 117u8, 115u8, 116u8, 45u8, 114u8, 101u8, 103u8, 101u8,
+                            120u8, 45u8, 97u8, 117u8, 116u8, 111u8, 109u8, 97u8, 116u8, 97u8,
+                            45u8, 100u8, 102u8, 97u8, 45u8, 115u8, 112u8, 97u8, 114u8, 115u8,
+                            101u8, 0u8, 0u8, 255u8, 254u8, 0u8, 0u8, 2u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 2u8,
+                            2u8, 2u8, 3u8, 4u8, 4u8, 5u8, 6u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 8u8, 9u8, 9u8, 9u8, 10u8, 11u8, 11u8, 12u8,
+                            13u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8,
+                            15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 18u8,
+                            18u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 20u8, 21u8,
+                            21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8,
+                            22u8, 23u8, 23u8, 24u8, 25u8, 25u8, 25u8, 26u8, 27u8, 27u8, 27u8,
+                            27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 40u8, 1u8, 0u8,
+                            0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 128u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 5u8, 0u8, 5u8,
+                            5u8, 6u8, 6u8, 12u8, 12u8, 13u8, 13u8, 0u8, 0u8, 83u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 0u8, 27u8, 0u8, 0u8, 18u8, 0u8,
+                            0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 3u8, 0u8, 6u8, 6u8, 13u8, 13u8,
+                            0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 2u8, 2u8, 4u8, 7u8, 9u8, 9u8,
+                            11u8, 14u8, 19u8, 19u8, 20u8, 20u8, 21u8, 21u8, 22u8, 22u8, 23u8,
+                            23u8, 24u8, 24u8, 25u8, 25u8, 26u8, 26u8, 0u8, 0u8, 68u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            68u8, 0u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 206u8, 0u8, 0u8, 0u8,
+                            221u8, 0u8, 0u8, 0u8, 236u8, 0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            251u8, 0u8, 0u8, 0u8, 10u8, 1u8, 0u8, 0u8, 25u8, 1u8, 0u8, 0u8,
+                            18u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 68u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 17u8, 17u8, 0u8,
+                            0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8,
+                            17u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8,
+                            0u8, 15u8, 16u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 2u8, 0u8, 16u8, 17u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 221u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 15u8, 0u8,
+                            0u8, 221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 4u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8,
+                            35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 9u8,
+                            0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8,
+                            0u8,
+                        ]
+                    } else {
+                        &[
+                            114u8, 117u8, 115u8, 116u8, 45u8, 114u8, 101u8, 103u8, 101u8,
+                            120u8, 45u8, 97u8, 117u8, 116u8, 111u8, 109u8, 97u8, 116u8, 97u8,
+                            45u8, 100u8, 102u8, 97u8, 45u8, 115u8, 112u8, 97u8, 114u8, 115u8,
+                            101u8, 0u8, 0u8, 0u8, 0u8, 254u8, 255u8, 0u8, 0u8, 0u8, 2u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 2u8,
+                            2u8, 2u8, 3u8, 4u8, 4u8, 5u8, 6u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 8u8, 9u8, 9u8, 9u8, 10u8, 11u8, 11u8, 12u8,
+                            13u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8,
+                            15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 18u8,
+                            18u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 20u8, 21u8,
+                            21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8,
+                            22u8, 23u8, 23u8, 24u8, 25u8, 25u8, 25u8, 26u8, 27u8, 27u8, 27u8,
+                            27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 0u8, 0u8, 1u8,
+                            40u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 128u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 5u8, 0u8, 5u8,
+                            5u8, 6u8, 6u8, 12u8, 12u8, 13u8, 13u8, 0u8, 0u8, 83u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 0u8, 27u8, 0u8, 0u8, 18u8, 0u8,
+                            0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 3u8, 0u8, 6u8, 6u8, 13u8, 13u8,
+                            0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 2u8, 2u8, 4u8, 7u8, 9u8, 9u8,
+                            11u8, 14u8, 19u8, 19u8, 20u8, 20u8, 21u8, 21u8, 22u8, 22u8, 23u8,
+                            23u8, 24u8, 24u8, 25u8, 25u8, 26u8, 26u8, 0u8, 0u8, 68u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            68u8, 0u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 206u8, 0u8, 0u8, 0u8,
+                            221u8, 0u8, 0u8, 0u8, 236u8, 0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            251u8, 0u8, 0u8, 0u8, 10u8, 1u8, 0u8, 0u8, 25u8, 1u8, 0u8, 0u8,
+                            18u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 68u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 17u8, 17u8, 0u8,
+                            0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8,
+                            17u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8,
+                            0u8, 15u8, 16u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 2u8, 0u8, 16u8, 17u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 221u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 15u8, 0u8,
+                            0u8, 221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            4u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8,
+                            35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8,
+                            0u8, 0u8, 9u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8,
+                            35u8,
+                        ]
+                    },
+                )
+            },
+            pattern: unsafe {
+                ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
+            },
+        }),
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" y ", 3u8)
+        },
+        special_case: Some(::icu_list::provider::SpecialCasePattern {
+            condition: unsafe {
+                ::icu_list::provider::SerdeDFA::from_dfa_bytes_unchecked(
+                    if cfg!(target_endian = "little") {
+                        &[
+                            114u8, 117u8, 115u8, 116u8, 45u8, 114u8, 101u8, 103u8, 101u8,
+                            120u8, 45u8, 97u8, 117u8, 116u8, 111u8, 109u8, 97u8, 116u8, 97u8,
+                            45u8, 100u8, 102u8, 97u8, 45u8, 115u8, 112u8, 97u8, 114u8, 115u8,
+                            101u8, 0u8, 0u8, 255u8, 254u8, 0u8, 0u8, 2u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 2u8,
+                            2u8, 2u8, 3u8, 4u8, 4u8, 5u8, 6u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 8u8, 9u8, 9u8, 9u8, 10u8, 11u8, 11u8, 12u8,
+                            13u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8,
+                            15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 18u8,
+                            18u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 20u8, 21u8,
+                            21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8,
+                            22u8, 23u8, 23u8, 24u8, 25u8, 25u8, 25u8, 26u8, 27u8, 27u8, 27u8,
+                            27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 40u8, 1u8, 0u8,
+                            0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 128u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 5u8, 0u8, 5u8,
+                            5u8, 6u8, 6u8, 12u8, 12u8, 13u8, 13u8, 0u8, 0u8, 83u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 0u8, 27u8, 0u8, 0u8, 18u8, 0u8,
+                            0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 3u8, 0u8, 6u8, 6u8, 13u8, 13u8,
+                            0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 2u8, 2u8, 4u8, 7u8, 9u8, 9u8,
+                            11u8, 14u8, 19u8, 19u8, 20u8, 20u8, 21u8, 21u8, 22u8, 22u8, 23u8,
+                            23u8, 24u8, 24u8, 25u8, 25u8, 26u8, 26u8, 0u8, 0u8, 68u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            68u8, 0u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 206u8, 0u8, 0u8, 0u8,
+                            221u8, 0u8, 0u8, 0u8, 236u8, 0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            251u8, 0u8, 0u8, 0u8, 10u8, 1u8, 0u8, 0u8, 25u8, 1u8, 0u8, 0u8,
+                            18u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 68u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 17u8, 17u8, 0u8,
+                            0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8,
+                            17u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8,
+                            0u8, 15u8, 16u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 2u8, 0u8, 16u8, 17u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 221u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 15u8, 0u8,
+                            0u8, 221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 4u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8,
+                            35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 9u8,
+                            0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8,
+                            0u8,
+                        ]
+                    } else {
+                        &[
+                            114u8, 117u8, 115u8, 116u8, 45u8, 114u8, 101u8, 103u8, 101u8,
+                            120u8, 45u8, 97u8, 117u8, 116u8, 111u8, 109u8, 97u8, 116u8, 97u8,
+                            45u8, 100u8, 102u8, 97u8, 45u8, 115u8, 112u8, 97u8, 114u8, 115u8,
+                            101u8, 0u8, 0u8, 0u8, 0u8, 254u8, 255u8, 0u8, 0u8, 0u8, 2u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 2u8,
+                            2u8, 2u8, 3u8, 4u8, 4u8, 5u8, 6u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8, 7u8,
+                            7u8, 7u8, 7u8, 7u8, 8u8, 9u8, 9u8, 9u8, 10u8, 11u8, 11u8, 12u8,
+                            13u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8, 14u8,
+                            14u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 15u8,
+                            15u8, 15u8, 15u8, 15u8, 15u8, 15u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8, 16u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8,
+                            17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 17u8, 18u8,
+                            18u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8,
+                            19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 19u8, 20u8, 21u8,
+                            21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8, 21u8,
+                            22u8, 23u8, 23u8, 24u8, 25u8, 25u8, 25u8, 26u8, 27u8, 27u8, 27u8,
+                            27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 27u8, 0u8, 0u8, 1u8,
+                            40u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 1u8, 128u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 1u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 5u8, 0u8, 5u8,
+                            5u8, 6u8, 6u8, 12u8, 12u8, 13u8, 13u8, 0u8, 0u8, 83u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 83u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 0u8, 27u8, 0u8, 0u8, 18u8, 0u8,
+                            0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 0u8, 3u8, 0u8, 6u8, 6u8, 13u8, 13u8,
+                            0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 104u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 14u8, 0u8, 0u8, 0u8, 2u8, 2u8, 4u8, 7u8, 9u8, 9u8,
+                            11u8, 14u8, 19u8, 19u8, 20u8, 20u8, 21u8, 21u8, 22u8, 22u8, 23u8,
+                            23u8, 24u8, 24u8, 25u8, 25u8, 26u8, 26u8, 0u8, 0u8, 68u8, 0u8, 0u8,
+                            0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8, 68u8, 0u8, 0u8, 0u8,
+                            68u8, 0u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 206u8, 0u8, 0u8, 0u8,
+                            221u8, 0u8, 0u8, 0u8, 236u8, 0u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            251u8, 0u8, 0u8, 0u8, 10u8, 1u8, 0u8, 0u8, 25u8, 1u8, 0u8, 0u8,
+                            18u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 68u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 17u8, 17u8, 0u8,
+                            0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8,
+                            17u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8,
+                            0u8, 15u8, 16u8, 0u8, 0u8, 191u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 2u8, 0u8, 16u8, 17u8, 0u8, 0u8, 221u8, 0u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 17u8, 0u8, 0u8, 221u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 2u8, 0u8, 15u8, 15u8, 0u8,
+                            0u8, 221u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8,
+                            4u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8,
+                            35u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8,
+                            0u8, 0u8, 9u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8, 0u8, 18u8, 0u8, 0u8,
+                            0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 0u8, 35u8, 0u8, 0u8, 0u8,
+                            35u8,
+                        ]
+                    },
+                )
+            },
+            pattern: unsafe {
+                ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
+            },
+        }),
+    },
+])
diff --git a/compiler/rustc_baked_icu_data/src/data/list/and_v1/fr.rs.data b/compiler/rustc_baked_icu_data/src/data/list/and_v1/fr.rs.data
new file mode 100644
index 000000000..66ec8f600
--- /dev/null
+++ b/compiler/rustc_baked_icu_data/src/data/list/and_v1/fr.rs.data
@@ -0,0 +1,74 @@
+::icu_list::provider::ListFormatterPatternsV1([
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" et ", 4u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" et ", 4u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" et ", 4u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" et ", 4u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+])
diff --git a/compiler/rustc_baked_icu_data/src/data/list/and_v1/it.rs.data b/compiler/rustc_baked_icu_data/src/data/list/and_v1/it.rs.data
new file mode 100644
index 000000000..cbccf1120
--- /dev/null
+++ b/compiler/rustc_baked_icu_data/src/data/list/and_v1/it.rs.data
@@ -0,0 +1,74 @@
+::icu_list::provider::ListFormatterPatternsV1([
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
+        },
+        special_case: None,
+    },
+])
diff --git a/compiler/rustc_baked_icu_data/src/data/list/and_v1/ja.rs.data b/compiler/rustc_baked_icu_data/src/data/list/and_v1/ja.rs.data
new file mode 100644
index 000000000..9fd168375
--- /dev/null
+++ b/compiler/rustc_baked_icu_data/src/data/list/and_v1/ja.rs.data
@@ -0,0 +1,74 @@
+::icu_list::provider::ListFormatterPatternsV1([
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+])
diff --git a/compiler/rustc_baked_icu_data/src/data/list/and_v1/mod.rs b/compiler/rustc_baked_icu_data/src/data/list/and_v1/mod.rs
new file mode 100644
index 000000000..e20941f0c
--- /dev/null
+++ b/compiler/rustc_baked_icu_data/src/data/list/and_v1/mod.rs
@@ -0,0 +1,22 @@
+// @generated
+type DataStruct = <::icu_list::provider::AndListV1Marker as ::icu_provider::DataMarker>::Yokeable;
+pub fn lookup(locale: &icu_provider::DataLocale) -> Option<&'static DataStruct> {
+    static KEYS: [&str; 12usize] =
+        ["en", "es", "fr", "it", "ja", "pt", "ru", "tr", "und", "zh", "zh-Hans", "zh-Hant"];
+    static DATA: [&DataStruct; 12usize] =
+        [&EN, &ES, &FR, &IT, &JA, &PT, &RU, &TR, &UND, &ZH, &ZH, &ZH_HANT];
+    KEYS.binary_search_by(|k| locale.strict_cmp(k.as_bytes()).reverse())
+        .ok()
+        .map(|i| unsafe { *DATA.get_unchecked(i) })
+}
+static EN: DataStruct = include!("en.rs.data");
+static ES: DataStruct = include!("es.rs.data");
+static FR: DataStruct = include!("fr.rs.data");
+static IT: DataStruct = include!("it.rs.data");
+static JA: DataStruct = include!("ja.rs.data");
+static PT: DataStruct = include!("pt.rs.data");
+static RU: DataStruct = include!("ru.rs.data");
+static TR: DataStruct = include!("tr.rs.data");
+static UND: DataStruct = include!("und.rs.data");
+static ZH_HANT: DataStruct = include!("zh-Hant.rs.data");
+static ZH: DataStruct = include!("zh.rs.data");
diff --git a/compiler/rustc_baked_icu_data/src/data/list/and_v1/pt.rs.data b/compiler/rustc_baked_icu_data/src/data/list/and_v1/pt.rs.data
new file mode 100644
index 000000000..403975213
--- /dev/null
+++ b/compiler/rustc_baked_icu_data/src/data/list/and_v1/pt.rs.data
@@ -0,0 +1,74 @@
+::icu_list::provider::ListFormatterPatternsV1([
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" e ", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+])
diff --git a/compiler/rustc_baked_icu_data/src/data/list/and_v1/ru.rs.data b/compiler/rustc_baked_icu_data/src/data/list/and_v1/ru.rs.data
new file mode 100644
index 000000000..933cb85c8
--- /dev/null
+++ b/compiler/rustc_baked_icu_data/src/data/list/and_v1/ru.rs.data
@@ -0,0 +1,74 @@
+::icu_list::provider::ListFormatterPatternsV1([
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" и ", 4u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" и ", 4u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" и ", 4u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" и ", 4u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+])
diff --git a/compiler/rustc_baked_icu_data/src/data/list/and_v1/tr.rs.data b/compiler/rustc_baked_icu_data/src/data/list/and_v1/tr.rs.data
new file mode 100644
index 000000000..286eaf69f
--- /dev/null
+++ b/compiler/rustc_baked_icu_data/src/data/list/and_v1/tr.rs.data
@@ -0,0 +1,74 @@
+::icu_list::provider::ListFormatterPatternsV1([
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" ve ", 4u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" ve ", 4u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" ve ", 4u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(" ve ", 4u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+])
diff --git a/compiler/rustc_baked_icu_data/src/data/list/and_v1/und.rs.data b/compiler/rustc_baked_icu_data/src/data/list/and_v1/und.rs.data
new file mode 100644
index 000000000..2d2c9bcec
--- /dev/null
+++ b/compiler/rustc_baked_icu_data/src/data/list/and_v1/und.rs.data
@@ -0,0 +1,74 @@
+::icu_list::provider::ListFormatterPatternsV1([
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked(", ", 2u8)
+        },
+        special_case: None,
+    },
+])
diff --git a/compiler/rustc_baked_icu_data/src/data/list/and_v1/zh-Hant.rs.data b/compiler/rustc_baked_icu_data/src/data/list/and_v1/zh-Hant.rs.data
new file mode 100644
index 000000000..5d96cc85e
--- /dev/null
+++ b/compiler/rustc_baked_icu_data/src/data/list/and_v1/zh-Hant.rs.data
@@ -0,0 +1,74 @@
+::icu_list::provider::ListFormatterPatternsV1([
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("和", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("和", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("和", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("和", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("和", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("和", 3u8)
+        },
+        special_case: None,
+    },
+])
diff --git a/compiler/rustc_baked_icu_data/src/data/list/and_v1/zh.rs.data b/compiler/rustc_baked_icu_data/src/data/list/and_v1/zh.rs.data
new file mode 100644
index 000000000..4a38374ca
--- /dev/null
+++ b/compiler/rustc_baked_icu_data/src/data/list/and_v1/zh.rs.data
@@ -0,0 +1,74 @@
+::icu_list::provider::ListFormatterPatternsV1([
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("和", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("和", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("和", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("和", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+    ::icu_list::provider::ConditionalListJoinerPattern {
+        default: unsafe {
+            ::icu_list::provider::ListJoinerPattern::from_parts_unchecked("、", 3u8)
+        },
+        special_case: None,
+    },
+])
diff --git a/compiler/rustc_baked_icu_data/src/data/mod.rs b/compiler/rustc_baked_icu_data/src/data/mod.rs
index a6a71c79c..ce33339ad 100644
--- a/compiler/rustc_baked_icu_data/src/data/mod.rs
+++ b/compiler/rustc_baked_icu_data/src/data/mod.rs
@@ -1,90 +1,113 @@
 // @generated
 mod fallback;
 mod list;
-/// This data provider was programmatically generated by [`icu_datagen`](
-/// https://unicode-org.github.io/icu4x-docs/doc/icu_datagen/enum.Out.html#variant.Module).
-#[non_exhaustive]
-pub struct BakedDataProvider;
 use ::icu_provider::prelude::*;
-impl DataProvider<::icu_list::provider::AndListV1Marker> for BakedDataProvider {
-    fn load(
-        &self,
-        req: DataRequest,
-    ) -> Result<DataResponse<::icu_list::provider::AndListV1Marker>, DataError> {
-        Ok(DataResponse {
-            metadata: Default::default(),
-            payload: Some(DataPayload::from_owned(zerofrom::ZeroFrom::zero_from(
-                *list::and_v1::DATA
-                    .get_by(|k| req.locale.strict_cmp(k.as_bytes()).reverse())
+/// Implement [`DataProvider<M>`] on the given struct using the data
+/// hardcoded in this module. This allows the struct to be used with
+/// `icu`'s `_unstable` constructors.
+///
+/// This macro can only be called from its definition-site, i.e. right
+/// after `include!`-ing the generated module.
+///
+/// ```compile_fail
+/// struct MyDataProvider;
+/// include!("/path/to/generated/mod.rs");
+/// impl_data_provider(MyDataProvider);
+/// ```
+#[allow(unused_macros)]
+macro_rules! impl_data_provider {
+    ($ provider : path) => {
+        impl DataProvider<::icu_list::provider::AndListV1Marker> for $provider {
+            fn load(&self, req: DataRequest) -> Result<DataResponse<::icu_list::provider::AndListV1Marker>, DataError> {
+                list::and_v1::lookup(&req.locale)
+                    .map(zerofrom::ZeroFrom::zero_from)
+                    .map(DataPayload::from_owned)
+                    .map(|payload| DataResponse { metadata: Default::default(), payload: Some(payload) })
+                    .ok_or_else(|| DataErrorKind::MissingLocale.with_req(::icu_list::provider::AndListV1Marker::KEY, req))
+            }
+        }
+        impl DataProvider<::icu_provider_adapters::fallback::provider::CollationFallbackSupplementV1Marker> for $provider {
+            fn load(
+                &self,
+                req: DataRequest,
+            ) -> Result<DataResponse<::icu_provider_adapters::fallback::provider::CollationFallbackSupplementV1Marker>, DataError> {
+                fallback::supplement::co_v1::lookup(&req.locale)
+                    .map(zerofrom::ZeroFrom::zero_from)
+                    .map(DataPayload::from_owned)
+                    .map(|payload| DataResponse { metadata: Default::default(), payload: Some(payload) })
                     .ok_or_else(|| {
                         DataErrorKind::MissingLocale
-                            .with_req(::icu_list::provider::AndListV1Marker::KEY, req)
-                    })?,
-            ))),
-        })
-    }
-}
-impl DataProvider<::icu_provider_adapters::fallback::provider::CollationFallbackSupplementV1Marker>
-    for BakedDataProvider
-{
-    fn load(
-        &self,
-        req: DataRequest,
-    ) -> Result<
-        DataResponse<
-            ::icu_provider_adapters::fallback::provider::CollationFallbackSupplementV1Marker,
-        >,
-        DataError,
-    > {
-        Ok(DataResponse {
-            metadata: Default::default(),
-            payload: Some(DataPayload::from_owned(zerofrom::ZeroFrom::zero_from(
-                *fallback::supplement::co_v1::DATA.get_by(|k| req.locale.strict_cmp(k.as_bytes()).reverse()).ok_or_else(|| {
-                    DataErrorKind::MissingLocale.with_req(::icu_provider_adapters::fallback::provider::CollationFallbackSupplementV1Marker::KEY, req)
-                })?,
-            ))),
-        })
-    }
-}
-impl DataProvider<::icu_provider_adapters::fallback::provider::LocaleFallbackLikelySubtagsV1Marker>
-    for BakedDataProvider
-{
-    fn load(
-        &self,
-        req: DataRequest,
-    ) -> Result<
-        DataResponse<
-            ::icu_provider_adapters::fallback::provider::LocaleFallbackLikelySubtagsV1Marker,
-        >,
-        DataError,
-    > {
-        Ok(DataResponse {
-            metadata: Default::default(),
-            payload: Some(DataPayload::from_owned(zerofrom::ZeroFrom::zero_from(
-                *fallback::likelysubtags_v1::DATA.get_by(|k| req.locale.strict_cmp(k.as_bytes()).reverse()).ok_or_else(|| {
-                    DataErrorKind::MissingLocale.with_req(::icu_provider_adapters::fallback::provider::LocaleFallbackLikelySubtagsV1Marker::KEY, req)
-                })?,
-            ))),
-        })
-    }
+                            .with_req(::icu_provider_adapters::fallback::provider::CollationFallbackSupplementV1Marker::KEY, req)
+                    })
+            }
+        }
+        impl DataProvider<::icu_provider_adapters::fallback::provider::LocaleFallbackLikelySubtagsV1Marker> for $provider {
+            fn load(
+                &self,
+                req: DataRequest,
+            ) -> Result<DataResponse<::icu_provider_adapters::fallback::provider::LocaleFallbackLikelySubtagsV1Marker>, DataError> {
+                fallback::likelysubtags_v1::lookup(&req.locale)
+                    .map(zerofrom::ZeroFrom::zero_from)
+                    .map(DataPayload::from_owned)
+                    .map(|payload| DataResponse { metadata: Default::default(), payload: Some(payload) })
+                    .ok_or_else(|| {
+                        DataErrorKind::MissingLocale
+                            .with_req(::icu_provider_adapters::fallback::provider::LocaleFallbackLikelySubtagsV1Marker::KEY, req)
+                    })
+            }
+        }
+        impl DataProvider<::icu_provider_adapters::fallback::provider::LocaleFallbackParentsV1Marker> for $provider {
+            fn load(
+                &self,
+                req: DataRequest,
+            ) -> Result<DataResponse<::icu_provider_adapters::fallback::provider::LocaleFallbackParentsV1Marker>, DataError> {
+                fallback::parents_v1::lookup(&req.locale)
+                    .map(zerofrom::ZeroFrom::zero_from)
+                    .map(DataPayload::from_owned)
+                    .map(|payload| DataResponse { metadata: Default::default(), payload: Some(payload) })
+                    .ok_or_else(|| {
+                        DataErrorKind::MissingLocale.with_req(::icu_provider_adapters::fallback::provider::LocaleFallbackParentsV1Marker::KEY, req)
+                    })
+            }
+        }
+    };
 }
-impl DataProvider<::icu_provider_adapters::fallback::provider::LocaleFallbackParentsV1Marker>
-    for BakedDataProvider
-{
-    fn load(
-        &self,
-        req: DataRequest,
-    ) -> Result<
-        DataResponse<::icu_provider_adapters::fallback::provider::LocaleFallbackParentsV1Marker>,
-        DataError,
-    > {
-        Ok(DataResponse {
-            metadata: Default::default(),
-            payload: Some(DataPayload::from_owned(zerofrom::ZeroFrom::zero_from(
-                *fallback::parents_v1::DATA.get_by(|k| req.locale.strict_cmp(k.as_bytes()).reverse()).ok_or_else(|| {
-                    DataErrorKind::MissingLocale.with_req(::icu_provider_adapters::fallback::provider::LocaleFallbackParentsV1Marker::KEY, req)
-                })?,
-            ))),
-        })
-    }
+/// Implement [`AnyProvider`] on the given struct using the data
+/// hardcoded in this module. This allows the struct to be used with
+/// `icu`'s `_any` constructors.
+///
+/// This macro can only be called from its definition-site, i.e. right
+/// after `include!`-ing the generated module.
+///
+/// ```compile_fail
+/// struct MyAnyProvider;
+/// include!("/path/to/generated/mod.rs");
+/// impl_any_provider(MyAnyProvider);
+/// ```
+#[allow(unused_macros)]
+macro_rules! impl_any_provider {
+    ($ provider : path) => {
+        impl AnyProvider for $provider {
+            fn load_any(&self, key: DataKey, req: DataRequest) -> Result<AnyResponse, DataError> {
+                const ANDLISTV1MARKER: ::icu_provider::DataKeyHash = ::icu_list::provider::AndListV1Marker::KEY.hashed();
+                const COLLATIONFALLBACKSUPPLEMENTV1MARKER: ::icu_provider::DataKeyHash =
+                    ::icu_provider_adapters::fallback::provider::CollationFallbackSupplementV1Marker::KEY.hashed();
+                const LOCALEFALLBACKLIKELYSUBTAGSV1MARKER: ::icu_provider::DataKeyHash =
+                    ::icu_provider_adapters::fallback::provider::LocaleFallbackLikelySubtagsV1Marker::KEY.hashed();
+                const LOCALEFALLBACKPARENTSV1MARKER: ::icu_provider::DataKeyHash =
+                    ::icu_provider_adapters::fallback::provider::LocaleFallbackParentsV1Marker::KEY.hashed();
+                match key.hashed() {
+                    ANDLISTV1MARKER => list::and_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
+                    COLLATIONFALLBACKSUPPLEMENTV1MARKER => fallback::supplement::co_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
+                    LOCALEFALLBACKLIKELYSUBTAGSV1MARKER => fallback::likelysubtags_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
+                    LOCALEFALLBACKPARENTSV1MARKER => fallback::parents_v1::lookup(&req.locale).map(AnyPayload::from_static_ref),
+                    _ => return Err(DataErrorKind::MissingDataKey.with_req(key, req)),
+                }
+                .map(|payload| AnyResponse { payload: Some(payload), metadata: Default::default() })
+                .ok_or_else(|| DataErrorKind::MissingLocale.with_req(key, req))
+            }
+        }
+    };
 }
+pub struct BakedDataProvider;
+impl_data_provider!(BakedDataProvider);
diff --git a/compiler/rustc_error_messages/locales/en-US/borrowck.ftl b/compiler/rustc_borrowck/locales/en-US.ftl
index 9e4332c42..a3b6b5e81 100644
--- a/compiler/rustc_error_messages/locales/en-US/borrowck.ftl
+++ b/compiler/rustc_borrowck/locales/en-US.ftl
@@ -18,7 +18,7 @@ borrowck_generic_does_not_live_long_enough =
     `{$kind}` does not live long enough
 
 borrowck_move_borrowed =
-    cannot move out of `{$desc}` beacause it is borrowed
+    cannot move out of `{$desc}` because it is borrowed
 
 borrowck_var_does_not_need_mut =
     variable does not need to be mutable
@@ -33,7 +33,7 @@ borrowck_var_here_defined = variable defined here
 
 borrowck_var_here_captured = variable captured here
 
-borrowck_closure_inferred_mut =  inferred to be a `FnMut` closure
+borrowck_closure_inferred_mut = inferred to be a `FnMut` closure
 
 borrowck_returned_closure_escaped =
     returns a closure that contains a reference to a captured variable, which then escapes the closure body
@@ -87,10 +87,10 @@ borrowck_use_due_to_use_closure =
     use occurs due to use in closure
 
 borrowck_assign_due_to_use_closure =
-    assign occurs due to use in closure
+    assignment occurs due to use in closure
 
 borrowck_assign_part_due_to_use_closure =
-    assign to part occurs due to use in closure
+    assignment to part occurs due to use in closure
 
 borrowck_capture_immute =
     capture is immutable because of use here
@@ -123,4 +123,7 @@ borrowck_cannot_move_when_borrowed =
 
 borrowck_opaque_type_non_generic_param =
     expected generic {$kind} parameter, found `{$ty}`
-    .label = this generic parameter must be used with a generic {$kind} parameter
+    .label = {STREQ($ty, "'static") ->
+        [true] cannot use static lifetime; use a bound lifetime instead or remove the lifetime parameter from the opaque type
+        *[other] this generic parameter must be used with a generic {$kind} parameter
+    }
diff --git a/compiler/rustc_borrowck/src/borrowck_errors.rs b/compiler/rustc_borrowck/src/borrowck_errors.rs
index a4943d112..2bbb9618d 100644
--- a/compiler/rustc_borrowck/src/borrowck_errors.rs
+++ b/compiler/rustc_borrowck/src/borrowck_errors.rs
@@ -37,7 +37,7 @@ impl<'cx, 'tcx> crate::MirBorrowckCtxt<'cx, 'tcx> {
             desc,
         );
 
-        err.span_label(borrow_span, format!("borrow of {} occurs here", borrow_desc));
+        err.span_label(borrow_span, format!("{} is borrowed here", borrow_desc));
         err.span_label(span, format!("use of borrowed {}", borrow_desc));
         err
     }
@@ -250,8 +250,8 @@ impl<'cx, 'tcx> crate::MirBorrowckCtxt<'cx, 'tcx> {
             desc,
         );
 
-        err.span_label(borrow_span, format!("borrow of {} occurs here", desc));
-        err.span_label(span, format!("assignment to borrowed {} occurs here", desc));
+        err.span_label(borrow_span, format!("{} is borrowed here", desc));
+        err.span_label(span, format!("{} is assigned to here but it was already borrowed", desc));
         err
     }
 
diff --git a/compiler/rustc_borrowck/src/constraint_generation.rs b/compiler/rustc_borrowck/src/constraint_generation.rs
index 11b31c3f1..1427f5cb3 100644
--- a/compiler/rustc_borrowck/src/constraint_generation.rs
+++ b/compiler/rustc_borrowck/src/constraint_generation.rs
@@ -9,7 +9,7 @@ use rustc_middle::mir::{
 };
 use rustc_middle::ty::subst::SubstsRef;
 use rustc_middle::ty::visit::TypeVisitable;
-use rustc_middle::ty::{self, RegionVid, Ty};
+use rustc_middle::ty::{self, RegionVid, Ty, TyCtxt};
 
 use crate::{
     borrow_set::BorrowSet, facts::AllFacts, location::LocationTable, nll::ToRegionVid,
@@ -165,7 +165,7 @@ impl<'cx, 'tcx> ConstraintGeneration<'cx, 'tcx> {
     /// `location`.
     fn add_regular_live_constraint<T>(&mut self, live_ty: T, location: Location)
     where
-        T: TypeVisitable<'tcx>,
+        T: TypeVisitable<TyCtxt<'tcx>>,
     {
         debug!("add_regular_live_constraint(live_ty={:?}, location={:?})", live_ty, location);
 
diff --git a/compiler/rustc_borrowck/src/constraints/mod.rs b/compiler/rustc_borrowck/src/constraints/mod.rs
index 1f0b8adea..f370c0216 100644
--- a/compiler/rustc_borrowck/src/constraints/mod.rs
+++ b/compiler/rustc_borrowck/src/constraints/mod.rs
@@ -17,7 +17,7 @@ pub(crate) mod graph;
 /// constraints of the form `R1: R2`. Each constraint is identified by
 /// a unique `OutlivesConstraintIndex` and you can index into the set
 /// (`constraint_set[i]`) to access the constraint details.
-#[derive(Clone, Default)]
+#[derive(Clone, Debug, Default)]
 pub(crate) struct OutlivesConstraintSet<'tcx> {
     outlives: IndexVec<OutlivesConstraintIndex, OutlivesConstraint<'tcx>>,
 }
diff --git a/compiler/rustc_borrowck/src/dataflow.rs b/compiler/rustc_borrowck/src/dataflow.rs
index 8c4885770..2821677c5 100644
--- a/compiler/rustc_borrowck/src/dataflow.rs
+++ b/compiler/rustc_borrowck/src/dataflow.rs
@@ -393,6 +393,7 @@ impl<'tcx> rustc_mir_dataflow::GenKillAnalysis<'tcx> for Borrows<'_, 'tcx> {
             | mir::StatementKind::AscribeUserType(..)
             | mir::StatementKind::Coverage(..)
             | mir::StatementKind::Intrinsic(..)
+            | mir::StatementKind::ConstEvalCounter
             | mir::StatementKind::Nop => {}
         }
     }
diff --git a/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs b/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs
index 1550958ab..68205fa45 100644
--- a/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs
@@ -106,7 +106,7 @@ impl<'tcx> ToUniverseInfo<'tcx>
     }
 }
 
-impl<'tcx, T: Copy + fmt::Display + TypeFoldable<'tcx> + 'tcx> ToUniverseInfo<'tcx>
+impl<'tcx, T: Copy + fmt::Display + TypeFoldable<TyCtxt<'tcx>> + 'tcx> ToUniverseInfo<'tcx>
     for Canonical<'tcx, ty::ParamEnvAnd<'tcx, type_op::Normalize<T>>>
 {
     fn to_universe_info(self, base_universe: ty::UniverseIndex) -> UniverseInfo<'tcx> {
@@ -180,20 +180,20 @@ trait TypeOpInfo<'tcx> {
             return;
         };
 
-        let placeholder_region = tcx.mk_region(ty::RePlaceholder(ty::Placeholder {
+        let placeholder_region = tcx.mk_re_placeholder(ty::Placeholder {
             name: placeholder.name,
             universe: adjusted_universe.into(),
-        }));
+        });
 
         let error_region =
             if let RegionElement::PlaceholderRegion(error_placeholder) = error_element {
                 let adjusted_universe =
                     error_placeholder.universe.as_u32().checked_sub(base_universe.as_u32());
                 adjusted_universe.map(|adjusted| {
-                    tcx.mk_region(ty::RePlaceholder(ty::Placeholder {
+                    tcx.mk_re_placeholder(ty::Placeholder {
                         name: error_placeholder.name,
                         universe: adjusted.into(),
-                    }))
+                    })
                 })
             } else {
                 None
@@ -258,7 +258,7 @@ struct NormalizeQuery<'tcx, T> {
 
 impl<'tcx, T> TypeOpInfo<'tcx> for NormalizeQuery<'tcx, T>
 where
-    T: Copy + fmt::Display + TypeFoldable<'tcx> + 'tcx,
+    T: Copy + fmt::Display + TypeFoldable<TyCtxt<'tcx>> + 'tcx,
 {
     fn fallback_error(
         &self,
@@ -390,7 +390,7 @@ fn try_extract_error_from_fulfill_cx<'tcx>(
         error_region,
         &region_constraints,
         |vid| ocx.infcx.region_var_origin(vid),
-        |vid| ocx.infcx.universe_of_region(ocx.infcx.tcx.mk_region(ty::ReVar(vid))),
+        |vid| ocx.infcx.universe_of_region(ocx.infcx.tcx.mk_re_var(vid)),
     )
 }
 
@@ -411,7 +411,7 @@ fn try_extract_error_from_region_constraints<'tcx>(
                 }
                 // FIXME: Should this check the universe of the var?
                 Constraint::VarSubReg(vid, sup) if sup == placeholder_region => {
-                    Some((infcx.tcx.mk_region(ty::ReVar(vid)), cause.clone()))
+                    Some((infcx.tcx.mk_re_var(vid), cause.clone()))
                 }
                 _ => None,
             }
diff --git a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
index e5a36259f..cb97699d7 100644
--- a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
@@ -6,7 +6,7 @@ use rustc_errors::{
     struct_span_err, Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed, MultiSpan,
 };
 use rustc_hir as hir;
-use rustc_hir::def::Res;
+use rustc_hir::def::{DefKind, Res};
 use rustc_hir::intravisit::{walk_block, walk_expr, Visitor};
 use rustc_hir::{AsyncGeneratorKind, GeneratorKind, LangItem};
 use rustc_infer::infer::TyCtxtInferExt;
@@ -236,10 +236,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
             let ty = used_place.ty(self.body, self.infcx.tcx).ty;
             let needs_note = match ty.kind() {
                 ty::Closure(id, _) => {
-                    let tables = self.infcx.tcx.typeck(id.expect_local());
-                    let hir_id = self.infcx.tcx.hir().local_def_id_to_hir_id(id.expect_local());
-
-                    tables.closure_kind_origins().get(hir_id).is_none()
+                    self.infcx.tcx.closure_kind_origin(id.expect_local()).is_none()
                 }
                 _ => true,
             };
@@ -766,7 +763,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         let copy_did = infcx.tcx.require_lang_item(LangItem::Copy, Some(span));
         let cause = ObligationCause::new(
             span,
-            self.mir_hir_id(),
+            self.mir_def_id(),
             rustc_infer::traits::ObligationCauseCode::MiscObligation,
         );
         let errors = rustc_trait_selection::traits::fully_solve_bound(
@@ -803,6 +800,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 predicates
                     .iter()
                     .map(|(param, constraint)| (param.name.as_str(), &**constraint, None)),
+                None,
             );
         }
     }
@@ -1185,11 +1183,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 return None;
             };
             debug!("checking call args for uses of inner_param: {:?}", args);
-            if args.contains(&Operand::Move(inner_param)) {
-                Some((loc, term))
-            } else {
-                None
-            }
+            args.contains(&Operand::Move(inner_param)).then_some((loc, term))
         }) else {
             debug!("no uses of inner_param found as a by-move call arg");
             return;
@@ -1497,7 +1491,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         assert!(root_place.projection.is_empty());
         let proper_span = self.body.local_decls[root_place.local].source_info.span;
 
-        let root_place_projection = self.infcx.tcx.intern_place_elems(root_place.projection);
+        let root_place_projection = self.infcx.tcx.mk_place_elems(root_place.projection);
 
         if self.access_place_error_reported.contains(&(
             Place { local: root_place.local, projection: root_place_projection },
@@ -1673,7 +1667,6 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 format!("`{}` would have to be valid for `{}`...", name, region_name),
             );
 
-            let fn_hir_id = self.mir_hir_id();
             err.span_label(
                 drop_span,
                 format!(
@@ -1681,19 +1674,12 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                     name,
                     self.infcx
                         .tcx
-                        .hir()
-                        .opt_name(fn_hir_id)
+                        .opt_item_name(self.mir_def_id().to_def_id())
                         .map(|name| format!("function `{}`", name))
                         .unwrap_or_else(|| {
-                            match &self
-                                .infcx
-                                .tcx
-                                .typeck(self.mir_def_id())
-                                .node_type(fn_hir_id)
-                                .kind()
-                            {
-                                ty::Closure(..) => "enclosing closure",
-                                ty::Generator(..) => "enclosing generator",
+                            match &self.infcx.tcx.def_kind(self.mir_def_id()) {
+                                DefKind::Closure => "enclosing closure",
+                                DefKind::Generator => "enclosing generator",
                                 kind => bug!("expected closure or generator, found {:?}", kind),
                             }
                             .to_string()
@@ -1736,7 +1722,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 &self.local_names,
                 &mut err,
                 "",
-                None,
+                Some(borrow_span),
                 None,
             );
         }
@@ -2138,7 +2124,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
     ) -> DiagnosticBuilder<'cx, ErrorGuaranteed> {
         let tcx = self.infcx.tcx;
 
-        let (_, escapes_from) = tcx.article_and_description(self.mir_def_id().to_def_id());
+        let escapes_from = tcx.def_descr(self.mir_def_id().to_def_id());
 
         let mut err =
             borrowck_errors::borrowed_data_escapes_closure(tcx, escape_span, escapes_from);
@@ -2595,11 +2581,11 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
             if is_closure {
                 None
             } else {
-                let ty = self.infcx.tcx.type_of(self.mir_def_id());
+                let ty = self.infcx.tcx.type_of(self.mir_def_id()).subst_identity();
                 match ty.kind() {
                     ty::FnDef(_, _) | ty::FnPtr(_) => self.annotate_fn_sig(
                         self.mir_def_id(),
-                        self.infcx.tcx.fn_sig(self.mir_def_id()),
+                        self.infcx.tcx.fn_sig(self.mir_def_id()).subst_identity(),
                     ),
                     _ => None,
                 }
@@ -2645,6 +2631,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                             operands,
                         ) = rvalue
                         {
+                            let def_id = def_id.expect_local();
                             for operand in operands {
                                 let (Operand::Copy(assigned_from) | Operand::Move(assigned_from)) = operand else {
                                     continue;
@@ -2667,7 +2654,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                                 // into a place then we should annotate the closure in
                                 // case it ends up being assigned into the return place.
                                 annotated_closure =
-                                    self.annotate_fn_sig(*def_id, substs.as_closure().sig());
+                                    self.annotate_fn_sig(def_id, substs.as_closure().sig());
                                 debug!(
                                     "annotate_argument_and_return_for_borrow: \
                                      annotated_closure={:?} assigned_from_local={:?} \
diff --git a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
index 209574709..19855075c 100644
--- a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
@@ -1,6 +1,8 @@
 //! Print diagnostics to explain why values are borrowed.
 
 use rustc_errors::{Applicability, Diagnostic};
+use rustc_hir as hir;
+use rustc_hir::intravisit::Visitor;
 use rustc_index::vec::IndexVec;
 use rustc_infer::infer::NllRegionVariableOrigin;
 use rustc_middle::mir::{
@@ -11,6 +13,7 @@ use rustc_middle::ty::adjustment::PointerCast;
 use rustc_middle::ty::{self, RegionVid, TyCtxt};
 use rustc_span::symbol::{kw, Symbol};
 use rustc_span::{sym, DesugaringKind, Span};
+use rustc_trait_selection::traits::error_reporting::FindExprBySpan;
 
 use crate::region_infer::{BlameConstraint, ExtraConstraintInfo};
 use crate::{
@@ -63,6 +66,36 @@ impl<'tcx> BorrowExplanation<'tcx> {
         borrow_span: Option<Span>,
         multiple_borrow_span: Option<(Span, Span)>,
     ) {
+        if let Some(span) = borrow_span {
+            let def_id = body.source.def_id();
+            if let Some(node) = tcx.hir().get_if_local(def_id)
+                && let Some(body_id) = node.body_id()
+            {
+                let body = tcx.hir().body(body_id);
+                let mut expr_finder = FindExprBySpan::new(span);
+                expr_finder.visit_expr(body.value);
+                if let Some(mut expr) = expr_finder.result {
+                    while let hir::ExprKind::AddrOf(_, _, inner)
+                        | hir::ExprKind::Unary(hir::UnOp::Deref, inner)
+                        | hir::ExprKind::Field(inner, _)
+                        | hir::ExprKind::MethodCall(_, inner, _, _)
+                        | hir::ExprKind::Index(inner, _) = &expr.kind
+                    {
+                        expr = inner;
+                    }
+                    if let hir::ExprKind::Path(hir::QPath::Resolved(None, p)) = expr.kind
+                        && let [hir::PathSegment { ident, args: None, .. }] = p.segments
+                        && let hir::def::Res::Local(hir_id) = p.res
+                        && let Some(hir::Node::Pat(pat)) = tcx.hir().find(hir_id)
+                    {
+                        err.span_label(
+                            pat.span,
+                            &format!("binding `{ident}` declared here"),
+                        );
+                    }
+                }
+            }
+        }
         match *self {
             BorrowExplanation::UsedLater(later_use_kind, var_or_use_span, path_span) => {
                 let message = match later_use_kind {
diff --git a/compiler/rustc_borrowck/src/diagnostics/mod.rs b/compiler/rustc_borrowck/src/diagnostics/mod.rs
index 1b40b7143..a99fd594a 100644
--- a/compiler/rustc_borrowck/src/diagnostics/mod.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/mod.rs
@@ -115,11 +115,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                     debug!("add_moved_or_invoked_closure_note: closure={:?}", closure);
                     if let ty::Closure(did, _) = self.body.local_decls[closure].ty.kind() {
                         let did = did.expect_local();
-                        let hir_id = self.infcx.tcx.hir().local_def_id_to_hir_id(did);
-
-                        if let Some((span, hir_place)) =
-                            self.infcx.tcx.typeck(did).closure_kind_origins().get(hir_id)
-                        {
+                        if let Some((span, hir_place)) = self.infcx.tcx.closure_kind_origin(did) {
                             diag.span_note(
                                 *span,
                                 &format!(
@@ -139,11 +135,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         if let Some(target) = target {
             if let ty::Closure(did, _) = self.body.local_decls[target].ty.kind() {
                 let did = did.expect_local();
-                let hir_id = self.infcx.tcx.hir().local_def_id_to_hir_id(did);
-
-                if let Some((span, hir_place)) =
-                    self.infcx.tcx.typeck(did).closure_kind_origins().get(hir_id)
-                {
+                if let Some((span, hir_place)) = self.infcx.tcx.closure_kind_origin(did) {
                     diag.span_note(
                         *span,
                         &format!(
@@ -373,14 +365,8 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                     //
                     // We know the field exists so it's safe to call operator[] and `unwrap` here.
                     let def_id = def_id.expect_local();
-                    let var_id = self
-                        .infcx
-                        .tcx
-                        .typeck(def_id)
-                        .closure_min_captures_flattened(def_id)
-                        .nth(field.index())
-                        .unwrap()
-                        .get_root_variable();
+                    let var_id =
+                        self.infcx.tcx.closure_captures(def_id)[field.index()].get_root_variable();
 
                     Some(self.infcx.tcx.hir().name(var_id).to_string())
                 }
@@ -817,6 +803,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
             && let AggregateKind::Closure(def_id, _) | AggregateKind::Generator(def_id, _, _) = **kind
         {
             debug!("move_spans: def_id={:?} places={:?}", def_id, places);
+            let def_id = def_id.expect_local();
             if let Some((args_span, generator_kind, capture_kind_span, path_span)) =
                 self.closure_span(def_id, moved_place, places)
             {
@@ -945,6 +932,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                     box AggregateKind::Generator(def_id, _, _) => (def_id, true),
                     _ => continue,
                 };
+                let def_id = def_id.expect_local();
 
                 debug!(
                     "borrow_spans: def_id={:?} is_generator={:?} places={:?}",
@@ -985,7 +973,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         debug!("closure_span: hir_id={:?} expr={:?}", hir_id, expr);
         if let hir::ExprKind::Closure(&hir::Closure { body, fn_decl_span, .. }) = expr {
             for (captured_place, place) in
-                self.infcx.tcx.typeck(def_id).closure_min_captures_flattened(def_id).zip(places)
+                self.infcx.tcx.closure_captures(def_id).iter().zip(places)
             {
                 match place {
                     Operand::Copy(place) | Operand::Move(place)
@@ -1064,7 +1052,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                         );
                     }
                 }
-                CallKind::Normal { self_arg, desugaring, method_did } => {
+                CallKind::Normal { self_arg, desugaring, method_did, method_substs } => {
                     let self_arg = self_arg.unwrap();
                     let tcx = self.infcx.tcx;
                     if let Some((CallDesugaringKind::ForLoopIntoIter, _)) = desugaring {
@@ -1128,17 +1116,21 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                                 "{place_name} {partially_str}moved due to this method call{loop_message}",
                             ),
                         );
+
                         let infcx = tcx.infer_ctxt().build();
+                        // Erase and shadow everything that could be passed to the new infcx.
                         let ty = tcx.erase_regions(moved_place.ty(self.body, tcx).ty);
+                        let method_substs = tcx.erase_regions(method_substs);
+
                         if let ty::Adt(def, substs) = ty.kind()
                             && Some(def.did()) == tcx.lang_items().pin_type()
                             && let ty::Ref(_, _, hir::Mutability::Mut) = substs.type_at(0).kind()
-                            && let self_ty = infcx.replace_bound_vars_with_fresh_vars(
+                            && let self_ty = infcx.instantiate_binder_with_fresh_vars(
                                 fn_call_span,
                                 LateBoundRegionConversionTime::FnCall,
-                                tcx.fn_sig(method_did).input(0),
+                                tcx.fn_sig(method_did).subst(tcx, method_substs).input(0),
                             )
-                            && infcx.can_eq(self.param_env, ty, self_ty).is_ok()
+                            && infcx.can_eq(self.param_env, ty, self_ty)
                         {
                             err.span_suggestion_verbose(
                                 fn_call_span.shrink_to_lo(),
@@ -1176,13 +1168,13 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                         );
                     }
                     let parent_did = tcx.parent(method_did);
-                    let parent_self_ty = (tcx.def_kind(parent_did)
-                        == rustc_hir::def::DefKind::Impl)
-                        .then_some(parent_did)
-                        .and_then(|did| match tcx.type_of(did).kind() {
-                            ty::Adt(def, ..) => Some(def.did()),
-                            _ => None,
-                        });
+                    let parent_self_ty =
+                        matches!(tcx.def_kind(parent_did), rustc_hir::def::DefKind::Impl { .. })
+                            .then_some(parent_did)
+                            .and_then(|did| match tcx.type_of(did).subst_identity().kind() {
+                                ty::Adt(def, ..) => Some(def.did()),
+                                _ => None,
+                            });
                     let is_option_or_result = parent_self_ty.map_or(false, |def_id| {
                         matches!(tcx.get_diagnostic_name(def_id), Some(sym::Option | sym::Result))
                     });
diff --git a/compiler/rustc_borrowck/src/diagnostics/move_errors.rs b/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
index 6db3c858a..5e4c7292e 100644
--- a/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
@@ -448,7 +448,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                 };
                 self.note_type_does_not_implement_copy(err, &place_desc, place_ty, Some(span), "");
 
-                use_spans.args_span_label(err, format!("move out of {place_desc} occurs here"));
+                use_spans.args_span_label(err, format!("{place_desc} is moved here"));
             }
         }
     }
@@ -467,7 +467,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                 err.span_suggestion_verbose(
                     span.shrink_to_lo(),
                     "consider borrowing here",
-                    "&".to_string(),
+                    '&',
                     Applicability::MaybeIncorrect,
                 );
             }
diff --git a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
index 45b15c2c5..328ac880d 100644
--- a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
@@ -344,7 +344,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                     } else {
                         err.span_help(source_info.span, "try removing `&mut` here");
                     }
-                } else if decl.mutability == Mutability::Not {
+                } else if decl.mutability.is_not() {
                     if matches!(
                         decl.local_info,
                         Some(box LocalInfo::User(ClearCrossCrate::Set(BindingForm::ImplicitSelf(
@@ -385,7 +385,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                     err.span_suggestion_verbose(
                         local_decl.source_info.span.shrink_to_lo(),
                         "consider changing this to be mutable",
-                        "mut ".to_string(),
+                        "mut ",
                         Applicability::MachineApplicable,
                     );
                     let tcx = self.infcx.tcx;
@@ -606,12 +606,63 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                                 }
                             }
                             Some((false, err_label_span, message)) => {
-                                err.span_label(
-                                    err_label_span,
-                                    &format!(
-                                        "consider changing this binding's type to be: `{message}`"
-                                    ),
-                                );
+                                struct BindingFinder {
+                                    span: Span,
+                                    hir_id: Option<hir::HirId>,
+                                }
+
+                                impl<'tcx> Visitor<'tcx> for BindingFinder {
+                                    fn visit_stmt(&mut self, s: &'tcx hir::Stmt<'tcx>) {
+                                        if let hir::StmtKind::Local(local) = s.kind {
+                                            if local.pat.span == self.span {
+                                                self.hir_id = Some(local.hir_id);
+                                            }
+                                        }
+                                        hir::intravisit::walk_stmt(self, s);
+                                    }
+                                }
+                                let hir_map = self.infcx.tcx.hir();
+                                let def_id = self.body.source.def_id();
+                                let hir_id = hir_map.local_def_id_to_hir_id(def_id.expect_local());
+                                let node = hir_map.find(hir_id);
+                                let hir_id = if let Some(hir::Node::Item(item)) = node
+                                    && let hir::ItemKind::Fn(.., body_id) = item.kind
+                                {
+                                    let body = hir_map.body(body_id);
+                                    let mut v = BindingFinder {
+                                        span: err_label_span,
+                                        hir_id: None,
+                                    };
+                                    v.visit_body(body);
+                                    v.hir_id
+                                } else {
+                                    None
+                                };
+                                if let Some(hir_id) = hir_id
+                                    && let Some(hir::Node::Local(local)) = hir_map.find(hir_id)
+                                {
+                                    let (changing, span, sugg) = match local.ty {
+                                        Some(ty) => ("changing", ty.span, message),
+                                        None => (
+                                            "specifying",
+                                            local.pat.span.shrink_to_hi(),
+                                            format!(": {message}"),
+                                        ),
+                                    };
+                                    err.span_suggestion_verbose(
+                                        span,
+                                        &format!("consider {changing} this binding's type"),
+                                        sugg,
+                                        Applicability::HasPlaceholders,
+                                    );
+                                } else {
+                                    err.span_label(
+                                        err_label_span,
+                                        &format!(
+                                            "consider changing this binding's type to be: `{message}`"
+                                        ),
+                                    );
+                                }
                             }
                             None => {}
                         }
@@ -850,10 +901,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
         err: &mut Diagnostic,
     ) {
         let tables = tcx.typeck(closure_local_def_id);
-        let closure_hir_id = tcx.hir().local_def_id_to_hir_id(closure_local_def_id);
-        if let Some((span, closure_kind_origin)) =
-            &tables.closure_kind_origins().get(closure_hir_id)
-        {
+        if let Some((span, closure_kind_origin)) = tcx.closure_kind_origin(closure_local_def_id) {
             let reason = if let PlaceBase::Upvar(upvar_id) = closure_kind_origin.base {
                 let upvar = ty::place_to_string_for_capture(tcx, closure_kind_origin);
                 let root_hir_id = upvar_id.var_path.hir_id;
diff --git a/compiler/rustc_borrowck/src/diagnostics/region_errors.rs b/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
index 187861ba1..ffe82b46c 100644
--- a/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
@@ -415,7 +415,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
     /// fn foo<'a, 'b>(x: &'a u32) -> &'b u32 { x }
     /// ```
     ///
-    /// Here we would be invoked with `fr = 'a` and `outlived_fr = `'b`.
+    /// Here we would be invoked with `fr = 'a` and `outlived_fr = 'b`.
     pub(crate) fn report_region_error(
         &mut self,
         fr: RegionVid,
@@ -575,7 +575,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
 
         let mut output_ty = self.regioncx.universal_regions().unnormalized_output_ty;
         if let ty::Alias(ty::Opaque, ty::AliasTy { def_id, .. }) = *output_ty.kind() {
-            output_ty = self.infcx.tcx.type_of(def_id)
+            output_ty = self.infcx.tcx.type_of(def_id).subst_identity()
         };
 
         debug!("report_fnmut_error: output_ty={:?}", output_ty);
@@ -583,10 +583,12 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
         let err = FnMutError {
             span: *span,
             ty_err: match output_ty.kind() {
-                ty::Closure(_, _) => FnMutReturnTypeErr::ReturnClosure { span: *span },
                 ty::Generator(def, ..) if self.infcx.tcx.generator_is_async(*def) => {
                     FnMutReturnTypeErr::ReturnAsyncBlock { span: *span }
                 }
+                _ if output_ty.contains_closure() => {
+                    FnMutReturnTypeErr::ReturnClosure { span: *span }
+                }
                 _ => FnMutReturnTypeErr::ReturnRef { span: *span },
             },
         };
@@ -658,10 +660,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
             errci.outlived_fr,
         );
 
-        let (_, escapes_from) = self
-            .infcx
-            .tcx
-            .article_and_description(self.regioncx.universal_regions().defining_ty.def_id());
+        let escapes_from =
+            self.infcx.tcx.def_descr(self.regioncx.universal_regions().defining_ty.def_id());
 
         // Revert to the normal error in these cases.
         // Assignments aren't "escapes" in function items.
@@ -755,8 +755,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
             ..
         } = errci;
 
-        let (_, mir_def_name) =
-            self.infcx.tcx.article_and_description(self.mir_def_id().to_def_id());
+        let mir_def_name = self.infcx.tcx.def_descr(self.mir_def_id().to_def_id());
 
         let err = LifetimeOutliveErr { span: *span };
         let mut diag = self.infcx.tcx.sess.create_err(err);
@@ -813,17 +812,10 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
             if *outlived_f != ty::ReStatic {
                 return;
             }
+            let suitable_region = self.infcx.tcx.is_suitable_region(f);
+            let Some(suitable_region) = suitable_region else { return; };
 
-            let fn_returns = self
-                .infcx
-                .tcx
-                .is_suitable_region(f)
-                .map(|r| self.infcx.tcx.return_type_impl_or_dyn_traits(r.def_id))
-                .unwrap_or_default();
-
-            if fn_returns.is_empty() {
-                return;
-            }
+            let fn_returns = self.infcx.tcx.return_type_impl_or_dyn_traits(suitable_region.def_id);
 
             let param = if let Some(param) = find_param_with_region(self.infcx.tcx, f, outlived_f) {
                 param
@@ -839,15 +831,43 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
             };
             let captures = format!("captures data from {arg}");
 
-            return nice_region_error::suggest_new_region_bound(
-                self.infcx.tcx,
-                diag,
-                fn_returns,
-                lifetime.to_string(),
-                Some(arg),
-                captures,
-                Some((param.param_ty_span, param.param_ty.to_string())),
-                self.infcx.tcx.is_suitable_region(f).map(|r| r.def_id),
+            if !fn_returns.is_empty() {
+                nice_region_error::suggest_new_region_bound(
+                    self.infcx.tcx,
+                    diag,
+                    fn_returns,
+                    lifetime.to_string(),
+                    Some(arg),
+                    captures,
+                    Some((param.param_ty_span, param.param_ty.to_string())),
+                    Some(suitable_region.def_id),
+                );
+                return;
+            }
+
+            let Some((alias_tys, alias_span)) = self
+                .infcx
+                .tcx
+                .return_type_impl_or_dyn_traits_with_type_alias(suitable_region.def_id) else { return; };
+
+            // in case the return type of the method is a type alias
+            let mut spans_suggs: Vec<_> = Vec::new();
+            for alias_ty in alias_tys {
+                if alias_ty.span.desugaring_kind().is_some() {
+                    // Skip `async` desugaring `impl Future`.
+                    ()
+                }
+                if let TyKind::TraitObject(_, lt, _) = alias_ty.kind {
+                    spans_suggs.push((lt.ident.span.shrink_to_hi(), " + 'a".to_string()));
+                }
+            }
+            spans_suggs.push((alias_span.shrink_to_hi(), "<'a>".to_string()));
+            diag.multipart_suggestion_verbose(
+                &format!(
+                    "to declare that the trait object {captures}, you can add a lifetime parameter `'a` in the type alias"
+                ),
+                spans_suggs,
+                Applicability::MaybeIncorrect,
             );
         }
     }
@@ -873,7 +893,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
             debug!(?fn_did, ?substs);
 
             // Only suggest this on function calls, not closures
-            let ty = tcx.type_of(fn_did);
+            let ty = tcx.type_of(fn_did).subst_identity();
             debug!("ty: {:?}, ty.kind: {:?}", ty, ty.kind());
             if let ty::Closure(_, _) = ty.kind() {
                 return;
@@ -929,7 +949,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                     .push_span_label(*span, "this has an implicit `'static` lifetime requirement");
                 multi_span.push_span_label(
                     ident.span,
-                    "calling this method introduces the `impl`'s 'static` requirement",
+                    "calling this method introduces the `impl`'s `'static` requirement",
                 );
                 err.subdiagnostic(RequireStaticErr::UsedImpl { multi_span });
                 err.span_suggestion_verbose(
@@ -976,7 +996,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
     fn suggest_move_on_borrowing_closure(&self, diag: &mut Diagnostic) {
         let map = self.infcx.tcx.hir();
         let body_id = map.body_owned_by(self.mir_def_id());
-        let expr = &map.body(body_id).value;
+        let expr = &map.body(body_id).value.peel_blocks();
         let mut closure_span = None::<rustc_span::Span>;
         match expr.kind {
             hir::ExprKind::MethodCall(.., args, _) => {
@@ -991,20 +1011,14 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                     }
                 }
             }
-            hir::ExprKind::Block(blk, _) => {
-                if let Some(expr) = blk.expr {
-                    // only when the block is a closure
-                    if let hir::ExprKind::Closure(hir::Closure {
-                        capture_clause: hir::CaptureBy::Ref,
-                        body,
-                        ..
-                    }) = expr.kind
-                    {
-                        let body = map.body(*body);
-                        if !matches!(body.generator_kind, Some(hir::GeneratorKind::Async(..))) {
-                            closure_span = Some(expr.span.shrink_to_lo());
-                        }
-                    }
+            hir::ExprKind::Closure(hir::Closure {
+                capture_clause: hir::CaptureBy::Ref,
+                body,
+                ..
+            }) => {
+                let body = map.body(*body);
+                if !matches!(body.generator_kind, Some(hir::GeneratorKind::Async(..))) {
+                    closure_span = Some(expr.span.shrink_to_lo());
                 }
             }
             _ => {}
diff --git a/compiler/rustc_borrowck/src/diagnostics/region_name.rs b/compiler/rustc_borrowck/src/diagnostics/region_name.rs
index 9233287cf..f6881a2e5 100644
--- a/compiler/rustc_borrowck/src/diagnostics/region_name.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/region_name.rs
@@ -187,6 +187,12 @@ impl Display for RegionName {
     }
 }
 
+impl rustc_errors::IntoDiagnosticArg for RegionName {
+    fn into_diagnostic_arg(self) -> rustc_errors::DiagnosticArgValue<'static> {
+        self.to_string().into_diagnostic_arg()
+    }
+}
+
 impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
     pub(crate) fn mir_def_id(&self) -> hir::def_id::LocalDefId {
         self.body.source.def_id().expect_local()
@@ -274,17 +280,10 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
 
         debug!("give_region_a_name: error_region = {:?}", error_region);
         match *error_region {
-            ty::ReEarlyBound(ebr) => {
-                if ebr.has_name() {
-                    let span = tcx.hir().span_if_local(ebr.def_id).unwrap_or(DUMMY_SP);
-                    Some(RegionName {
-                        name: ebr.name,
-                        source: RegionNameSource::NamedEarlyBoundRegion(span),
-                    })
-                } else {
-                    None
-                }
-            }
+            ty::ReEarlyBound(ebr) => ebr.has_name().then(|| {
+                let span = tcx.hir().span_if_local(ebr.def_id).unwrap_or(DUMMY_SP);
+                RegionName { name: ebr.name, source: RegionNameSource::NamedEarlyBoundRegion(span) }
+            }),
 
             ty::ReStatic => {
                 Some(RegionName { name: kw::StaticLifetime, source: RegionNameSource::Static })
@@ -337,11 +336,11 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
                     let note = match closure_kind_ty.to_opt_closure_kind() {
                         Some(ty::ClosureKind::Fn) => {
                             "closure implements `Fn`, so references to captured variables \
-                                can't escape the closure"
+                             can't escape the closure"
                         }
                         Some(ty::ClosureKind::FnMut) => {
                             "closure implements `FnMut`, so references to captured variables \
-                                can't escape the closure"
+                             can't escape the closure"
                         }
                         Some(ty::ClosureKind::FnOnce) => {
                             bug!("BrEnv in a `FnOnce` closure");
@@ -358,7 +357,11 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
                 ty::BoundRegionKind::BrAnon(..) => None,
             },
 
-            ty::ReLateBound(..) | ty::ReVar(..) | ty::RePlaceholder(..) | ty::ReErased => None,
+            ty::ReLateBound(..)
+            | ty::ReVar(..)
+            | ty::RePlaceholder(..)
+            | ty::ReErased
+            | ty::ReError(_) => None,
         }
     }
 
@@ -842,12 +845,13 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
 
         let tcx = self.infcx.tcx;
         let region_parent = tcx.parent(region.def_id);
-        if tcx.def_kind(region_parent) != DefKind::Impl {
+        let DefKind::Impl { .. } = tcx.def_kind(region_parent) else {
             return None;
-        }
+        };
 
-        let found = tcx
-            .any_free_region_meets(&tcx.type_of(region_parent), |r| *r == ty::ReEarlyBound(region));
+        let found = tcx.any_free_region_meets(&tcx.type_of(region_parent).subst_identity(), |r| {
+            *r == ty::ReEarlyBound(region)
+        });
 
         Some(RegionName {
             name: self.synthesize_region_name(),
diff --git a/compiler/rustc_borrowck/src/invalidation.rs b/compiler/rustc_borrowck/src/invalidation.rs
index 6fd929005..6217676d5 100644
--- a/compiler/rustc_borrowck/src/invalidation.rs
+++ b/compiler/rustc_borrowck/src/invalidation.rs
@@ -91,7 +91,8 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
                     LocalMutationIsAllowed::Yes,
                 );
             }
-            StatementKind::Nop
+            StatementKind::ConstEvalCounter
+            | StatementKind::Nop
             | StatementKind::Retag { .. }
             | StatementKind::Deinit(..)
             | StatementKind::SetDiscriminant { .. } => {
diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs
index 73ea7314b..0f591460e 100644
--- a/compiler/rustc_borrowck/src/lib.rs
+++ b/compiler/rustc_borrowck/src/lib.rs
@@ -1,6 +1,7 @@
 //! This query borrow-checks the MIR to (further) ensure it is not broken.
 
 #![allow(rustc::potential_query_instability)]
+#![feature(associated_type_bounds)]
 #![feature(box_patterns)]
 #![feature(let_chains)]
 #![feature(min_specialization)]
@@ -20,12 +21,15 @@ extern crate tracing;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::graph::dominators::Dominators;
 use rustc_data_structures::vec_map::VecMap;
-use rustc_errors::{Diagnostic, DiagnosticBuilder};
+use rustc_errors::{Diagnostic, DiagnosticBuilder, DiagnosticMessage, SubdiagnosticMessage};
 use rustc_hir as hir;
 use rustc_hir::def_id::LocalDefId;
 use rustc_index::bit_set::ChunkedBitSet;
 use rustc_index::vec::IndexVec;
-use rustc_infer::infer::{DefiningAnchor, InferCtxt, TyCtxtInferExt};
+use rustc_infer::infer::{
+    DefiningAnchor, InferCtxt, NllRegionVariableOrigin, RegionVariableOrigin, TyCtxtInferExt,
+};
+use rustc_macros::fluent_messages;
 use rustc_middle::mir::{
     traversal, Body, ClearCrossCrate, Local, Location, Mutability, NonDivergingIntrinsic, Operand,
     Place, PlaceElem, PlaceRef, VarDebugInfoContents,
@@ -43,6 +47,7 @@ use smallvec::SmallVec;
 use std::cell::OnceCell;
 use std::cell::RefCell;
 use std::collections::BTreeMap;
+use std::ops::Deref;
 use std::rc::Rc;
 
 use rustc_mir_dataflow::impls::{
@@ -94,6 +99,9 @@ use nll::{PoloniusOutput, ToRegionVid};
 use place_ext::PlaceExt;
 use places_conflict::{places_conflict, PlaceConflictBias};
 use region_infer::RegionInferenceContext;
+use renumber::RegionCtxt;
+
+fluent_messages! { "../locales/en-US.ftl" }
 
 // FIXME(eddyb) perhaps move this somewhere more centrally.
 #[derive(Debug)]
@@ -167,10 +175,10 @@ fn do_mir_borrowck<'tcx>(
     return_body_with_facts: bool,
 ) -> (BorrowCheckResult<'tcx>, Option<Box<BodyWithBorrowckFacts<'tcx>>>) {
     let def = input_body.source.with_opt_param().as_local().unwrap();
-
     debug!(?def);
 
     let tcx = infcx.tcx;
+    let infcx = BorrowckInferCtxt::new(infcx);
     let param_env = tcx.param_env(def.did);
 
     let mut local_names = IndexVec::from_elem(None, &input_body.local_decls);
@@ -194,14 +202,14 @@ fn do_mir_borrowck<'tcx>(
     let mut errors = error::BorrowckErrors::new(infcx.tcx);
 
     // Gather the upvars of a closure, if any.
-    let tables = tcx.typeck_opt_const_arg(def);
-    if let Some(e) = tables.tainted_by_errors {
+    if let Some(e) = input_body.tainted_by_errors {
         infcx.set_tainted_by_errors(e);
         errors.set_tainted_by_errors(e);
     }
-    let upvars: Vec<_> = tables
-        .closure_min_captures_flattened(def.did)
-        .map(|captured_place| {
+    let upvars: Vec<_> = tcx
+        .closure_captures(def.did)
+        .iter()
+        .map(|&captured_place| {
             let capture = captured_place.info.capture_kind;
             let by_ref = match capture {
                 ty::UpvarCapture::ByValue => false,
@@ -218,7 +226,7 @@ fn do_mir_borrowck<'tcx>(
     let mut body_owned = input_body.clone();
     let mut promoted = input_promoted.clone();
     let free_regions =
-        nll::replace_regions_in_mir(infcx, param_env, &mut body_owned, &mut promoted);
+        nll::replace_regions_in_mir(&infcx, param_env, &mut body_owned, &mut promoted);
     let body = &body_owned; // no further changes
 
     let location_table_owned = LocationTable::new(body);
@@ -256,7 +264,7 @@ fn do_mir_borrowck<'tcx>(
         opt_closure_req,
         nll_errors,
     } = nll::compute_regions(
-        infcx,
+        &infcx,
         free_regions,
         body,
         &promoted,
@@ -271,12 +279,12 @@ fn do_mir_borrowck<'tcx>(
 
     // Dump MIR results into a file, if that is enabled. This let us
     // write unit-tests, as well as helping with debugging.
-    nll::dump_mir_results(infcx, &body, &regioncx, &opt_closure_req);
+    nll::dump_mir_results(&infcx, &body, &regioncx, &opt_closure_req);
 
     // We also have a `#[rustc_regions]` annotation that causes us to dump
     // information.
     nll::dump_annotation(
-        infcx,
+        &infcx,
         &body,
         &regioncx,
         &opt_closure_req,
@@ -320,7 +328,7 @@ fn do_mir_borrowck<'tcx>(
 
         if let Err((move_data, move_errors)) = move_data_results {
             let mut promoted_mbcx = MirBorrowckCtxt {
-                infcx,
+                infcx: &infcx,
                 param_env,
                 body: promoted_body,
                 move_data: &move_data,
@@ -349,7 +357,7 @@ fn do_mir_borrowck<'tcx>(
     }
 
     let mut mbcx = MirBorrowckCtxt {
-        infcx,
+        infcx: &infcx,
         param_env,
         body,
         move_data: &mdpe.move_data,
@@ -481,8 +489,84 @@ pub struct BodyWithBorrowckFacts<'tcx> {
     pub location_table: LocationTable,
 }
 
+pub struct BorrowckInferCtxt<'cx, 'tcx> {
+    pub(crate) infcx: &'cx InferCtxt<'tcx>,
+    pub(crate) reg_var_to_origin: RefCell<FxHashMap<ty::RegionVid, RegionCtxt>>,
+}
+
+impl<'cx, 'tcx> BorrowckInferCtxt<'cx, 'tcx> {
+    pub(crate) fn new(infcx: &'cx InferCtxt<'tcx>) -> Self {
+        BorrowckInferCtxt { infcx, reg_var_to_origin: RefCell::new(Default::default()) }
+    }
+
+    pub(crate) fn next_region_var<F>(
+        &self,
+        origin: RegionVariableOrigin,
+        get_ctxt_fn: F,
+    ) -> ty::Region<'tcx>
+    where
+        F: Fn() -> RegionCtxt,
+    {
+        let next_region = self.infcx.next_region_var(origin);
+        let vid = next_region
+            .as_var()
+            .unwrap_or_else(|| bug!("expected RegionKind::RegionVar on {:?}", next_region));
+
+        if cfg!(debug_assertions) {
+            debug!("inserting vid {:?} with origin {:?} into var_to_origin", vid, origin);
+            let ctxt = get_ctxt_fn();
+            let mut var_to_origin = self.reg_var_to_origin.borrow_mut();
+            let prev = var_to_origin.insert(vid, ctxt);
+
+            // This only makes sense if not called in a canonicalization context. If this
+            // ever changes we either want to get rid of `BorrowckInferContext::reg_var_to_origin`
+            // or modify how we track nll region vars for that map.
+            assert!(matches!(prev, None));
+        }
+
+        next_region
+    }
+
+    #[instrument(skip(self, get_ctxt_fn), level = "debug")]
+    pub(crate) fn next_nll_region_var<F>(
+        &self,
+        origin: NllRegionVariableOrigin,
+        get_ctxt_fn: F,
+    ) -> ty::Region<'tcx>
+    where
+        F: Fn() -> RegionCtxt,
+    {
+        let next_region = self.infcx.next_nll_region_var(origin.clone());
+        let vid = next_region
+            .as_var()
+            .unwrap_or_else(|| bug!("expected RegionKind::RegionVar on {:?}", next_region));
+
+        if cfg!(debug_assertions) {
+            debug!("inserting vid {:?} with origin {:?} into var_to_origin", vid, origin);
+            let ctxt = get_ctxt_fn();
+            let mut var_to_origin = self.reg_var_to_origin.borrow_mut();
+            let prev = var_to_origin.insert(vid, ctxt);
+
+            // This only makes sense if not called in a canonicalization context. If this
+            // ever changes we either want to get rid of `BorrowckInferContext::reg_var_to_origin`
+            // or modify how we track nll region vars for that map.
+            assert!(matches!(prev, None));
+        }
+
+        next_region
+    }
+}
+
+impl<'cx, 'tcx> Deref for BorrowckInferCtxt<'cx, 'tcx> {
+    type Target = InferCtxt<'tcx>;
+
+    fn deref(&self) -> &'cx Self::Target {
+        self.infcx
+    }
+}
+
 struct MirBorrowckCtxt<'cx, 'tcx> {
-    infcx: &'cx InferCtxt<'tcx>,
+    infcx: &'cx BorrowckInferCtxt<'cx, 'tcx>,
     param_env: ParamEnv<'tcx>,
     body: &'cx Body<'tcx>,
     move_data: &'cx MoveData<'tcx>,
@@ -609,7 +693,8 @@ impl<'cx, 'tcx> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtx
             StatementKind::AscribeUserType(..)
             // Doesn't have any language semantics
             | StatementKind::Coverage(..)
-            // Does not actually affect borrowck
+            // These do not actually affect borrowck
+            | StatementKind::ConstEvalCounter
             | StatementKind::StorageLive(..) => {}
             StatementKind::StorageDead(local) => {
                 self.access_place(
@@ -1277,6 +1362,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 // in order to populate our used_mut set.
                 match **aggregate_kind {
                     AggregateKind::Closure(def_id, _) | AggregateKind::Generator(def_id, _, _) => {
+                        let def_id = def_id.expect_local();
                         let BorrowCheckResult { used_mut_upvars, .. } =
                             self.infcx.tcx.mir_borrowck(def_id);
                         debug!("{:?} used_mut_upvars={:?}", def_id, used_mut_upvars);
diff --git a/compiler/rustc_borrowck/src/nll.rs b/compiler/rustc_borrowck/src/nll.rs
index b2d92d0db..96228338a 100644
--- a/compiler/rustc_borrowck/src/nll.rs
+++ b/compiler/rustc_borrowck/src/nll.rs
@@ -5,16 +5,14 @@
 use rustc_data_structures::vec_map::VecMap;
 use rustc_hir::def_id::LocalDefId;
 use rustc_index::vec::IndexVec;
-use rustc_infer::infer::InferCtxt;
 use rustc_middle::mir::{create_dump_file, dump_enabled, dump_mir, PassWhere};
 use rustc_middle::mir::{
     BasicBlock, Body, ClosureOutlivesSubject, ClosureRegionRequirements, LocalKind, Location,
     Promoted,
 };
-use rustc_middle::ty::{self, OpaqueHiddenType, Region, RegionVid};
+use rustc_middle::ty::{self, OpaqueHiddenType, Region, RegionVid, TyCtxt};
 use rustc_span::symbol::sym;
 use std::env;
-use std::fmt::Debug;
 use std::io;
 use std::path::PathBuf;
 use std::rc::Rc;
@@ -37,7 +35,7 @@ use crate::{
     renumber,
     type_check::{self, MirTypeckRegionConstraints, MirTypeckResults},
     universal_regions::UniversalRegions,
-    Upvar,
+    BorrowckInferCtxt, Upvar,
 };
 
 pub type PoloniusOutput = Output<RustcFacts>;
@@ -58,7 +56,7 @@ pub(crate) struct NllOutput<'tcx> {
 /// `compute_regions`.
 #[instrument(skip(infcx, param_env, body, promoted), level = "debug")]
 pub(crate) fn replace_regions_in_mir<'tcx>(
-    infcx: &InferCtxt<'tcx>,
+    infcx: &BorrowckInferCtxt<'_, 'tcx>,
     param_env: ty::ParamEnv<'tcx>,
     body: &mut Body<'tcx>,
     promoted: &mut IndexVec<Promoted, Body<'tcx>>,
@@ -157,7 +155,7 @@ fn populate_polonius_move_facts(
 ///
 /// This may result in errors being reported.
 pub(crate) fn compute_regions<'cx, 'tcx>(
-    infcx: &InferCtxt<'tcx>,
+    infcx: &BorrowckInferCtxt<'_, 'tcx>,
     universal_regions: UniversalRegions<'tcx>,
     body: &Body<'tcx>,
     promoted: &IndexVec<Promoted, Body<'tcx>>,
@@ -259,6 +257,7 @@ pub(crate) fn compute_regions<'cx, 'tcx>(
     );
 
     let mut regioncx = RegionInferenceContext::new(
+        infcx,
         var_origins,
         universal_regions,
         placeholder_indices,
@@ -322,10 +321,10 @@ pub(crate) fn compute_regions<'cx, 'tcx>(
 }
 
 pub(super) fn dump_mir_results<'tcx>(
-    infcx: &InferCtxt<'tcx>,
+    infcx: &BorrowckInferCtxt<'_, 'tcx>,
     body: &Body<'tcx>,
     regioncx: &RegionInferenceContext<'tcx>,
-    closure_region_requirements: &Option<ClosureRegionRequirements<'_>>,
+    closure_region_requirements: &Option<ClosureRegionRequirements<'tcx>>,
 ) {
     if !dump_enabled(infcx.tcx, "nll", body.source.def_id()) {
         return;
@@ -340,9 +339,11 @@ pub(super) fn dump_mir_results<'tcx>(
 
                 if let Some(closure_region_requirements) = closure_region_requirements {
                     writeln!(out, "| Free Region Constraints")?;
-                    for_each_region_constraint(closure_region_requirements, &mut |msg| {
-                        writeln!(out, "| {}", msg)
-                    })?;
+                    for_each_region_constraint(
+                        infcx.tcx,
+                        closure_region_requirements,
+                        &mut |msg| writeln!(out, "| {}", msg),
+                    )?;
                     writeln!(out, "|")?;
                 }
             }
@@ -369,11 +370,13 @@ pub(super) fn dump_mir_results<'tcx>(
     };
 }
 
+#[allow(rustc::diagnostic_outside_of_impl)]
+#[allow(rustc::untranslatable_diagnostic)]
 pub(super) fn dump_annotation<'tcx>(
-    infcx: &InferCtxt<'tcx>,
+    infcx: &BorrowckInferCtxt<'_, 'tcx>,
     body: &Body<'tcx>,
     regioncx: &RegionInferenceContext<'tcx>,
-    closure_region_requirements: &Option<ClosureRegionRequirements<'_>>,
+    closure_region_requirements: &Option<ClosureRegionRequirements<'tcx>>,
     opaque_type_values: &VecMap<LocalDefId, OpaqueHiddenType<'tcx>>,
     errors: &mut crate::error::BorrowckErrors<'tcx>,
 ) {
@@ -403,7 +406,7 @@ pub(super) fn dump_annotation<'tcx>(
 
         // Dump the region constraints we are imposing *between* those
         // newly created variables.
-        for_each_region_constraint(closure_region_requirements, &mut |msg| {
+        for_each_region_constraint(tcx, closure_region_requirements, &mut |msg| {
             err.note(msg);
             Ok(())
         })
@@ -424,16 +427,19 @@ pub(super) fn dump_annotation<'tcx>(
     errors.buffer_non_error_diag(err);
 }
 
-fn for_each_region_constraint(
-    closure_region_requirements: &ClosureRegionRequirements<'_>,
+fn for_each_region_constraint<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    closure_region_requirements: &ClosureRegionRequirements<'tcx>,
     with_msg: &mut dyn FnMut(&str) -> io::Result<()>,
 ) -> io::Result<()> {
     for req in &closure_region_requirements.outlives_requirements {
-        let subject: &dyn Debug = match &req.subject {
-            ClosureOutlivesSubject::Region(subject) => subject,
-            ClosureOutlivesSubject::Ty(ty) => ty,
+        let subject = match req.subject {
+            ClosureOutlivesSubject::Region(subject) => format!("{:?}", subject),
+            ClosureOutlivesSubject::Ty(ty) => {
+                format!("{:?}", ty.instantiate(tcx, |vid| tcx.mk_re_var(vid)))
+            }
         };
-        with_msg(&format!("where {:?}: {:?}", subject, req.outlived_free_region,))?;
+        with_msg(&format!("where {}: {:?}", subject, req.outlived_free_region,))?;
     }
     Ok(())
 }
diff --git a/compiler/rustc_borrowck/src/region_infer/mod.rs b/compiler/rustc_borrowck/src/region_infer/mod.rs
index 238172ea3..e6195de40 100644
--- a/compiler/rustc_borrowck/src/region_infer/mod.rs
+++ b/compiler/rustc_borrowck/src/region_infer/mod.rs
@@ -7,18 +7,18 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::graph::scc::Sccs;
 use rustc_errors::Diagnostic;
 use rustc_hir::def_id::CRATE_DEF_ID;
-use rustc_hir::CRATE_HIR_ID;
 use rustc_index::vec::IndexVec;
 use rustc_infer::infer::outlives::test_type_match;
 use rustc_infer::infer::region_constraints::{GenericKind, VarInfos, VerifyBound, VerifyIfEq};
 use rustc_infer::infer::{InferCtxt, NllRegionVariableOrigin, RegionVariableOrigin};
 use rustc_middle::mir::{
-    Body, ClosureOutlivesRequirement, ClosureOutlivesSubject, ClosureRegionRequirements,
-    ConstraintCategory, Local, Location, ReturnConstraint, TerminatorKind,
+    Body, ClosureOutlivesRequirement, ClosureOutlivesSubject, ClosureOutlivesSubjectTy,
+    ClosureRegionRequirements, ConstraintCategory, Local, Location, ReturnConstraint,
+    TerminatorKind,
 };
 use rustc_middle::traits::ObligationCause;
 use rustc_middle::traits::ObligationCauseCode;
-use rustc_middle::ty::{self, RegionVid, Ty, TyCtxt, TypeFoldable, TypeVisitable};
+use rustc_middle::ty::{self, RegionVid, Ty, TyCtxt, TypeFoldable, TypeVisitableExt};
 use rustc_span::Span;
 
 use crate::{
@@ -35,6 +35,7 @@ use crate::{
     },
     type_check::{free_region_relations::UniversalRegionRelations, Locations},
     universal_regions::UniversalRegions,
+    BorrowckInferCtxt,
 };
 
 mod dump_mir;
@@ -244,6 +245,70 @@ pub enum ExtraConstraintInfo {
     PlaceholderFromPredicate(Span),
 }
 
+#[instrument(skip(infcx, sccs), level = "debug")]
+fn sccs_info<'cx, 'tcx>(
+    infcx: &'cx BorrowckInferCtxt<'cx, 'tcx>,
+    sccs: Rc<Sccs<RegionVid, ConstraintSccIndex>>,
+) {
+    use crate::renumber::RegionCtxt;
+
+    let var_to_origin = infcx.reg_var_to_origin.borrow();
+
+    let mut var_to_origin_sorted = var_to_origin.clone().into_iter().collect::<Vec<_>>();
+    var_to_origin_sorted.sort_by(|a, b| a.0.cmp(&b.0));
+    let mut debug_str = "region variables to origins:\n".to_string();
+    for (reg_var, origin) in var_to_origin_sorted.into_iter() {
+        debug_str.push_str(&format!("{:?}: {:?}\n", reg_var, origin));
+    }
+    debug!(debug_str);
+
+    let num_components = sccs.scc_data().ranges().len();
+    let mut components = vec![FxHashSet::default(); num_components];
+
+    for (reg_var_idx, scc_idx) in sccs.scc_indices().iter().enumerate() {
+        let reg_var = ty::RegionVid::from_usize(reg_var_idx);
+        let origin = var_to_origin.get(&reg_var).unwrap_or_else(|| &RegionCtxt::Unknown);
+        components[scc_idx.as_usize()].insert((reg_var, *origin));
+    }
+
+    let mut components_str = "strongly connected components:".to_string();
+    for (scc_idx, reg_vars_origins) in components.iter().enumerate() {
+        let regions_info = reg_vars_origins.clone().into_iter().collect::<Vec<_>>();
+        components_str.push_str(&format!(
+            "{:?}: {:?})",
+            ConstraintSccIndex::from_usize(scc_idx),
+            regions_info,
+        ))
+    }
+    debug!(components_str);
+
+    // calculate the best representative for each component
+    let components_representatives = components
+        .into_iter()
+        .enumerate()
+        .map(|(scc_idx, region_ctxts)| {
+            let repr = region_ctxts
+                .into_iter()
+                .map(|reg_var_origin| reg_var_origin.1)
+                .max_by(|x, y| x.preference_value().cmp(&y.preference_value()))
+                .unwrap();
+
+            (ConstraintSccIndex::from_usize(scc_idx), repr)
+        })
+        .collect::<FxHashMap<_, _>>();
+
+    let mut scc_node_to_edges = FxHashMap::default();
+    for (scc_idx, repr) in components_representatives.iter() {
+        let edges_range = sccs.scc_data().ranges()[*scc_idx].clone();
+        let edges = &sccs.scc_data().all_successors()[edges_range];
+        let edge_representatives =
+            edges.iter().map(|scc_idx| components_representatives[scc_idx]).collect::<Vec<_>>();
+        scc_node_to_edges.insert((scc_idx, repr), edge_representatives);
+    }
+
+    debug!("SCC edges {:#?}", scc_node_to_edges);
+}
+
 impl<'tcx> RegionInferenceContext<'tcx> {
     /// Creates a new region inference context with a total of
     /// `num_region_variables` valid inference variables; the first N
@@ -252,7 +317,8 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     ///
     /// The `outlives_constraints` and `type_tests` are an initial set
     /// of constraints produced by the MIR type check.
-    pub(crate) fn new(
+    pub(crate) fn new<'cx>(
+        _infcx: &BorrowckInferCtxt<'cx, 'tcx>,
         var_infos: VarInfos,
         universal_regions: Rc<UniversalRegions<'tcx>>,
         placeholder_indices: Rc<PlaceholderIndices>,
@@ -264,6 +330,11 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         liveness_constraints: LivenessValues<RegionVid>,
         elements: &Rc<RegionValueElements>,
     ) -> Self {
+        debug!("universal_regions: {:#?}", universal_regions);
+        debug!("outlives constraints: {:#?}", outlives_constraints);
+        debug!("placeholder_indices: {:#?}", placeholder_indices);
+        debug!("type tests: {:#?}", type_tests);
+
         // Create a RegionDefinition for each inference variable.
         let definitions: IndexVec<_, _> = var_infos
             .iter()
@@ -275,6 +346,10 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         let fr_static = universal_regions.fr_static;
         let constraint_sccs = Rc::new(constraints.compute_sccs(&constraint_graph, fr_static));
 
+        if cfg!(debug_assertions) {
+            sccs_info(_infcx, constraint_sccs.clone());
+        }
+
         let mut scc_values =
             RegionValues::new(elements, universal_regions.len(), &placeholder_indices);
 
@@ -747,20 +822,33 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         }
         debug!(?choice_regions, "after ub");
 
-        // If we ruled everything out, we're done.
-        if choice_regions.is_empty() {
-            return false;
-        }
-
-        // Otherwise, we need to find the minimum remaining choice, if
-        // any, and take that.
-        debug!("choice_regions remaining are {:#?}", choice_regions);
-        let Some(&min_choice) = choice_regions.iter().find(|&r1| {
+        // At this point we can pick any member of `choice_regions`, but to avoid potential
+        // non-determinism we will pick the *unique minimum* choice.
+        //
+        // Because universal regions are only partially ordered (i.e, not every two regions are
+        // comparable), we will ignore any region that doesn't compare to all others when picking
+        // the minimum choice.
+        // For example, consider `choice_regions = ['static, 'a, 'b, 'c, 'd, 'e]`, where
+        // `'static: 'a, 'static: 'b, 'a: 'c, 'b: 'c, 'c: 'd, 'c: 'e`.
+        // `['d, 'e]` are ignored because they do not compare - the same goes for `['a, 'b]`.
+        let totally_ordered_subset = choice_regions.iter().copied().filter(|&r1| {
             choice_regions.iter().all(|&r2| {
-                self.universal_region_relations.outlives(r2, *r1)
+                self.universal_region_relations.outlives(r1, r2)
+                    || self.universal_region_relations.outlives(r2, r1)
             })
+        });
+        // Now we're left with `['static, 'c]`. Pick `'c` as the minimum!
+        let Some(min_choice) = totally_ordered_subset.reduce(|r1, r2| {
+            let r1_outlives_r2 = self.universal_region_relations.outlives(r1, r2);
+            let r2_outlives_r1 = self.universal_region_relations.outlives(r2, r1);
+            match (r1_outlives_r2, r2_outlives_r1) {
+                (true, true) => r1.min(r2),
+                (true, false) => r2,
+                (false, true) => r1,
+                (false, false) => bug!("incomparable regions in total order"),
+            }
         }) else {
-            debug!("no choice region outlived by all others");
+            debug!("no unique minimum choice");
             return false;
         };
 
@@ -802,7 +890,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     /// from a universe it can't name; at present, the only way for
     /// this to be true is if `scc` outlives `'static`. This is
     /// actually stricter than necessary: ideally, we'd support bounds
-    /// like `for<'a: 'b`>` that might then allow us to approximate
+    /// like `for<'a: 'b>` that might then allow us to approximate
     /// `'a` with `'b` and not `'static`. But it will have to do for
     /// now.
     fn add_incompatible_universe(&mut self, scc: ConstraintSccIndex) {
@@ -997,18 +1085,10 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         true
     }
 
-    /// When we promote a type test `T: 'r`, we have to convert the
-    /// type `T` into something we can store in a query result (so
-    /// something allocated for `'tcx`). This is problematic if `ty`
-    /// contains regions. During the course of NLL region checking, we
-    /// will have replaced all of those regions with fresh inference
-    /// variables. To create a test subject, we want to replace those
-    /// inference variables with some region from the closure
-    /// signature -- this is not always possible, so this is a
-    /// fallible process. Presuming we do find a suitable region, we
-    /// will use it's *external name*, which will be a `RegionKind`
-    /// variant that can be used in query responses such as
-    /// `ReEarlyBound`.
+    /// When we promote a type test `T: 'r`, we have to replace all region
+    /// variables in the type `T` with an equal universal region from the
+    /// closure signature.
+    /// This is not always possible, so this is a fallible process.
     #[instrument(level = "debug", skip(self, infcx))]
     fn try_promote_type_test_subject(
         &self,
@@ -1017,91 +1097,63 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     ) -> Option<ClosureOutlivesSubject<'tcx>> {
         let tcx = infcx.tcx;
 
+        // Opaque types' substs may include useless lifetimes.
+        // We will replace them with ReStatic.
+        struct OpaqueFolder<'tcx> {
+            tcx: TyCtxt<'tcx>,
+        }
+        impl<'tcx> ty::TypeFolder<TyCtxt<'tcx>> for OpaqueFolder<'tcx> {
+            fn interner(&self) -> TyCtxt<'tcx> {
+                self.tcx
+            }
+            fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
+                use ty::TypeSuperFoldable as _;
+                let tcx = self.tcx;
+                let &ty::Alias(ty::Opaque, ty::AliasTy { substs, def_id, .. }) = t.kind() else {
+                    return t.super_fold_with(self);
+                };
+                let substs =
+                    std::iter::zip(substs, tcx.variances_of(def_id)).map(|(arg, v)| {
+                        match (arg.unpack(), v) {
+                            (ty::GenericArgKind::Lifetime(_), ty::Bivariant) => {
+                                tcx.lifetimes.re_static.into()
+                            }
+                            _ => arg.fold_with(self),
+                        }
+                    });
+                tcx.mk_opaque(def_id, tcx.mk_substs_from_iter(substs))
+            }
+        }
+
+        let ty = ty.fold_with(&mut OpaqueFolder { tcx });
+
         let ty = tcx.fold_regions(ty, |r, _depth| {
-            let region_vid = self.to_region_vid(r);
+            let r_vid = self.to_region_vid(r);
+            let r_scc = self.constraint_sccs.scc(r_vid);
 
             // The challenge if this. We have some region variable `r`
             // whose value is a set of CFG points and universal
             // regions. We want to find if that set is *equivalent* to
             // any of the named regions found in the closure.
-            //
-            // To do so, we compute the
-            // `non_local_universal_upper_bound`. This will be a
-            // non-local, universal region that is greater than `r`.
-            // However, it might not be *contained* within `r`, so
-            // then we further check whether this bound is contained
-            // in `r`. If so, we can say that `r` is equivalent to the
-            // bound.
-            //
-            // Let's work through a few examples. For these, imagine
-            // that we have 3 non-local regions (I'll denote them as
-            // `'static`, `'a`, and `'b`, though of course in the code
-            // they would be represented with indices) where:
-            //
-            // - `'static: 'a`
-            // - `'static: 'b`
-            //
-            // First, let's assume that `r` is some existential
-            // variable with an inferred value `{'a, 'static}` (plus
-            // some CFG nodes). In this case, the non-local upper
-            // bound is `'static`, since that outlives `'a`. `'static`
-            // is also a member of `r` and hence we consider `r`
-            // equivalent to `'static` (and replace it with
-            // `'static`).
-            //
-            // Now let's consider the inferred value `{'a, 'b}`. This
-            // means `r` is effectively `'a | 'b`. I'm not sure if
-            // this can come about, actually, but assuming it did, we
-            // would get a non-local upper bound of `'static`. Since
-            // `'static` is not contained in `r`, we would fail to
-            // find an equivalent.
-            let upper_bound = self.non_local_universal_upper_bound(region_vid);
-            if self.region_contains(region_vid, upper_bound) {
-                self.definitions[upper_bound].external_name.unwrap_or(r)
-            } else {
-                // In the case of a failure, use a `ReVar` result. This will
-                // cause the `needs_infer` later on to return `None`.
-                r
-            }
+            // To do so, we simply check every candidate `u_r` for equality.
+            self.scc_values
+                .universal_regions_outlived_by(r_scc)
+                .filter(|&u_r| !self.universal_regions.is_local_free_region(u_r))
+                .find(|&u_r| self.eval_equal(u_r, r_vid))
+                .map(|u_r| tcx.mk_re_var(u_r))
+                // In the case of a failure, use `ReErased`. We will eventually
+                // return `None` in this case.
+                .unwrap_or(tcx.lifetimes.re_erased)
         });
 
         debug!("try_promote_type_test_subject: folded ty = {:?}", ty);
 
-        // `needs_infer` will only be true if we failed to promote some region.
-        if ty.needs_infer() {
+        // This will be true if we failed to promote some region.
+        if ty.has_erased_regions() {
             return None;
         }
 
-        Some(ClosureOutlivesSubject::Ty(ty))
-    }
-
-    /// Given some universal or existential region `r`, finds a
-    /// non-local, universal region `r+` that outlives `r` at entry to (and
-    /// exit from) the closure. In the worst case, this will be
-    /// `'static`.
-    ///
-    /// This is used for two purposes. First, if we are propagated
-    /// some requirement `T: r`, we can use this method to enlarge `r`
-    /// to something we can encode for our creator (which only knows
-    /// about non-local, universal regions). It is also used when
-    /// encoding `T` as part of `try_promote_type_test_subject` (see
-    /// that fn for details).
-    ///
-    /// This is based on the result `'y` of `universal_upper_bound`,
-    /// except that it converts further takes the non-local upper
-    /// bound of `'y`, so that the final result is non-local.
-    fn non_local_universal_upper_bound(&self, r: RegionVid) -> RegionVid {
-        debug!("non_local_universal_upper_bound(r={:?}={})", r, self.region_value_str(r));
-
-        let lub = self.universal_upper_bound(r);
-
-        // Grow further to get smallest universal region known to
-        // creator.
-        let non_local_lub = self.universal_region_relations.non_local_upper_bound(lub);
-
-        debug!("non_local_universal_upper_bound: non_local_lub={:?}", non_local_lub);
-
-        non_local_lub
+        Some(ClosureOutlivesSubject::Ty(ClosureOutlivesSubjectTy::bind(tcx, ty)))
     }
 
     /// Returns a universally quantified region that outlives the
@@ -1279,13 +1331,13 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     /// we use this kind of hacky solution.
     fn normalize_to_scc_representatives<T>(&self, tcx: TyCtxt<'tcx>, value: T) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         tcx.fold_regions(value, |r, _db| {
             let vid = self.to_region_vid(r);
             let scc = self.constraint_sccs.scc(vid);
             let repr = self.scc_representatives[scc];
-            tcx.mk_region(ty::ReVar(repr))
+            tcx.mk_re_var(repr)
         })
     }
 
@@ -1707,7 +1759,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
             }
 
             // If not, report an error.
-            let member_region = infcx.tcx.mk_region(ty::ReVar(member_region_vid));
+            let member_region = infcx.tcx.mk_re_var(member_region_vid);
             errors_buffer.push(RegionErrorKind::UnexpectedHiddenRegion {
                 span: m_c.definition_span,
                 hidden_ty: m_c.hidden_ty,
@@ -2022,7 +2074,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
             .map(|constraint| BlameConstraint {
                 category: constraint.category,
                 from_closure: constraint.from_closure,
-                cause: ObligationCause::new(constraint.span, CRATE_HIR_ID, cause_code.clone()),
+                cause: ObligationCause::new(constraint.span, CRATE_DEF_ID, cause_code.clone()),
                 variance_info: constraint.variance_info,
                 outlives_constraint: *constraint,
             })
diff --git a/compiler/rustc_borrowck/src/region_infer/opaque_types.rs b/compiler/rustc_borrowck/src/region_infer/opaque_types.rs
index db5a67a8b..c550e37c6 100644
--- a/compiler/rustc_borrowck/src/region_infer/opaque_types.rs
+++ b/compiler/rustc_borrowck/src/region_infer/opaque_types.rs
@@ -1,12 +1,13 @@
 use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
 use rustc_data_structures::vec_map::VecMap;
+use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def_id::LocalDefId;
 use rustc_hir::OpaqueTyOrigin;
 use rustc_infer::infer::TyCtxtInferExt as _;
 use rustc_infer::infer::{DefiningAnchor, InferCtxt};
 use rustc_infer::traits::{Obligation, ObligationCause};
 use rustc_middle::ty::subst::{GenericArgKind, InternalSubsts};
-use rustc_middle::ty::visit::TypeVisitable;
+use rustc_middle::ty::visit::TypeVisitableExt;
 use rustc_middle::ty::{self, OpaqueHiddenType, OpaqueTypeKey, Ty, TyCtxt, TypeFoldable};
 use rustc_span::Span;
 use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt as _;
@@ -91,11 +92,10 @@ impl<'tcx> RegionInferenceContext<'tcx> {
                     }
                     None => {
                         subst_regions.push(vid);
-                        infcx.tcx.sess.delay_span_bug(
+                        infcx.tcx.mk_re_error_with_message(
                             concrete_type.span,
                             "opaque type with non-universal region substs",
-                        );
-                        infcx.tcx.lifetimes.re_static
+                        )
                     }
                 }
             };
@@ -150,13 +150,13 @@ impl<'tcx> RegionInferenceContext<'tcx> {
             // once we convert the generic parameters to those of the opaque type.
             if let Some(prev) = result.get_mut(&opaque_type_key.def_id) {
                 if prev.ty != ty {
-                    if !ty.references_error() {
+                    let guar = ty.error_reported().err().unwrap_or_else(|| {
                         prev.report_mismatch(
                             &OpaqueHiddenType { ty, span: concrete_type.span },
                             infcx.tcx,
-                        );
-                    }
-                    prev.ty = infcx.tcx.ty_error();
+                        )
+                    });
+                    prev.ty = infcx.tcx.ty_error(guar);
                 }
                 // Pick a better span if there is one.
                 // FIXME(oli-obk): collect multiple spans for better diagnostics down the road.
@@ -179,7 +179,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     /// region names in error messages.
     pub(crate) fn name_regions<T>(&self, tcx: TyCtxt<'tcx>, ty: T) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         tcx.fold_regions(ty, |region, _| match *region {
             ty::ReVar(vid) => {
@@ -248,20 +248,20 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
         origin: OpaqueTyOrigin,
     ) -> Ty<'tcx> {
         if let Some(e) = self.tainted_by_errors() {
-            return self.tcx.ty_error_with_guaranteed(e);
+            return self.tcx.ty_error(e);
         }
 
         let definition_ty = instantiated_ty
             .remap_generic_params_to_declaration_params(opaque_type_key, self.tcx, false)
             .ty;
 
-        if !check_opaque_type_parameter_valid(
+        if let Err(guar) = check_opaque_type_parameter_valid(
             self.tcx,
             opaque_type_key,
             origin,
             instantiated_ty.span,
         ) {
-            return self.tcx.ty_error();
+            return self.tcx.ty_error(guar);
         }
 
         // Only check this for TAIT. RPIT already supports `tests/ui/impl-trait/nested-return-type2.rs`
@@ -273,9 +273,9 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
         // This logic duplicates most of `check_opaque_meets_bounds`.
         // FIXME(oli-obk): Also do region checks here and then consider removing `check_opaque_meets_bounds` entirely.
         let param_env = self.tcx.param_env(def_id);
-        let body_id = self.tcx.local_def_id_to_hir_id(def_id);
         // HACK This bubble is required for this tests to pass:
-        // type-alias-impl-trait/issue-67844-nested-opaque.rs
+        // nested-return-type2-tait2.rs
+        // nested-return-type2-tait3.rs
         let infcx =
             self.tcx.infer_ctxt().with_opaque_type_inference(DefiningAnchor::Bubble).build();
         let ocx = ObligationCtxt::new(&infcx);
@@ -290,7 +290,7 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
         // the bounds that the function supplies.
         let opaque_ty = self.tcx.mk_opaque(def_id.to_def_id(), id_substs);
         if let Err(err) = ocx.eq(
-            &ObligationCause::misc(instantiated_ty.span, body_id),
+            &ObligationCause::misc(instantiated_ty.span, def_id),
             param_env,
             opaque_ty,
             definition_ty,
@@ -298,7 +298,7 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
             infcx
                 .err_ctxt()
                 .report_mismatched_types(
-                    &ObligationCause::misc(instantiated_ty.span, body_id),
+                    &ObligationCause::misc(instantiated_ty.span, def_id),
                     opaque_ty,
                     definition_ty,
                     err,
@@ -309,7 +309,7 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
         ocx.register_obligation(Obligation::misc(
             infcx.tcx,
             instantiated_ty.span,
-            body_id,
+            def_id,
             param_env,
             predicate,
         ));
@@ -326,7 +326,7 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
             definition_ty
         } else {
             let reported = infcx.err_ctxt().report_fulfillment_errors(&errors, None);
-            self.tcx.ty_error_with_guaranteed(reported)
+            self.tcx.ty_error(reported)
         }
     }
 }
@@ -336,7 +336,7 @@ fn check_opaque_type_parameter_valid(
     opaque_type_key: OpaqueTypeKey<'_>,
     origin: OpaqueTyOrigin,
     span: Span,
-) -> bool {
+) -> Result<(), ErrorGuaranteed> {
     match origin {
         // No need to check return position impl trait (RPIT)
         // because for type and const parameters they are correct
@@ -359,7 +359,7 @@ fn check_opaque_type_parameter_valid(
         // fn foo<l0..'ln>() -> foo::<'static..'static>::Foo<'l0..'lm>.
         //
         // which would error here on all of the `'static` args.
-        OpaqueTyOrigin::FnReturn(..) | OpaqueTyOrigin::AsyncFn(..) => return true,
+        OpaqueTyOrigin::FnReturn(..) | OpaqueTyOrigin::AsyncFn(..) => return Ok(()),
         // Check these
         OpaqueTyOrigin::TyAlias => {}
     }
@@ -368,18 +368,6 @@ fn check_opaque_type_parameter_valid(
     for (i, arg) in opaque_type_key.substs.iter().enumerate() {
         let arg_is_param = match arg.unpack() {
             GenericArgKind::Type(ty) => matches!(ty.kind(), ty::Param(_)),
-            GenericArgKind::Lifetime(lt) if lt.is_static() => {
-                tcx.sess
-                    .struct_span_err(span, "non-defining opaque type use in defining scope")
-                    .span_label(
-                        tcx.def_span(opaque_generics.param_at(i, tcx).def_id),
-                        "cannot use static lifetime; use a bound lifetime \
-                                    instead or remove the lifetime parameter from the \
-                                    opaque type",
-                    )
-                    .emit();
-                return false;
-            }
             GenericArgKind::Lifetime(lt) => {
                 matches!(*lt, ty::ReEarlyBound(_) | ty::ReFree(_))
             }
@@ -392,13 +380,13 @@ fn check_opaque_type_parameter_valid(
             // Prevent `fn foo() -> Foo<u32>` from being defining.
             let opaque_param = opaque_generics.param_at(i, tcx);
             let kind = opaque_param.kind.descr();
-            tcx.sess.emit_err(NonGenericOpaqueTypeParam {
+
+            return Err(tcx.sess.emit_err(NonGenericOpaqueTypeParam {
                 ty: arg,
                 kind,
                 span,
                 param_span: tcx.def_span(opaque_param.def_id),
-            });
-            return false;
+            }));
         }
     }
 
@@ -409,12 +397,13 @@ fn check_opaque_type_parameter_valid(
                 .into_iter()
                 .map(|i| tcx.def_span(opaque_generics.param_at(i, tcx).def_id))
                 .collect();
-            tcx.sess
+            return Err(tcx
+                .sess
                 .struct_span_err(span, "non-defining opaque type use in defining scope")
                 .span_note(spans, &format!("{} used multiple times", descr))
-                .emit();
-            return false;
+                .emit());
         }
     }
-    true
+
+    Ok(())
 }
diff --git a/compiler/rustc_borrowck/src/region_infer/values.rs b/compiler/rustc_borrowck/src/region_infer/values.rs
index c3dfeedc2..8132800f1 100644
--- a/compiler/rustc_borrowck/src/region_infer/values.rs
+++ b/compiler/rustc_borrowck/src/region_infer/values.rs
@@ -181,12 +181,13 @@ impl<N: Idx> LivenessValues<N> {
 /// Maps from `ty::PlaceholderRegion` values that are used in the rest of
 /// rustc to the internal `PlaceholderIndex` values that are used in
 /// NLL.
-#[derive(Default)]
+#[derive(Debug, Default)]
 pub(crate) struct PlaceholderIndices {
     indices: FxIndexSet<ty::PlaceholderRegion>,
 }
 
 impl PlaceholderIndices {
+    /// Returns the `PlaceholderIndex` for the inserted `PlaceholderRegion`
     pub(crate) fn insert(&mut self, placeholder: ty::PlaceholderRegion) -> PlaceholderIndex {
         let (index, _) = self.indices.insert_full(placeholder);
         index.into()
@@ -234,7 +235,7 @@ pub(crate) struct RegionValues<N: Idx> {
     free_regions: SparseBitMatrix<N, RegionVid>,
 
     /// Placeholders represent bound regions -- so something like `'a`
-    /// in for<'a> fn(&'a u32)`.
+    /// in `for<'a> fn(&'a u32)`.
     placeholders: SparseBitMatrix<N, PlaceholderIndex>,
 }
 
diff --git a/compiler/rustc_borrowck/src/renumber.rs b/compiler/rustc_borrowck/src/renumber.rs
index 084754830..016f6f78d 100644
--- a/compiler/rustc_borrowck/src/renumber.rs
+++ b/compiler/rustc_borrowck/src/renumber.rs
@@ -1,18 +1,20 @@
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
+use crate::BorrowckInferCtxt;
 use rustc_index::vec::IndexVec;
-use rustc_infer::infer::{InferCtxt, NllRegionVariableOrigin};
+use rustc_infer::infer::NllRegionVariableOrigin;
 use rustc_middle::mir::visit::{MutVisitor, TyContext};
 use rustc_middle::mir::Constant;
 use rustc_middle::mir::{Body, Location, Promoted};
 use rustc_middle::ty::subst::SubstsRef;
 use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable};
+use rustc_span::{Span, Symbol};
 
 /// Replaces all free regions appearing in the MIR with fresh
 /// inference variables, returning the number of variables created.
 #[instrument(skip(infcx, body, promoted), level = "debug")]
 pub fn renumber_mir<'tcx>(
-    infcx: &InferCtxt<'tcx>,
+    infcx: &BorrowckInferCtxt<'_, 'tcx>,
     body: &mut Body<'tcx>,
     promoted: &mut IndexVec<Promoted, Body<'tcx>>,
 ) {
@@ -29,27 +31,68 @@ pub fn renumber_mir<'tcx>(
 
 /// Replaces all regions appearing in `value` with fresh inference
 /// variables.
-#[instrument(skip(infcx), level = "debug")]
-pub fn renumber_regions<'tcx, T>(infcx: &InferCtxt<'tcx>, value: T) -> T
+#[instrument(skip(infcx, get_ctxt_fn), level = "debug")]
+pub(crate) fn renumber_regions<'tcx, T, F>(
+    infcx: &BorrowckInferCtxt<'_, 'tcx>,
+    value: T,
+    get_ctxt_fn: F,
+) -> T
 where
-    T: TypeFoldable<'tcx>,
+    T: TypeFoldable<TyCtxt<'tcx>>,
+    F: Fn() -> RegionCtxt,
 {
     infcx.tcx.fold_regions(value, |_region, _depth| {
         let origin = NllRegionVariableOrigin::Existential { from_forall: false };
-        infcx.next_nll_region_var(origin)
+        infcx.next_nll_region_var(origin, || get_ctxt_fn())
     })
 }
 
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
+pub(crate) enum BoundRegionInfo {
+    Name(Symbol),
+    Span(Span),
+}
+
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
+pub(crate) enum RegionCtxt {
+    Location(Location),
+    TyContext(TyContext),
+    Free(Symbol),
+    Bound(BoundRegionInfo),
+    LateBound(BoundRegionInfo),
+    Existential(Option<Symbol>),
+    Placeholder(BoundRegionInfo),
+    Unknown,
+}
+
+impl RegionCtxt {
+    /// Used to determine the representative of a component in the strongly connected
+    /// constraint graph
+    pub(crate) fn preference_value(self) -> usize {
+        let _anon = Symbol::intern("anon");
+
+        match self {
+            RegionCtxt::Unknown => 1,
+            RegionCtxt::Existential(None) => 2,
+            RegionCtxt::Existential(Some(_anon)) | RegionCtxt::Free(_anon) => 2,
+            RegionCtxt::Location(_) => 3,
+            RegionCtxt::TyContext(_) => 4,
+            _ => 5,
+        }
+    }
+}
+
 struct NllVisitor<'a, 'tcx> {
-    infcx: &'a InferCtxt<'tcx>,
+    infcx: &'a BorrowckInferCtxt<'a, 'tcx>,
 }
 
 impl<'a, 'tcx> NllVisitor<'a, 'tcx> {
-    fn renumber_regions<T>(&mut self, value: T) -> T
+    fn renumber_regions<T, F>(&mut self, value: T, region_ctxt_fn: F) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
+        F: Fn() -> RegionCtxt,
     {
-        renumber_regions(self.infcx, value)
+        renumber_regions(self.infcx, value, region_ctxt_fn)
     }
 }
 
@@ -60,14 +103,14 @@ impl<'a, 'tcx> MutVisitor<'tcx> for NllVisitor<'a, 'tcx> {
 
     #[instrument(skip(self), level = "debug")]
     fn visit_ty(&mut self, ty: &mut Ty<'tcx>, ty_context: TyContext) {
-        *ty = self.renumber_regions(*ty);
+        *ty = self.renumber_regions(*ty, || RegionCtxt::TyContext(ty_context));
 
         debug!(?ty);
     }
 
     #[instrument(skip(self), level = "debug")]
     fn visit_substs(&mut self, substs: &mut SubstsRef<'tcx>, location: Location) {
-        *substs = self.renumber_regions(*substs);
+        *substs = self.renumber_regions(*substs, || RegionCtxt::Location(location));
 
         debug!(?substs);
     }
@@ -75,7 +118,7 @@ impl<'a, 'tcx> MutVisitor<'tcx> for NllVisitor<'a, 'tcx> {
     #[instrument(skip(self), level = "debug")]
     fn visit_region(&mut self, region: &mut ty::Region<'tcx>, location: Location) {
         let old_region = *region;
-        *region = self.renumber_regions(old_region);
+        *region = self.renumber_regions(old_region, || RegionCtxt::Location(location));
 
         debug!(?region);
     }
@@ -83,7 +126,7 @@ impl<'a, 'tcx> MutVisitor<'tcx> for NllVisitor<'a, 'tcx> {
     #[instrument(skip(self), level = "debug")]
     fn visit_constant(&mut self, constant: &mut Constant<'tcx>, _location: Location) {
         let literal = constant.literal;
-        constant.literal = self.renumber_regions(literal);
+        constant.literal = self.renumber_regions(literal, || RegionCtxt::Location(_location));
         debug!("constant: {:#?}", constant);
     }
 }
diff --git a/compiler/rustc_borrowck/src/session_diagnostics.rs b/compiler/rustc_borrowck/src/session_diagnostics.rs
index 23acf1592..a36789290 100644
--- a/compiler/rustc_borrowck/src/session_diagnostics.rs
+++ b/compiler/rustc_borrowck/src/session_diagnostics.rs
@@ -1,4 +1,4 @@
-use rustc_errors::{IntoDiagnosticArg, MultiSpan};
+use rustc_errors::MultiSpan;
 use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
 use rustc_middle::ty::{GenericArg, Ty};
 use rustc_span::Span;
@@ -55,7 +55,7 @@ pub(crate) struct VarNeedNotMut {
 #[derive(Diagnostic)]
 #[diag(borrowck_var_cannot_escape_closure)]
 #[note]
-#[note(cannot_escape)]
+#[note(borrowck_cannot_escape)]
 pub(crate) struct FnMutError {
     #[primary_span]
     pub span: Span,
@@ -128,18 +128,6 @@ pub(crate) enum LifetimeReturnCategoryErr<'a> {
     },
 }
 
-impl IntoDiagnosticArg for &RegionName {
-    fn into_diagnostic_arg(self) -> rustc_errors::DiagnosticArgValue<'static> {
-        format!("{}", self).into_diagnostic_arg()
-    }
-}
-
-impl IntoDiagnosticArg for RegionName {
-    fn into_diagnostic_arg(self) -> rustc_errors::DiagnosticArgValue<'static> {
-        format!("{}", self).into_diagnostic_arg()
-    }
-}
-
 #[derive(Subdiagnostic)]
 pub(crate) enum RequireStaticErr {
     #[note(borrowck_used_impl_require_static)]
@@ -235,7 +223,7 @@ pub(crate) struct MoveBorrow<'a> {
     pub borrow_place: &'a str,
     pub value_place: &'a str,
     #[primary_span]
-    #[label(move_label)]
+    #[label(borrowck_move_label)]
     pub span: Span,
     #[label]
     pub borrow_span: Span,
diff --git a/compiler/rustc_borrowck/src/type_check/canonical.rs b/compiler/rustc_borrowck/src/type_check/canonical.rs
index 11729e2c8..b27d5d205 100644
--- a/compiler/rustc_borrowck/src/type_check/canonical.rs
+++ b/compiler/rustc_borrowck/src/type_check/canonical.rs
@@ -2,7 +2,7 @@ use std::fmt;
 
 use rustc_infer::infer::{canonical::Canonical, InferOk};
 use rustc_middle::mir::ConstraintCategory;
-use rustc_middle::ty::{self, ToPredicate, Ty, TypeFoldable};
+use rustc_middle::ty::{self, ToPredicate, Ty, TyCtxt, TypeFoldable};
 use rustc_span::def_id::DefId;
 use rustc_span::Span;
 use rustc_trait_selection::traits::query::type_op::{self, TypeOpOutput};
@@ -66,7 +66,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         canonical: &Canonical<'tcx, T>,
     ) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         let old_universe = self.infcx.universe();
 
@@ -117,7 +117,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
 
     pub(super) fn prove_predicates(
         &mut self,
-        predicates: impl IntoIterator<Item = impl ToPredicate<'tcx> + std::fmt::Debug>,
+        predicates: impl IntoIterator<Item: ToPredicate<'tcx> + std::fmt::Debug>,
         locations: Locations,
         category: ConstraintCategory<'tcx>,
     ) {
@@ -181,9 +181,6 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         user_ty: ty::UserType<'tcx>,
         span: Span,
     ) {
-        // FIXME: Ideally MIR types are normalized, but this is not always true.
-        let mir_ty = self.normalize(mir_ty, Locations::All(span));
-
         self.fully_perform_op(
             Locations::All(span),
             ConstraintCategory::Boring,
@@ -217,7 +214,9 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             return;
         }
 
+        // FIXME: Ideally MIR types are normalized, but this is not always true.
         let mir_ty = self.normalize(mir_ty, Locations::All(span));
+
         let cause = ObligationCause::dummy_with_span(span);
         let param_env = self.param_env;
         let op = |infcx: &'_ _| {
diff --git a/compiler/rustc_borrowck/src/type_check/constraint_conversion.rs b/compiler/rustc_borrowck/src/type_check/constraint_conversion.rs
index e15d1b99a..a93561350 100644
--- a/compiler/rustc_borrowck/src/type_check/constraint_conversion.rs
+++ b/compiler/rustc_borrowck/src/type_check/constraint_conversion.rs
@@ -6,8 +6,8 @@ use rustc_infer::infer::region_constraints::{GenericKind, VerifyBound};
 use rustc_infer::infer::{self, InferCtxt, SubregionOrigin};
 use rustc_middle::mir::{ClosureOutlivesSubject, ClosureRegionRequirements, ConstraintCategory};
 use rustc_middle::ty::subst::GenericArgKind;
-use rustc_middle::ty::TypeFoldable;
 use rustc_middle::ty::{self, TyCtxt};
+use rustc_middle::ty::{TypeFoldable, TypeVisitableExt};
 use rustc_span::{Span, DUMMY_SP};
 
 use crate::{
@@ -83,16 +83,8 @@ impl<'a, 'tcx> ConstraintConversion<'a, 'tcx> {
         }
         self.constraints.member_constraints = tmp;
 
-        for (predicate, constraint_category) in outlives {
-            // At the moment, we never generate any "higher-ranked"
-            // region constraints like `for<'a> 'a: 'b`. At some point
-            // when we move to universes, we will, and this assertion
-            // will start to fail.
-            let predicate = predicate.no_bound_vars().unwrap_or_else(|| {
-                bug!("query_constraint {:?} contained bound vars", predicate,);
-            });
-
-            self.convert(predicate, *constraint_category);
+        for &(predicate, constraint_category) in outlives {
+            self.convert(predicate, constraint_category);
         }
     }
 
@@ -124,7 +116,9 @@ impl<'a, 'tcx> ConstraintConversion<'a, 'tcx> {
             let outlived_region = closure_mapping[outlives_requirement.outlived_free_region];
             let subject = match outlives_requirement.subject {
                 ClosureOutlivesSubject::Region(re) => closure_mapping[re].into(),
-                ClosureOutlivesSubject::Ty(ty) => ty.into(),
+                ClosureOutlivesSubject::Ty(subject_ty) => {
+                    subject_ty.instantiate(self.tcx, |vid| closure_mapping[vid]).into()
+                }
             };
 
             self.category = outlives_requirement.category;
@@ -179,7 +173,7 @@ impl<'a, 'tcx> ConstraintConversion<'a, 'tcx> {
     ///
     /// FIXME: This should get removed once higher ranked region obligations
     /// are dealt with during trait solving.
-    fn replace_placeholders_with_nll<T: TypeFoldable<'tcx>>(&mut self, value: T) -> T {
+    fn replace_placeholders_with_nll<T: TypeFoldable<TyCtxt<'tcx>>>(&mut self, value: T) -> T {
         if value.has_placeholders() {
             self.tcx.fold_regions(value, |r, _| match *r {
                 ty::RePlaceholder(placeholder) => {
diff --git a/compiler/rustc_borrowck/src/type_check/free_region_relations.rs b/compiler/rustc_borrowck/src/type_check/free_region_relations.rs
index 82ff86247..4004966c4 100644
--- a/compiler/rustc_borrowck/src/type_check/free_region_relations.rs
+++ b/compiler/rustc_borrowck/src/type_check/free_region_relations.rs
@@ -8,6 +8,7 @@ use rustc_infer::infer::InferCtxt;
 use rustc_middle::mir::ConstraintCategory;
 use rustc_middle::traits::query::OutlivesBound;
 use rustc_middle::ty::{self, RegionVid, Ty};
+use rustc_span::Span;
 use rustc_trait_selection::traits::query::type_op::{self, TypeOp};
 use std::rc::Rc;
 use type_op::TypeOpOutput;
@@ -92,31 +93,6 @@ impl UniversalRegionRelations<'_> {
         res
     }
 
-    /// Returns the "postdominating" bound of the set of
-    /// `non_local_upper_bounds` for the given region.
-    pub(crate) fn non_local_upper_bound(&self, fr: RegionVid) -> RegionVid {
-        let upper_bounds = self.non_local_upper_bounds(fr);
-
-        // In case we find more than one, reduce to one for
-        // convenience. This is to prevent us from generating more
-        // complex constraints, but it will cause spurious errors.
-        let post_dom = self.inverse_outlives.mutual_immediate_postdominator(upper_bounds);
-
-        debug!("non_local_bound: post_dom={:?}", post_dom);
-
-        post_dom
-            .and_then(|post_dom| {
-                // If the mutual immediate postdom is not local, then
-                // there is no non-local result we can return.
-                if !self.universal_regions.is_local_free_region(post_dom) {
-                    Some(post_dom)
-                } else {
-                    None
-                }
-            })
-            .unwrap_or(self.universal_regions.fr_static)
-    }
-
     /// Finds a "lower bound" for `fr` that is not local. In other
     /// words, returns the largest (*) known region `fr1` that (a) is
     /// outlived by `fr` and (b) is not local.
@@ -217,8 +193,27 @@ impl<'tcx> UniversalRegionRelationsBuilder<'_, 'tcx> {
         self.inverse_outlives.add(fr_b, fr_a);
     }
 
+    #[instrument(level = "debug", skip(self))]
     pub(crate) fn create(mut self) -> CreateResult<'tcx> {
         let span = self.infcx.tcx.def_span(self.universal_regions.defining_ty.def_id());
+
+        // Insert the facts we know from the predicates. Why? Why not.
+        let param_env = self.param_env;
+        self.add_outlives_bounds(outlives::explicit_outlives_bounds(param_env));
+
+        // - outlives is reflexive, so `'r: 'r` for every region `'r`
+        // - `'static: 'r` for every region `'r`
+        // - `'r: 'fn_body` for every (other) universally quantified
+        //   region `'r`, all of which are provided by our caller
+        let fr_static = self.universal_regions.fr_static;
+        let fr_fn_body = self.universal_regions.fr_fn_body;
+        for fr in self.universal_regions.universal_regions() {
+            debug!("build: relating free region {:?} to itself and to 'static", fr);
+            self.relate_universal_regions(fr, fr);
+            self.relate_universal_regions(fr_static, fr);
+            self.relate_universal_regions(fr, fr_fn_body);
+        }
+
         let unnormalized_input_output_tys = self
             .universal_regions
             .unnormalized_input_tys
@@ -236,78 +231,59 @@ impl<'tcx> UniversalRegionRelationsBuilder<'_, 'tcx> {
         //   the `relations` is built.
         let mut normalized_inputs_and_output =
             Vec::with_capacity(self.universal_regions.unnormalized_input_tys.len() + 1);
-        let constraint_sets: Vec<_> = unnormalized_input_output_tys
-            .flat_map(|ty| {
-                debug!("build: input_or_output={:?}", ty);
-                // We add implied bounds from both the unnormalized and normalized ty.
-                // See issue #87748
-                let constraints_implied1 = self.add_implied_bounds(ty);
-                let TypeOpOutput { output: norm_ty, constraints: constraints1, .. } = self
-                    .param_env
-                    .and(type_op::normalize::Normalize::new(ty))
-                    .fully_perform(self.infcx)
-                    .unwrap_or_else(|_| {
-                        let reported = self
-                            .infcx
-                            .tcx
-                            .sess
-                            .delay_span_bug(span, &format!("failed to normalize {:?}", ty));
-                        TypeOpOutput {
-                            output: self.infcx.tcx.ty_error_with_guaranteed(reported),
-                            constraints: None,
-                            error_info: None,
-                        }
-                    });
-                // Note: we need this in examples like
-                // ```
-                // trait Foo {
-                //   type Bar;
-                //   fn foo(&self) -> &Self::Bar;
-                // }
-                // impl Foo for () {
-                //   type Bar = ();
-                //   fn foo(&self) -> &() {}
-                // }
-                // ```
-                // Both &Self::Bar and &() are WF
-                let constraints_implied2 =
-                    if ty != norm_ty { self.add_implied_bounds(norm_ty) } else { None };
-                normalized_inputs_and_output.push(norm_ty);
-                constraints1.into_iter().chain(constraints_implied1).chain(constraints_implied2)
-            })
-            .collect();
+        let mut constraints = vec![];
+        for ty in unnormalized_input_output_tys {
+            debug!("build: input_or_output={:?}", ty);
+            // We add implied bounds from both the unnormalized and normalized ty.
+            // See issue #87748
+            let constraints_unnorm = self.add_implied_bounds(ty);
+            if let Some(c) = constraints_unnorm {
+                constraints.push(c)
+            }
+            let TypeOpOutput { output: norm_ty, constraints: constraints_normalize, .. } = self
+                .param_env
+                .and(type_op::normalize::Normalize::new(ty))
+                .fully_perform(self.infcx)
+                .unwrap_or_else(|_| {
+                    let guar = self
+                        .infcx
+                        .tcx
+                        .sess
+                        .delay_span_bug(span, &format!("failed to normalize {:?}", ty));
+                    TypeOpOutput {
+                        output: self.infcx.tcx.ty_error(guar),
+                        constraints: None,
+                        error_info: None,
+                    }
+                });
+            if let Some(c) = constraints_normalize {
+                constraints.push(c)
+            }
 
-        // Insert the facts we know from the predicates. Why? Why not.
-        let param_env = self.param_env;
-        self.add_outlives_bounds(outlives::explicit_outlives_bounds(param_env));
+            // Note: we need this in examples like
+            // ```
+            // trait Foo {
+            //   type Bar;
+            //   fn foo(&self) -> &Self::Bar;
+            // }
+            // impl Foo for () {
+            //   type Bar = ();
+            //   fn foo(&self) ->&() {}
+            // }
+            // ```
+            // Both &Self::Bar and &() are WF
+            if ty != norm_ty {
+                let constraints_norm = self.add_implied_bounds(norm_ty);
+                if let Some(c) = constraints_norm {
+                    constraints.push(c)
+                }
+            }
 
-        // Finally:
-        // - outlives is reflexive, so `'r: 'r` for every region `'r`
-        // - `'static: 'r` for every region `'r`
-        // - `'r: 'fn_body` for every (other) universally quantified
-        //   region `'r`, all of which are provided by our caller
-        let fr_static = self.universal_regions.fr_static;
-        let fr_fn_body = self.universal_regions.fr_fn_body;
-        for fr in self.universal_regions.universal_regions() {
-            debug!("build: relating free region {:?} to itself and to 'static", fr);
-            self.relate_universal_regions(fr, fr);
-            self.relate_universal_regions(fr_static, fr);
-            self.relate_universal_regions(fr, fr_fn_body);
+            normalized_inputs_and_output.push(norm_ty);
         }
 
-        for data in &constraint_sets {
-            constraint_conversion::ConstraintConversion::new(
-                self.infcx,
-                &self.universal_regions,
-                &self.region_bound_pairs,
-                self.implicit_region_bound,
-                self.param_env,
-                Locations::All(span),
-                span,
-                ConstraintCategory::Internal,
-                &mut self.constraints,
-            )
-            .convert_all(data);
+        for c in constraints {
+            self.push_region_constraints(c, span);
         }
 
         CreateResult {
@@ -321,6 +297,24 @@ impl<'tcx> UniversalRegionRelationsBuilder<'_, 'tcx> {
         }
     }
 
+    #[instrument(skip(self, data), level = "debug")]
+    fn push_region_constraints(&mut self, data: &QueryRegionConstraints<'tcx>, span: Span) {
+        debug!("constraints generated: {:#?}", data);
+
+        constraint_conversion::ConstraintConversion::new(
+            self.infcx,
+            &self.universal_regions,
+            &self.region_bound_pairs,
+            self.implicit_region_bound,
+            self.param_env,
+            Locations::All(span),
+            span,
+            ConstraintCategory::Internal,
+            &mut self.constraints,
+        )
+        .convert_all(data);
+    }
+
     /// Update the type of a single local, which should represent
     /// either the return type of the MIR or one of its arguments. At
     /// the same time, compute and add any implied bounds that come
@@ -332,6 +326,7 @@ impl<'tcx> UniversalRegionRelationsBuilder<'_, 'tcx> {
             .and(type_op::implied_outlives_bounds::ImpliedOutlivesBounds { ty })
             .fully_perform(self.infcx)
             .unwrap_or_else(|_| bug!("failed to compute implied bounds {:?}", ty));
+        debug!(?bounds, ?constraints);
         self.add_outlives_bounds(bounds);
         constraints
     }
diff --git a/compiler/rustc_borrowck/src/type_check/input_output.rs b/compiler/rustc_borrowck/src/type_check/input_output.rs
index fa9ea769a..717020ea5 100644
--- a/compiler/rustc_borrowck/src/type_check/input_output.rs
+++ b/compiler/rustc_borrowck/src/type_check/input_output.rs
@@ -26,11 +26,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         if !self.tcx().is_closure(mir_def_id.to_def_id()) {
             return;
         }
-        let Some(user_provided_poly_sig) =
-            self.tcx().typeck(mir_def_id).user_provided_sigs.get(&mir_def_id)
-        else {
-            return;
-        };
+        let user_provided_poly_sig = self.tcx().closure_user_provided_sig(mir_def_id);
 
         // Instantiate the canonicalized variables from user-provided signature
         // (e.g., the `_` in the code above) with fresh variables.
@@ -38,7 +34,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         // so that they represent the view from "inside" the closure.
         let user_provided_sig = self
             .instantiate_canonical_with_fresh_inference_vars(body.span, &user_provided_poly_sig);
-        let user_provided_sig = self.infcx.replace_bound_vars_with_fresh_vars(
+        let user_provided_sig = self.infcx.instantiate_binder_with_fresh_vars(
             body.span,
             LateBoundRegionConversionTime::FnCall,
             user_provided_sig,
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
index d5c401ae1..a411aec51 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
@@ -50,13 +50,11 @@ pub(super) fn generate<'mir, 'tcx>(
         compute_relevant_live_locals(typeck.tcx(), &free_regions, &body);
     let facts_enabled = use_polonius || AllFacts::enabled(typeck.tcx());
 
-    let polonius_drop_used = if facts_enabled {
+    let polonius_drop_used = facts_enabled.then(|| {
         let mut drop_used = Vec::new();
         polonius::populate_access_facts(typeck, body, location_table, move_data, &mut drop_used);
-        Some(drop_used)
-    } else {
-        None
-    };
+        drop_used
+    });
 
     trace::trace(
         typeck,
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
index 3ff5d188a..473c05963 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
@@ -3,7 +3,7 @@ use rustc_index::bit_set::HybridBitSet;
 use rustc_index::interval::IntervalSet;
 use rustc_infer::infer::canonical::QueryRegionConstraints;
 use rustc_middle::mir::{BasicBlock, Body, ConstraintCategory, Local, Location};
-use rustc_middle::ty::{Ty, TypeVisitable};
+use rustc_middle::ty::{Ty, TyCtxt, TypeVisitable, TypeVisitableExt};
 use rustc_trait_selection::traits::query::dropck_outlives::DropckOutlivesResult;
 use rustc_trait_selection::traits::query::type_op::outlives::DropckOutlives;
 use rustc_trait_selection::traits::query::type_op::{TypeOp, TypeOpOutput};
@@ -477,7 +477,7 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
     /// points `live_at`.
     fn add_use_live_facts_for(
         &mut self,
-        value: impl TypeVisitable<'tcx>,
+        value: impl TypeVisitable<TyCtxt<'tcx>>,
         live_at: &IntervalSet<PointIndex>,
     ) {
         debug!("add_use_live_facts_for(value={:?})", value);
@@ -542,7 +542,7 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
     fn make_all_regions_live(
         elements: &RegionValueElements,
         typeck: &mut TypeChecker<'_, 'tcx>,
-        value: impl TypeVisitable<'tcx>,
+        value: impl TypeVisitable<TyCtxt<'tcx>>,
         live_at: &IntervalSet<PointIndex>,
     ) {
         debug!("make_all_regions_live(value={:?})", value);
diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs
index 81bd4c2a7..a49da3da6 100644
--- a/compiler/rustc_borrowck/src/type_check/mod.rs
+++ b/compiler/rustc_borrowck/src/type_check/mod.rs
@@ -30,7 +30,7 @@ use rustc_middle::mir::*;
 use rustc_middle::ty::adjustment::PointerCast;
 use rustc_middle::ty::cast::CastTy;
 use rustc_middle::ty::subst::{SubstsRef, UserSubsts};
-use rustc_middle::ty::visit::TypeVisitable;
+use rustc_middle::ty::visit::TypeVisitableExt;
 use rustc_middle::ty::{
     self, Binder, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, Dynamic,
     OpaqueHiddenType, OpaqueTypeKey, RegionVid, Ty, TyCtxt, UserType, UserTypeAnnotationIndex,
@@ -64,7 +64,7 @@ use crate::{
     region_infer::TypeTest,
     type_check::free_region_relations::{CreateResult, UniversalRegionRelations},
     universal_regions::{DefiningTy, UniversalRegions},
-    Upvar,
+    BorrowckInferCtxt, Upvar,
 };
 
 macro_rules! span_mirbug {
@@ -123,7 +123,7 @@ mod relate_tys;
 /// - `move_data` -- move-data constructed when performing the maybe-init dataflow analysis
 /// - `elements` -- MIR region map
 pub(crate) fn type_check<'mir, 'tcx>(
-    infcx: &InferCtxt<'tcx>,
+    infcx: &BorrowckInferCtxt<'_, 'tcx>,
     param_env: ty::ParamEnv<'tcx>,
     body: &Body<'tcx>,
     promoted: &IndexVec<Promoted, Body<'tcx>>,
@@ -137,7 +137,7 @@ pub(crate) fn type_check<'mir, 'tcx>(
     upvars: &[Upvar<'tcx>],
     use_polonius: bool,
 ) -> MirTypeckResults<'tcx> {
-    let implicit_region_bound = infcx.tcx.mk_region(ty::ReVar(universal_regions.fr_fn_body));
+    let implicit_region_bound = infcx.tcx.mk_re_var(universal_regions.fr_fn_body);
     let mut constraints = MirTypeckRegionConstraints {
         placeholder_indices: PlaceholderIndices::default(),
         placeholder_index_to_region: IndexVec::default(),
@@ -239,7 +239,7 @@ pub(crate) fn type_check<'mir, 'tcx>(
                     decl.hidden_type.span,
                     &format!("could not resolve {:#?}", hidden_type.ty.kind()),
                 );
-                hidden_type.ty = infcx.tcx.ty_error_with_guaranteed(reported);
+                hidden_type.ty = infcx.tcx.ty_error(reported);
             }
 
             (opaque_type_key, (hidden_type, decl.origin))
@@ -402,7 +402,7 @@ impl<'a, 'b, 'tcx> Visitor<'tcx> for TypeVerifier<'a, 'b, 'tcx> {
                     );
                 }
             } else if let Some(static_def_id) = constant.check_static_ptr(tcx) {
-                let unnormalized_ty = tcx.type_of(static_def_id);
+                let unnormalized_ty = tcx.type_of(static_def_id).subst_identity();
                 let normalized_ty = self.cx.normalize(unnormalized_ty, locations);
                 let literal_ty = constant.literal.ty().builtin_deref(true).unwrap().ty;
 
@@ -529,12 +529,12 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> {
 
         for elem in place.projection.iter() {
             if place_ty.variant_index.is_none() {
-                if place_ty.ty.references_error() {
+                if let Err(guar) = place_ty.ty.error_reported() {
                     assert!(self.errors_reported);
-                    return PlaceTy::from_ty(self.tcx().ty_error());
+                    return PlaceTy::from_ty(self.tcx().ty_error(guar));
                 }
             }
-            place_ty = self.sanitize_projection(place_ty, elem, place, location);
+            place_ty = self.sanitize_projection(place_ty, elem, place, location, context);
         }
 
         if let PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy) = context {
@@ -630,12 +630,14 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> {
         }
     }
 
+    #[instrument(skip(self), level = "debug")]
     fn sanitize_projection(
         &mut self,
         base: PlaceTy<'tcx>,
         pi: PlaceElem<'tcx>,
         place: &Place<'tcx>,
         location: Location,
+        context: PlaceContext,
     ) -> PlaceTy<'tcx> {
         debug!("sanitize_projection: {:?} {:?} {:?}", base, pi, place);
         let tcx = self.tcx();
@@ -713,8 +715,11 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> {
                 match self.field_ty(place, base, field, location) {
                     Ok(ty) => {
                         let ty = self.cx.normalize(ty, location);
-                        if let Err(terr) = self.cx.eq_types(
+                        debug!(?fty, ?ty);
+
+                        if let Err(terr) = self.cx.relate_types(
                             ty,
+                            self.get_ambient_variance(context),
                             fty,
                             location.to_locations(),
                             ConstraintCategory::Boring,
@@ -743,9 +748,10 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> {
                 let ty = self.sanitize_type(place, ty);
                 let ty = self.cx.normalize(ty, location);
                 self.cx
-                    .eq_types(
-                        base.ty,
+                    .relate_types(
                         ty,
+                        self.get_ambient_variance(context),
+                        base.ty,
                         location.to_locations(),
                         ConstraintCategory::TypeAnnotation,
                     )
@@ -757,7 +763,22 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> {
 
     fn error(&mut self) -> Ty<'tcx> {
         self.errors_reported = true;
-        self.tcx().ty_error()
+        self.tcx().ty_error_misc()
+    }
+
+    fn get_ambient_variance(&self, context: PlaceContext) -> ty::Variance {
+        use rustc_middle::mir::visit::NonMutatingUseContext::*;
+        use rustc_middle::mir::visit::NonUseContext::*;
+
+        match context {
+            PlaceContext::MutatingUse(_) => ty::Invariant,
+            PlaceContext::NonUse(StorageDead | StorageLive | VarDebugInfo) => ty::Invariant,
+            PlaceContext::NonMutatingUse(
+                Inspect | Copy | Move | SharedBorrow | ShallowBorrow | UniqueBorrow | AddressOf
+                | Projection,
+            ) => ty::Covariant,
+            PlaceContext::NonUse(AscribeUserTy) => ty::Covariant,
+        }
     }
 
     fn field_ty(
@@ -845,7 +866,7 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> {
 /// way, it accrues region constraints -- these can later be used by
 /// NLL region checking.
 struct TypeChecker<'a, 'tcx> {
-    infcx: &'a InferCtxt<'tcx>,
+    infcx: &'a BorrowckInferCtxt<'a, 'tcx>,
     param_env: ty::ParamEnv<'tcx>,
     last_span: Span,
     body: &'a Body<'tcx>,
@@ -910,6 +931,8 @@ pub(crate) struct MirTypeckRegionConstraints<'tcx> {
 }
 
 impl<'tcx> MirTypeckRegionConstraints<'tcx> {
+    /// Creates a `Region` for a given `PlaceholderRegion`, or returns the
+    /// region that corresponds to a previously created one.
     fn placeholder_region(
         &mut self,
         infcx: &InferCtxt<'tcx>,
@@ -996,7 +1019,7 @@ impl Locations {
 
 impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
     fn new(
-        infcx: &'a InferCtxt<'tcx>,
+        infcx: &'a BorrowckInferCtxt<'a, 'tcx>,
         body: &'a Body<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
         region_bound_pairs: &'a RegionBoundPairs<'tcx>,
@@ -1258,6 +1281,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             | StatementKind::StorageDead(..)
             | StatementKind::Retag { .. }
             | StatementKind::Coverage(..)
+            | StatementKind::ConstEvalCounter
             | StatementKind::Nop => {}
             StatementKind::Deinit(..) | StatementKind::SetDiscriminant { .. } => {
                 bug!("Statement not allowed in this MIR phase")
@@ -1332,11 +1356,34 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                     }
                 };
                 let (sig, map) = tcx.replace_late_bound_regions(sig, |br| {
-                    self.infcx.next_region_var(LateBoundRegion(
-                        term.source_info.span,
-                        br.kind,
-                        LateBoundRegionConversionTime::FnCall,
-                    ))
+                    use crate::renumber::{BoundRegionInfo, RegionCtxt};
+                    use rustc_span::Symbol;
+
+                    let region_ctxt_fn = || {
+                        let reg_info = match br.kind {
+                            ty::BoundRegionKind::BrAnon(_, Some(span)) => {
+                                BoundRegionInfo::Span(span)
+                            }
+                            ty::BoundRegionKind::BrAnon(..) => {
+                                BoundRegionInfo::Name(Symbol::intern("anon"))
+                            }
+                            ty::BoundRegionKind::BrNamed(_, name) => BoundRegionInfo::Name(name),
+                            ty::BoundRegionKind::BrEnv => {
+                                BoundRegionInfo::Name(Symbol::intern("env"))
+                            }
+                        };
+
+                        RegionCtxt::LateBound(reg_info)
+                    };
+
+                    self.infcx.next_region_var(
+                        LateBoundRegion(
+                            term.source_info.span,
+                            br.kind,
+                            LateBoundRegionConversionTime::FnCall,
+                        ),
+                        region_ctxt_fn,
+                    )
                 });
                 debug!(?sig);
                 // IMPORTANT: We have to prove well formed for the function signature before
@@ -1483,7 +1530,10 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 }
             }
             None => {
-                if !sig.output().is_privately_uninhabited(self.tcx(), self.param_env) {
+                // The signature in this call can reference region variables,
+                // so erase them before calling a query.
+                let output_ty = self.tcx().erase_regions(sig.output());
+                if !output_ty.is_privately_uninhabited(self.tcx(), self.param_env) {
                     span_mirbug!(self, term, "call to converging function {:?} w/o dest", sig);
                 }
             }
@@ -1776,7 +1826,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 // than 1.
                 // If the length is larger than 1, the repeat expression will need to copy the
                 // element, so we require the `Copy` trait.
-                if len.try_eval_usize(tcx, self.param_env).map_or(true, |len| len > 1) {
+                if len.try_eval_target_usize(tcx, self.param_env).map_or(true, |len| len > 1) {
                     match operand {
                         Operand::Copy(..) | Operand::Constant(..) => {
                             // These are always okay: direct use of a const, or a value that can evidently be copied.
@@ -2027,7 +2077,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                             }
                         };
 
-                        if ty_to_mut == Mutability::Mut && ty_mut == Mutability::Not {
+                        if ty_to_mut.is_mut() && ty_mut.is_not() {
                             span_mirbug!(
                                 self,
                                 rvalue,
@@ -2532,7 +2582,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             // clauses on the struct.
             AggregateKind::Closure(def_id, substs)
             | AggregateKind::Generator(def_id, substs, _) => {
-                (def_id.to_def_id(), self.prove_closure_bounds(tcx, def_id, substs, location))
+                (def_id, self.prove_closure_bounds(tcx, def_id.expect_local(), substs, location))
             }
 
             AggregateKind::Array(_) | AggregateKind::Tuple => {
@@ -2583,7 +2633,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             DefKind::InlineConst => substs.as_inline_const().parent_substs(),
             other => bug!("unexpected item {:?}", other),
         };
-        let parent_substs = tcx.mk_substs(parent_substs.iter());
+        let parent_substs = tcx.mk_substs(parent_substs);
 
         assert_eq!(typeck_root_substs.len(), parent_substs.len());
         if let Err(_) = self.eq_substs(
diff --git a/compiler/rustc_borrowck/src/type_check/relate_tys.rs b/compiler/rustc_borrowck/src/type_check/relate_tys.rs
index b2702eafd..d96372fb9 100644
--- a/compiler/rustc_borrowck/src/type_check/relate_tys.rs
+++ b/compiler/rustc_borrowck/src/type_check/relate_tys.rs
@@ -1,14 +1,15 @@
-use rustc_infer::infer::nll_relate::{NormalizationStrategy, TypeRelating, TypeRelatingDelegate};
+use rustc_infer::infer::nll_relate::{TypeRelating, TypeRelatingDelegate};
 use rustc_infer::infer::NllRegionVariableOrigin;
 use rustc_infer::traits::PredicateObligations;
 use rustc_middle::mir::ConstraintCategory;
 use rustc_middle::ty::relate::TypeRelation;
 use rustc_middle::ty::{self, Ty};
-use rustc_span::Span;
+use rustc_span::{Span, Symbol};
 use rustc_trait_selection::traits::query::Fallible;
 
 use crate::constraints::OutlivesConstraint;
 use crate::diagnostics::UniverseInfo;
+use crate::renumber::{BoundRegionInfo, RegionCtxt};
 use crate::type_check::{InstantiateOpaqueType, Locations, TypeChecker};
 
 impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
@@ -100,23 +101,65 @@ impl<'tcx> TypeRelatingDelegate<'tcx> for NllTypeRelatingDelegate<'_, '_, 'tcx>
         universe
     }
 
-    fn next_existential_region_var(&mut self, from_forall: bool) -> ty::Region<'tcx> {
+    #[instrument(skip(self), level = "debug")]
+    fn next_existential_region_var(
+        &mut self,
+        from_forall: bool,
+        _name: Option<Symbol>,
+    ) -> ty::Region<'tcx> {
         let origin = NllRegionVariableOrigin::Existential { from_forall };
-        self.type_checker.infcx.next_nll_region_var(origin)
+
+        let reg_var =
+            self.type_checker.infcx.next_nll_region_var(origin, || RegionCtxt::Existential(_name));
+
+        reg_var
     }
 
+    #[instrument(skip(self), level = "debug")]
     fn next_placeholder_region(&mut self, placeholder: ty::PlaceholderRegion) -> ty::Region<'tcx> {
-        self.type_checker
+        let reg = self
+            .type_checker
             .borrowck_context
             .constraints
-            .placeholder_region(self.type_checker.infcx, placeholder)
+            .placeholder_region(self.type_checker.infcx, placeholder);
+
+        let reg_info = match placeholder.name {
+            ty::BoundRegionKind::BrAnon(_, Some(span)) => BoundRegionInfo::Span(span),
+            ty::BoundRegionKind::BrAnon(..) => BoundRegionInfo::Name(Symbol::intern("anon")),
+            ty::BoundRegionKind::BrNamed(_, name) => BoundRegionInfo::Name(name),
+            ty::BoundRegionKind::BrEnv => BoundRegionInfo::Name(Symbol::intern("env")),
+        };
+
+        let reg_var =
+            reg.as_var().unwrap_or_else(|| bug!("expected region {:?} to be of kind ReVar", reg));
+        let mut var_to_origin = self.type_checker.infcx.reg_var_to_origin.borrow_mut();
+        let prev = var_to_origin.insert(reg_var, RegionCtxt::Placeholder(reg_info));
+        assert!(matches!(prev, None));
+
+        reg
     }
 
+    #[instrument(skip(self), level = "debug")]
     fn generalize_existential(&mut self, universe: ty::UniverseIndex) -> ty::Region<'tcx> {
-        self.type_checker.infcx.next_nll_region_var_in_universe(
+        let reg = self.type_checker.infcx.next_nll_region_var_in_universe(
             NllRegionVariableOrigin::Existential { from_forall: false },
             universe,
-        )
+        );
+
+        let reg_var =
+            reg.as_var().unwrap_or_else(|| bug!("expected region {:?} to be of kind ReVar", reg));
+
+        if cfg!(debug_assertions) {
+            let mut var_to_origin = self.type_checker.infcx.reg_var_to_origin.borrow_mut();
+            let prev = var_to_origin.insert(reg_var, RegionCtxt::Existential(None));
+
+            // It only makes sense to track region vars in non-canonicalization contexts. If this
+            // ever changes we either want to get rid of `BorrowckInferContext::reg_var_to_origin`
+            // or modify how we track nll region vars for that map.
+            assert!(matches!(prev, None));
+        }
+
+        reg
     }
 
     fn push_outlives(
@@ -140,10 +183,6 @@ impl<'tcx> TypeRelatingDelegate<'tcx> for NllTypeRelatingDelegate<'_, '_, 'tcx>
         );
     }
 
-    fn normalization() -> NormalizationStrategy {
-        NormalizationStrategy::Eager
-    }
-
     fn forbid_inference_vars() -> bool {
         true
     }
diff --git a/compiler/rustc_borrowck/src/universal_regions.rs b/compiler/rustc_borrowck/src/universal_regions.rs
index 5b4d99682..15d7613a8 100644
--- a/compiler/rustc_borrowck/src/universal_regions.rs
+++ b/compiler/rustc_borrowck/src/universal_regions.rs
@@ -18,17 +18,20 @@ use rustc_errors::Diagnostic;
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::lang_items::LangItem;
-use rustc_hir::{BodyOwnerKind, HirId};
+use rustc_hir::BodyOwnerKind;
 use rustc_index::vec::{Idx, IndexVec};
-use rustc_infer::infer::{InferCtxt, NllRegionVariableOrigin};
+use rustc_infer::infer::NllRegionVariableOrigin;
 use rustc_middle::ty::fold::TypeFoldable;
 use rustc_middle::ty::{
     self, DefIdTree, InlineConstSubsts, InlineConstSubstsParts, RegionVid, Ty, TyCtxt,
 };
 use rustc_middle::ty::{InternalSubsts, SubstsRef};
+use rustc_span::Symbol;
 use std::iter;
 
 use crate::nll::ToRegionVid;
+use crate::renumber::{BoundRegionInfo, RegionCtxt};
+use crate::BorrowckInferCtxt;
 
 #[derive(Debug)]
 pub struct UniversalRegions<'tcx> {
@@ -162,11 +165,14 @@ struct UniversalRegionIndices<'tcx> {
     /// `ty::Region` to the internal `RegionVid` we are using. This is
     /// used because trait matching and type-checking will feed us
     /// region constraints that reference those regions and we need to
-    /// be able to map them our internal `RegionVid`. This is
+    /// be able to map them to our internal `RegionVid`. This is
     /// basically equivalent to an `InternalSubsts`, except that it also
     /// contains an entry for `ReStatic` -- it might be nice to just
     /// use a substs, and then handle `ReStatic` another way.
     indices: FxHashMap<ty::Region<'tcx>, RegionVid>,
+
+    /// The vid assigned to `'static`. Used only for diagnostics.
+    pub fr_static: RegionVid,
 }
 
 #[derive(Debug, PartialEq)]
@@ -221,13 +227,11 @@ impl<'tcx> UniversalRegions<'tcx> {
     /// signature. This will also compute the relationships that are
     /// known between those regions.
     pub fn new(
-        infcx: &InferCtxt<'tcx>,
+        infcx: &BorrowckInferCtxt<'_, 'tcx>,
         mir_def: ty::WithOptConstParam<LocalDefId>,
         param_env: ty::ParamEnv<'tcx>,
     ) -> Self {
-        let tcx = infcx.tcx;
-        let mir_hir_id = tcx.hir().local_def_id_to_hir_id(mir_def.did);
-        UniversalRegionsBuilder { infcx, mir_def, mir_hir_id, param_env }.build()
+        UniversalRegionsBuilder { infcx, mir_def, param_env }.build()
     }
 
     /// Given a reference to a closure type, extracts all the values
@@ -382,9 +386,8 @@ impl<'tcx> UniversalRegions<'tcx> {
 }
 
 struct UniversalRegionsBuilder<'cx, 'tcx> {
-    infcx: &'cx InferCtxt<'tcx>,
+    infcx: &'cx BorrowckInferCtxt<'cx, 'tcx>,
     mir_def: ty::WithOptConstParam<LocalDefId>,
-    mir_hir_id: HirId,
     param_env: ty::ParamEnv<'tcx>,
 }
 
@@ -400,7 +403,10 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
         assert_eq!(FIRST_GLOBAL_INDEX, self.infcx.num_region_vars());
 
         // Create the "global" region that is always free in all contexts: 'static.
-        let fr_static = self.infcx.next_nll_region_var(FR).to_region_vid();
+        let fr_static = self
+            .infcx
+            .next_nll_region_var(FR, || RegionCtxt::Free(Symbol::intern("static")))
+            .to_region_vid();
 
         // We've now added all the global regions. The next ones we
         // add will be external.
@@ -432,7 +438,17 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
                 |r| {
                     debug!(?r);
                     if !indices.indices.contains_key(&r) {
-                        let region_vid = self.infcx.next_nll_region_var(FR);
+                        let region_vid = {
+                            let name = match r.get_name() {
+                                Some(name) => name,
+                                _ => Symbol::intern("anon"),
+                            };
+
+                            self.infcx.next_nll_region_var(FR, || {
+                                RegionCtxt::LateBound(BoundRegionInfo::Name(name))
+                            })
+                        };
+
                         debug!(?region_vid);
                         indices.insert_late_bound_region(r, region_vid.to_region_vid());
                     }
@@ -460,7 +476,17 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
         for_each_late_bound_region_in_item(self.infcx.tcx, self.mir_def.did, |r| {
             debug!(?r);
             if !indices.indices.contains_key(&r) {
-                let region_vid = self.infcx.next_nll_region_var(FR);
+                let region_vid = {
+                    let name = match r.get_name() {
+                        Some(name) => name,
+                        _ => Symbol::intern("anon"),
+                    };
+
+                    self.infcx.next_nll_region_var(FR, || {
+                        RegionCtxt::LateBound(BoundRegionInfo::Name(name))
+                    })
+                };
+
                 debug!(?region_vid);
                 indices.insert_late_bound_region(r, region_vid.to_region_vid());
             }
@@ -472,28 +498,32 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
         // C-variadic fns also have a `VaList` input that's not listed in the signature
         // (as it's created inside the body itself, not passed in from outside).
         if let DefiningTy::FnDef(def_id, _) = defining_ty {
-            if self.infcx.tcx.fn_sig(def_id).c_variadic() {
+            if self.infcx.tcx.fn_sig(def_id).skip_binder().c_variadic() {
                 let va_list_did = self.infcx.tcx.require_lang_item(
                     LangItem::VaList,
                     Some(self.infcx.tcx.def_span(self.mir_def.did)),
                 );
-                let region = self
-                    .infcx
-                    .tcx
-                    .mk_region(ty::ReVar(self.infcx.next_nll_region_var(FR).to_region_vid()));
-                let va_list_ty = self
+
+                let reg_vid = self
                     .infcx
-                    .tcx
-                    .bound_type_of(va_list_did)
-                    .subst(self.infcx.tcx, &[region.into()]);
+                    .next_nll_region_var(FR, || RegionCtxt::Free(Symbol::intern("c-variadic")))
+                    .to_region_vid();
+
+                let region = self.infcx.tcx.mk_re_var(reg_vid);
+                let va_list_ty =
+                    self.infcx.tcx.type_of(va_list_did).subst(self.infcx.tcx, &[region.into()]);
 
-                unnormalized_input_tys = self.infcx.tcx.mk_type_list(
+                unnormalized_input_tys = self.infcx.tcx.mk_type_list_from_iter(
                     unnormalized_input_tys.iter().copied().chain(iter::once(va_list_ty)),
                 );
             }
         }
 
-        let fr_fn_body = self.infcx.next_nll_region_var(FR).to_region_vid();
+        let fr_fn_body = self
+            .infcx
+            .next_nll_region_var(FR, || RegionCtxt::Free(Symbol::intern("fn_body")))
+            .to_region_vid();
+
         let num_universals = self.infcx.num_region_vars();
 
         debug!("build: global regions = {}..{}", FIRST_GLOBAL_INDEX, first_extern_index);
@@ -527,12 +557,7 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
 
         match tcx.hir().body_owner_kind(self.mir_def.did) {
             BodyOwnerKind::Closure | BodyOwnerKind::Fn => {
-                let defining_ty = if self.mir_def.did.to_def_id() == typeck_root_def_id {
-                    tcx.type_of(typeck_root_def_id)
-                } else {
-                    let tables = tcx.typeck(self.mir_def.did);
-                    tables.node_type(self.mir_hir_id)
-                };
+                let defining_ty = tcx.type_of(self.mir_def.def_id_for_type_of()).subst_identity();
 
                 debug!("defining_ty (pre-replacement): {:?}", defining_ty);
 
@@ -561,7 +586,18 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
                         self.infcx.replace_free_regions_with_nll_infer_vars(FR, identity_substs);
                     DefiningTy::Const(self.mir_def.did.to_def_id(), substs)
                 } else {
-                    let ty = tcx.typeck(self.mir_def.did).node_type(self.mir_hir_id);
+                    // FIXME this line creates a dependency between borrowck and typeck.
+                    //
+                    // This is required for `AscribeUserType` canonical query, which will call
+                    // `type_of(inline_const_def_id)`. That `type_of` would inject erased lifetimes
+                    // into borrowck, which is ICE #78174.
+                    //
+                    // As a workaround, inline consts have an additional generic param (`ty`
+                    // below), so that `type_of(inline_const_def_id).substs(substs)` uses the
+                    // proper type with NLL infer vars.
+                    let ty = tcx
+                        .typeck(self.mir_def.did)
+                        .node_type(tcx.local_def_id_to_hir_id(self.mir_def.did));
                     let substs = InlineConstSubsts::new(
                         tcx,
                         InlineConstSubstsParts { parent_substs: identity_substs, ty },
@@ -609,7 +645,7 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
         let subst_mapping =
             iter::zip(identity_substs.regions(), fr_substs.regions().map(|r| r.to_region_vid()));
 
-        UniversalRegionIndices { indices: global_mapping.chain(subst_mapping).collect() }
+        UniversalRegionIndices { indices: global_mapping.chain(subst_mapping).collect(), fr_static }
     }
 
     fn compute_inputs_and_output(
@@ -623,7 +659,7 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
                 assert_eq!(self.mir_def.did.to_def_id(), def_id);
                 let closure_sig = substs.as_closure().sig();
                 let inputs_and_output = closure_sig.inputs_and_output();
-                let bound_vars = tcx.mk_bound_variable_kinds(
+                let bound_vars = tcx.mk_bound_variable_kinds_from_iter(
                     inputs_and_output
                         .bound_vars()
                         .iter()
@@ -633,7 +669,7 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
                     var: ty::BoundVar::from_usize(bound_vars.len() - 1),
                     kind: ty::BrEnv,
                 };
-                let env_region = ty::ReLateBound(ty::INNERMOST, br);
+                let env_region = tcx.mk_re_late_bound(ty::INNERMOST, br);
                 let closure_ty = tcx.closure_env_ty(def_id, substs, env_region).unwrap();
 
                 // The "inputs" of the closure in the
@@ -647,7 +683,7 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
                 };
 
                 ty::Binder::bind_with_vars(
-                    tcx.mk_type_list(
+                    tcx.mk_type_list_from_iter(
                         iter::once(closure_ty).chain(inputs).chain(iter::once(output)),
                     ),
                     bound_vars,
@@ -660,12 +696,12 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
                 let output = substs.as_generator().return_ty();
                 let generator_ty = tcx.mk_generator(def_id, substs, movability);
                 let inputs_and_output =
-                    self.infcx.tcx.intern_type_list(&[generator_ty, resume_ty, output]);
+                    self.infcx.tcx.mk_type_list(&[generator_ty, resume_ty, output]);
                 ty::Binder::dummy(inputs_and_output)
             }
 
             DefiningTy::FnDef(def_id, _) => {
-                let sig = tcx.fn_sig(def_id);
+                let sig = tcx.fn_sig(def_id).subst_identity();
                 let sig = indices.fold_to_region_vids(tcx, sig);
                 sig.inputs_and_output()
             }
@@ -674,15 +710,15 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
                 // For a constant body, there are no inputs, and one
                 // "output" (the type of the constant).
                 assert_eq!(self.mir_def.did.to_def_id(), def_id);
-                let ty = tcx.type_of(self.mir_def.def_id_for_type_of());
+                let ty = tcx.type_of(self.mir_def.def_id_for_type_of()).subst_identity();
                 let ty = indices.fold_to_region_vids(tcx, ty);
-                ty::Binder::dummy(tcx.intern_type_list(&[ty]))
+                ty::Binder::dummy(tcx.mk_type_list(&[ty]))
             }
 
             DefiningTy::InlineConst(def_id, substs) => {
                 assert_eq!(self.mir_def.did.to_def_id(), def_id);
                 let ty = substs.as_inline_const().ty();
-                ty::Binder::dummy(tcx.intern_type_list(&[ty]))
+                ty::Binder::dummy(tcx.mk_type_list(&[ty]))
             }
         }
     }
@@ -695,7 +731,7 @@ trait InferCtxtExt<'tcx> {
         value: T,
     ) -> T
     where
-        T: TypeFoldable<'tcx>;
+        T: TypeFoldable<TyCtxt<'tcx>>;
 
     fn replace_bound_regions_with_nll_infer_vars<T>(
         &self,
@@ -705,7 +741,7 @@ trait InferCtxtExt<'tcx> {
         indices: &mut UniversalRegionIndices<'tcx>,
     ) -> T
     where
-        T: TypeFoldable<'tcx>;
+        T: TypeFoldable<TyCtxt<'tcx>>;
 
     fn replace_late_bound_regions_with_nll_infer_vars_in_recursive_scope(
         &self,
@@ -720,16 +756,27 @@ trait InferCtxtExt<'tcx> {
     );
 }
 
-impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
+impl<'cx, 'tcx> InferCtxtExt<'tcx> for BorrowckInferCtxt<'cx, 'tcx> {
+    #[instrument(skip(self), level = "debug")]
     fn replace_free_regions_with_nll_infer_vars<T>(
         &self,
         origin: NllRegionVariableOrigin,
         value: T,
     ) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
-        self.tcx.fold_regions(value, |_region, _depth| self.next_nll_region_var(origin))
+        self.infcx.tcx.fold_regions(value, |region, _depth| {
+            let name = match region.get_name() {
+                Some(name) => name,
+                _ => Symbol::intern("anon"),
+            };
+            debug!(?region, ?name);
+
+            let reg_var = self.next_nll_region_var(origin, || RegionCtxt::Free(name));
+
+            reg_var
+        })
     }
 
     #[instrument(level = "debug", skip(self, indices))]
@@ -741,15 +788,20 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
         indices: &mut UniversalRegionIndices<'tcx>,
     ) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         let (value, _map) = self.tcx.replace_late_bound_regions(value, |br| {
             debug!(?br);
-            let liberated_region = self.tcx.mk_region(ty::ReFree(ty::FreeRegion {
-                scope: all_outlive_scope.to_def_id(),
-                bound_region: br.kind,
-            }));
-            let region_vid = self.next_nll_region_var(origin);
+            let liberated_region = self.tcx.mk_re_free(all_outlive_scope.to_def_id(), br.kind);
+            let region_vid = {
+                let name = match br.kind.get_name() {
+                    Some(name) => name,
+                    _ => Symbol::intern("anon"),
+                };
+
+                self.next_nll_region_var(origin, || RegionCtxt::Bound(BoundRegionInfo::Name(name)))
+            };
+
             indices.insert_late_bound_region(liberated_region, region_vid.to_region_vid());
             debug!(?liberated_region, ?region_vid);
             region_vid
@@ -775,7 +827,17 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
         for_each_late_bound_region_in_recursive_scope(self.tcx, mir_def_id, |r| {
             debug!(?r);
             if !indices.indices.contains_key(&r) {
-                let region_vid = self.next_nll_region_var(FR);
+                let region_vid = {
+                    let name = match r.get_name() {
+                        Some(name) => name,
+                        _ => Symbol::intern("anon"),
+                    };
+
+                    self.next_nll_region_var(FR, || {
+                        RegionCtxt::LateBound(BoundRegionInfo::Name(name))
+                    })
+                };
+
                 debug!(?region_vid);
                 indices.insert_late_bound_region(r, region_vid.to_region_vid());
             }
@@ -791,8 +853,17 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
         for_each_late_bound_region_in_item(self.tcx, mir_def_id, |r| {
             debug!(?r);
             if !indices.indices.contains_key(&r) {
-                let region_vid = self.next_nll_region_var(FR);
-                debug!(?region_vid);
+                let region_vid = {
+                    let name = match r.get_name() {
+                        Some(name) => name,
+                        _ => Symbol::intern("anon"),
+                    };
+
+                    self.next_nll_region_var(FR, || {
+                        RegionCtxt::LateBound(BoundRegionInfo::Name(name))
+                    })
+                };
+
                 indices.insert_late_bound_region(r, region_vid.to_region_vid());
             }
         });
@@ -821,6 +892,11 @@ impl<'tcx> UniversalRegionIndices<'tcx> {
     pub fn to_region_vid(&self, r: ty::Region<'tcx>) -> RegionVid {
         if let ty::ReVar(..) = *r {
             r.to_region_vid()
+        } else if r.is_error() {
+            // We use the `'static` `RegionVid` because `ReError` doesn't actually exist in the
+            // `UniversalRegionIndices`. This is fine because 1) it is a fallback only used if
+            // errors are being emitted and 2) it leaves the happy path unaffected.
+            self.fr_static
         } else {
             *self
                 .indices
@@ -833,9 +909,9 @@ impl<'tcx> UniversalRegionIndices<'tcx> {
     /// returned by `to_region_vid`.
     pub fn fold_to_region_vids<T>(&self, tcx: TyCtxt<'tcx>, value: T) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
-        tcx.fold_regions(value, |region, _| tcx.mk_region(ty::ReVar(self.to_region_vid(region))))
+        tcx.fold_regions(value, |region, _| tcx.mk_re_var(self.to_region_vid(region)))
     }
 }
 
@@ -875,8 +951,7 @@ fn for_each_late_bound_region_in_item<'tcx>(
 
     for bound_var in tcx.late_bound_vars(tcx.hir().local_def_id_to_hir_id(mir_def_id)) {
         let ty::BoundVariableKind::Region(bound_region) = bound_var else { continue; };
-        let liberated_region = tcx
-            .mk_region(ty::ReFree(ty::FreeRegion { scope: mir_def_id.to_def_id(), bound_region }));
+        let liberated_region = tcx.mk_re_free(mir_def_id.to_def_id(), bound_region);
         f(liberated_region);
     }
 }
diff --git a/compiler/rustc_builtin_macros/Cargo.toml b/compiler/rustc_builtin_macros/Cargo.toml
index 467fa932a..336e14ef9 100644
--- a/compiler/rustc_builtin_macros/Cargo.toml
+++ b/compiler/rustc_builtin_macros/Cargo.toml
@@ -23,5 +23,5 @@ rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-thin-vec = "0.2.9"
+thin-vec = "0.2.12"
 tracing = "0.1"
diff --git a/compiler/rustc_error_messages/locales/en-US/builtin_macros.ftl b/compiler/rustc_builtin_macros/locales/en-US.ftl
index 4d088e27b..4d088e27b 100644
--- a/compiler/rustc_error_messages/locales/en-US/builtin_macros.ftl
+++ b/compiler/rustc_builtin_macros/locales/en-US.ftl
diff --git a/compiler/rustc_builtin_macros/src/alloc_error_handler.rs b/compiler/rustc_builtin_macros/src/alloc_error_handler.rs
index dcf500ddb..ac6697232 100644
--- a/compiler/rustc_builtin_macros/src/alloc_error_handler.rs
+++ b/compiler/rustc_builtin_macros/src/alloc_error_handler.rs
@@ -6,7 +6,7 @@ use rustc_ast::{Fn, ItemKind, Stmt, TyKind, Unsafe};
 use rustc_expand::base::{Annotatable, ExtCtxt};
 use rustc_span::symbol::{kw, sym, Ident};
 use rustc_span::Span;
-use thin_vec::thin_vec;
+use thin_vec::{thin_vec, ThinVec};
 
 pub fn expand(
     ecx: &mut ExtCtxt<'_>,
@@ -39,10 +39,10 @@ pub fn expand(
     let span = ecx.with_def_site_ctxt(item.span);
 
     // Generate item statements for the allocator methods.
-    let stmts = vec![generate_handler(ecx, item.ident, span, sig_span)];
+    let stmts = thin_vec![generate_handler(ecx, item.ident, span, sig_span)];
 
     // Generate anonymous constant serving as container for the allocator methods.
-    let const_ty = ecx.ty(sig_span, TyKind::Tup(Vec::new()));
+    let const_ty = ecx.ty(sig_span, TyKind::Tup(ThinVec::new()));
     let const_body = ecx.expr_block(ecx.block(span, stmts));
     let const_item = ecx.item_const(span, Ident::new(kw::Underscore, span), const_ty, const_body);
     let const_item = if is_stmt {
@@ -67,13 +67,16 @@ fn generate_handler(cx: &ExtCtxt<'_>, handler: Ident, span: Span, sig_span: Span
 
     let layout_new = cx.std_path(&[sym::alloc, sym::Layout, sym::from_size_align_unchecked]);
     let layout_new = cx.expr_path(cx.path(span, layout_new));
-    let layout =
-        cx.expr_call(span, layout_new, vec![cx.expr_ident(span, size), cx.expr_ident(span, align)]);
+    let layout = cx.expr_call(
+        span,
+        layout_new,
+        thin_vec![cx.expr_ident(span, size), cx.expr_ident(span, align)],
+    );
 
-    let call = cx.expr_call_ident(sig_span, handler, vec![layout]);
+    let call = cx.expr_call_ident(sig_span, handler, thin_vec![layout]);
 
     let never = ast::FnRetTy::Ty(cx.ty(span, TyKind::Never));
-    let params = vec![cx.param(span, size, ty_usize.clone()), cx.param(span, align, ty_usize)];
+    let params = thin_vec![cx.param(span, size, ty_usize.clone()), cx.param(span, align, ty_usize)];
     let decl = cx.fn_decl(params, never);
     let header = FnHeader { unsafety: Unsafe::Yes(span), ..FnHeader::default() };
     let sig = FnSig { decl, header, span: span };
diff --git a/compiler/rustc_builtin_macros/src/asm.rs b/compiler/rustc_builtin_macros/src/asm.rs
index 925392b50..3fdbc9715 100644
--- a/compiler/rustc_builtin_macros/src/asm.rs
+++ b/compiler/rustc_builtin_macros/src/asm.rs
@@ -152,7 +152,7 @@ pub fn parse_asm_args<'a>(
                 ast::InlineAsmOperand::InOut { reg, expr, late: true }
             }
         } else if p.eat_keyword(kw::Const) {
-            let anon_const = p.parse_anon_const_expr()?;
+            let anon_const = p.parse_expr_anon_const()?;
             ast::InlineAsmOperand::Const { anon_const }
         } else if p.eat_keyword(sym::sym) {
             let expr = p.parse_expr()?;
diff --git a/compiler/rustc_builtin_macros/src/assert.rs b/compiler/rustc_builtin_macros/src/assert.rs
index 8555c3593..75af5e2b1 100644
--- a/compiler/rustc_builtin_macros/src/assert.rs
+++ b/compiler/rustc_builtin_macros/src/assert.rs
@@ -11,6 +11,7 @@ use rustc_expand::base::{DummyResult, ExtCtxt, MacEager, MacResult};
 use rustc_parse::parser::Parser;
 use rustc_span::symbol::{sym, Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
+use thin_vec::thin_vec;
 
 pub fn expand_assert<'cx>(
     cx: &'cx mut ExtCtxt<'_>,
@@ -79,7 +80,7 @@ pub fn expand_assert<'cx>(
         let then = cx.expr_call_global(
             call_site_span,
             cx.std_path(&[sym::panicking, sym::panic]),
-            vec![cx.expr_str(
+            thin_vec![cx.expr_str(
                 DUMMY_SP,
                 Symbol::intern(&format!(
                     "assertion failed: {}",
diff --git a/compiler/rustc_builtin_macros/src/assert/context.rs b/compiler/rustc_builtin_macros/src/assert/context.rs
index 93b07801e..b0b4dda16 100644
--- a/compiler/rustc_builtin_macros/src/assert/context.rs
+++ b/compiler/rustc_builtin_macros/src/assert/context.rs
@@ -12,7 +12,7 @@ use rustc_span::{
     symbol::{sym, Ident, Symbol},
     Span,
 };
-use thin_vec::thin_vec;
+use thin_vec::{thin_vec, ThinVec};
 
 pub(super) struct Context<'cx, 'a> {
     // An optimization.
@@ -83,12 +83,12 @@ impl<'cx, 'a> Context<'cx, 'a> {
 
         let Self { best_case_captures, capture_decls, cx, local_bind_decls, span, .. } = self;
 
-        let mut assert_then_stmts = Vec::with_capacity(2);
+        let mut assert_then_stmts = ThinVec::with_capacity(2);
         assert_then_stmts.extend(best_case_captures);
         assert_then_stmts.push(self.cx.stmt_expr(panic));
         let assert_then = self.cx.block(span, assert_then_stmts);
 
-        let mut stmts = Vec::with_capacity(4);
+        let mut stmts = ThinVec::with_capacity(4);
         stmts.push(initial_imports);
         stmts.extend(capture_decls.into_iter().map(|c| c.decl));
         stmts.extend(local_bind_decls);
@@ -120,7 +120,7 @@ impl<'cx, 'a> Context<'cx, 'a> {
                 thin_vec![self.cx.attr_nested_word(sym::allow, sym::unused_imports, self.span)],
                 ItemKind::Use(UseTree {
                     prefix: self.cx.path(self.span, self.cx.std_path(&[sym::asserting])),
-                    kind: UseTreeKind::Nested(vec![
+                    kind: UseTreeKind::Nested(thin_vec![
                         nested_tree(self, sym::TryCaptureGeneric),
                         nested_tree(self, sym::TryCapturePrintable),
                     ]),
@@ -136,7 +136,7 @@ impl<'cx, 'a> Context<'cx, 'a> {
         self.cx.expr_call(
             self.span,
             self.cx.expr_path(self.cx.path(self.span, unlikely_path)),
-            vec![self.cx.expr(self.span, ExprKind::Unary(UnOp::Not, cond_expr))],
+            thin_vec![self.cx.expr(self.span, ExprKind::Unary(UnOp::Not, cond_expr))],
         )
     }
 
@@ -297,6 +297,7 @@ impl<'cx, 'a> Context<'cx, 'a> {
             | ExprKind::Continue(_)
             | ExprKind::Err
             | ExprKind::Field(_, _)
+            | ExprKind::FormatArgs(_)
             | ExprKind::ForLoop(_, _, _, _)
             | ExprKind::If(_, _, _)
             | ExprKind::IncludedBytes(..)
@@ -338,7 +339,7 @@ impl<'cx, 'a> Context<'cx, 'a> {
         let init = self.cx.expr_call(
             self.span,
             self.cx.expr_path(self.cx.path(self.span, init_std_path)),
-            vec![],
+            ThinVec::new(),
         );
         let capture = Capture { decl: self.cx.stmt_let(self.span, true, ident, init), ident };
         self.capture_decls.push(capture);
@@ -365,7 +366,7 @@ impl<'cx, 'a> Context<'cx, 'a> {
             self.cx.expr_path(
                 self.cx.path(self.span, self.cx.std_path(&[sym::asserting, sym::Wrapper])),
             ),
-            vec![self.cx.expr_path(Path::from_ident(local_bind))],
+            thin_vec![self.cx.expr_path(Path::from_ident(local_bind))],
         );
         let try_capture_call = self
             .cx
@@ -377,7 +378,7 @@ impl<'cx, 'a> Context<'cx, 'a> {
                     ident: Ident::new(sym::try_capture, self.span),
                 },
                 expr_paren(self.cx, self.span, self.cx.expr_addr_of(self.span, wrapper)),
-                vec![expr_addr_of_mut(
+                thin_vec![expr_addr_of_mut(
                     self.cx,
                     self.span,
                     self.cx.expr_path(Path::from_ident(capture)),
@@ -388,7 +389,7 @@ impl<'cx, 'a> Context<'cx, 'a> {
         let local_bind_path = self.cx.expr_path(Path::from_ident(local_bind));
         let rslt = if self.is_consumed {
             let ret = self.cx.stmt_expr(local_bind_path);
-            self.cx.expr_block(self.cx.block(self.span, vec![try_capture_call, ret]))
+            self.cx.expr_block(self.cx.block(self.span, thin_vec![try_capture_call, ret]))
         } else {
             self.best_case_captures.push(try_capture_call);
             local_bind_path
@@ -440,7 +441,7 @@ fn expr_method_call(
     cx: &ExtCtxt<'_>,
     seg: PathSegment,
     receiver: P<Expr>,
-    args: Vec<P<Expr>>,
+    args: ThinVec<P<Expr>>,
     span: Span,
 ) -> P<Expr> {
     cx.expr(span, ExprKind::MethodCall(Box::new(MethodCall { seg, receiver, args, span })))
diff --git a/compiler/rustc_builtin_macros/src/deriving/bounds.rs b/compiler/rustc_builtin_macros/src/deriving/bounds.rs
index 240167146..0481a1189 100644
--- a/compiler/rustc_builtin_macros/src/deriving/bounds.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/bounds.rs
@@ -17,6 +17,7 @@ pub fn expand_deriving_copy(
         span,
         path: path_std!(marker::Copy),
         skip_path_as_bound: false,
+        needs_copy_as_bound_if_packed: false,
         additional_bounds: Vec::new(),
         supports_unions: true,
         methods: Vec::new(),
diff --git a/compiler/rustc_builtin_macros/src/deriving/clone.rs b/compiler/rustc_builtin_macros/src/deriving/clone.rs
index ef5a75f42..dfee2d3ce 100644
--- a/compiler/rustc_builtin_macros/src/deriving/clone.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/clone.rs
@@ -6,7 +6,7 @@ use rustc_data_structures::fx::FxHashSet;
 use rustc_expand::base::{Annotatable, ExtCtxt};
 use rustc_span::symbol::{kw, sym, Ident};
 use rustc_span::Span;
-use thin_vec::thin_vec;
+use thin_vec::{thin_vec, ThinVec};
 
 pub fn expand_deriving_clone(
     cx: &mut ExtCtxt<'_>,
@@ -73,6 +73,7 @@ pub fn expand_deriving_clone(
         span,
         path: path_std!(clone::Clone),
         skip_path_as_bound: false,
+        needs_copy_as_bound_if_packed: true,
         additional_bounds: bounds,
         supports_unions: true,
         methods: vec![MethodDef {
@@ -99,7 +100,7 @@ fn cs_clone_simple(
     substr: &Substructure<'_>,
     is_union: bool,
 ) -> BlockOrExpr {
-    let mut stmts = Vec::new();
+    let mut stmts = ThinVec::new();
     let mut seen_type_names = FxHashSet::default();
     let mut process_variant = |variant: &VariantData| {
         for field in variant.fields() {
@@ -161,7 +162,7 @@ fn cs_clone(
     let all_fields;
     let fn_path = cx.std_path(&[sym::clone, sym::Clone, sym::clone]);
     let subcall = |cx: &mut ExtCtxt<'_>, field: &FieldInfo| {
-        let args = vec![field.self_expr.clone()];
+        let args = thin_vec![field.self_expr.clone()];
         cx.expr_call_global(field.span, fn_path.clone(), args)
     };
 
@@ -199,7 +200,7 @@ fn cs_clone(
                     let call = subcall(cx, field);
                     cx.field_imm(field.span, ident, call)
                 })
-                .collect::<Vec<_>>();
+                .collect::<ThinVec<_>>();
 
             cx.expr_struct(trait_span, ctor_path, fields)
         }
diff --git a/compiler/rustc_builtin_macros/src/deriving/cmp/eq.rs b/compiler/rustc_builtin_macros/src/deriving/cmp/eq.rs
index 3e994f037..af9719586 100644
--- a/compiler/rustc_builtin_macros/src/deriving/cmp/eq.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/cmp/eq.rs
@@ -7,7 +7,7 @@ use rustc_data_structures::fx::FxHashSet;
 use rustc_expand::base::{Annotatable, ExtCtxt};
 use rustc_span::symbol::sym;
 use rustc_span::Span;
-use thin_vec::thin_vec;
+use thin_vec::{thin_vec, ThinVec};
 
 pub fn expand_deriving_eq(
     cx: &mut ExtCtxt<'_>,
@@ -27,6 +27,7 @@ pub fn expand_deriving_eq(
         span,
         path: path_std!(cmp::Eq),
         skip_path_as_bound: false,
+        needs_copy_as_bound_if_packed: true,
         additional_bounds: Vec::new(),
         supports_unions: true,
         methods: vec![MethodDef {
@@ -55,7 +56,7 @@ fn cs_total_eq_assert(
     trait_span: Span,
     substr: &Substructure<'_>,
 ) -> BlockOrExpr {
-    let mut stmts = Vec::new();
+    let mut stmts = ThinVec::new();
     let mut seen_type_names = FxHashSet::default();
     let mut process_variant = |variant: &ast::VariantData| {
         for field in variant.fields() {
diff --git a/compiler/rustc_builtin_macros/src/deriving/cmp/ord.rs b/compiler/rustc_builtin_macros/src/deriving/cmp/ord.rs
index a926fca4e..cfd36f030 100644
--- a/compiler/rustc_builtin_macros/src/deriving/cmp/ord.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/cmp/ord.rs
@@ -20,6 +20,7 @@ pub fn expand_deriving_ord(
         span,
         path: path_std!(cmp::Ord),
         skip_path_as_bound: false,
+        needs_copy_as_bound_if_packed: true,
         additional_bounds: Vec::new(),
         supports_unions: false,
         methods: vec![MethodDef {
@@ -63,14 +64,14 @@ pub fn cs_cmp(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>) -> Bl
                 let [other_expr] = &field.other_selflike_exprs[..] else {
                         cx.span_bug(field.span, "not exactly 2 arguments in `derive(Ord)`");
                     };
-                let args = vec![field.self_expr.clone(), other_expr.clone()];
+                let args = thin_vec![field.self_expr.clone(), other_expr.clone()];
                 cx.expr_call_global(field.span, cmp_path.clone(), args)
             }
             CsFold::Combine(span, expr1, expr2) => {
                 let eq_arm = cx.arm(span, cx.pat_path(span, equal_path.clone()), expr1);
                 let neq_arm =
                     cx.arm(span, cx.pat_ident(span, test_id), cx.expr_ident(span, test_id));
-                cx.expr_match(span, expr2, vec![eq_arm, neq_arm])
+                cx.expr_match(span, expr2, thin_vec![eq_arm, neq_arm])
             }
             CsFold::Fieldless => cx.expr_path(equal_path.clone()),
         },
diff --git a/compiler/rustc_builtin_macros/src/deriving/cmp/partial_eq.rs b/compiler/rustc_builtin_macros/src/deriving/cmp/partial_eq.rs
index 9051fe0b2..bad47db0d 100644
--- a/compiler/rustc_builtin_macros/src/deriving/cmp/partial_eq.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/cmp/partial_eq.rs
@@ -29,16 +29,30 @@ pub fn expand_deriving_partial_eq(
                         cx.span_bug(field.span, "not exactly 2 arguments in `derive(PartialEq)`");
                     };
 
-                    // We received `&T` arguments. Convert them to `T` by
-                    // stripping `&` or adding `*`. This isn't necessary for
-                    // type checking, but it results in much better error
-                    // messages if something goes wrong.
+                    // We received arguments of type `&T`. Convert them to type `T` by stripping
+                    // any leading `&` or adding `*`. This isn't necessary for type checking, but
+                    // it results in better error messages if something goes wrong.
+                    //
+                    // Note: for arguments that look like `&{ x }`, which occur with packed
+                    // structs, this would cause expressions like `{ self.x } == { other.x }`,
+                    // which isn't valid Rust syntax. This wouldn't break compilation because these
+                    // AST nodes are constructed within the compiler. But it would mean that code
+                    // printed by `-Zunpretty=expanded` (or `cargo expand`) would have invalid
+                    // syntax, which would be suboptimal. So we wrap these in parens, giving
+                    // `({ self.x }) == ({ other.x })`, which is valid syntax.
                     let convert = |expr: &P<Expr>| {
                         if let ExprKind::AddrOf(BorrowKind::Ref, Mutability::Not, inner) =
                             &expr.kind
                         {
-                            inner.clone()
+                            if let ExprKind::Block(..) = &inner.kind {
+                                // `&{ x }` form: remove the `&`, add parens.
+                                cx.expr_paren(field.span, inner.clone())
+                            } else {
+                                // `&x` form: remove the `&`.
+                                inner.clone()
+                            }
                         } else {
+                            // No leading `&`: add a leading `*`.
                             cx.expr_deref(field.span, expr.clone())
                         }
                     };
@@ -84,6 +98,7 @@ pub fn expand_deriving_partial_eq(
         span,
         path: path_std!(cmp::PartialEq),
         skip_path_as_bound: false,
+        needs_copy_as_bound_if_packed: true,
         additional_bounds: Vec::new(),
         supports_unions: false,
         methods,
diff --git a/compiler/rustc_builtin_macros/src/deriving/cmp/partial_ord.rs b/compiler/rustc_builtin_macros/src/deriving/cmp/partial_ord.rs
index c9dc89212..9f4624790 100644
--- a/compiler/rustc_builtin_macros/src/deriving/cmp/partial_ord.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/cmp/partial_ord.rs
@@ -1,7 +1,7 @@
 use crate::deriving::generic::ty::*;
 use crate::deriving::generic::*;
 use crate::deriving::{path_std, pathvec_std};
-use rustc_ast::MetaItem;
+use rustc_ast::{ExprKind, ItemKind, MetaItem, PatKind};
 use rustc_expand::base::{Annotatable, ExtCtxt};
 use rustc_span::symbol::{sym, Ident};
 use rustc_span::Span;
@@ -21,6 +21,27 @@ pub fn expand_deriving_partial_ord(
 
     let attrs = thin_vec![cx.attr_word(sym::inline, span)];
 
+    // Order in which to perform matching
+    let tag_then_data = if let Annotatable::Item(item) = item
+        && let ItemKind::Enum(def, _) = &item.kind {
+            let dataful: Vec<bool> = def.variants.iter().map(|v| !v.data.fields().is_empty()).collect();
+            match dataful.iter().filter(|&&b| b).count() {
+                // No data, placing the tag check first makes codegen simpler
+                0 => true,
+                1..=2 => false,
+                _ => {
+                    (0..dataful.len()-1).any(|i| {
+                        if dataful[i] && let Some(idx) = dataful[i+1..].iter().position(|v| *v) {
+                            idx >= 2
+                        } else {
+                            false
+                        }
+                    })
+                }
+            }
+        } else {
+            true
+        };
     let partial_cmp_def = MethodDef {
         name: sym::partial_cmp,
         generics: Bounds::empty(),
@@ -30,7 +51,7 @@ pub fn expand_deriving_partial_ord(
         attributes: attrs,
         fieldless_variants_strategy: FieldlessVariantsStrategy::Unify,
         combine_substructure: combine_substructure(Box::new(|cx, span, substr| {
-            cs_partial_cmp(cx, span, substr)
+            cs_partial_cmp(cx, span, substr, tag_then_data)
         })),
     };
 
@@ -38,6 +59,7 @@ pub fn expand_deriving_partial_ord(
         span,
         path: path_std!(cmp::PartialOrd),
         skip_path_as_bound: false,
+        needs_copy_as_bound_if_packed: true,
         additional_bounds: vec![],
         supports_unions: false,
         methods: vec![partial_cmp_def],
@@ -47,7 +69,12 @@ pub fn expand_deriving_partial_ord(
     trait_def.expand(cx, mitem, item, push)
 }
 
-pub fn cs_partial_cmp(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>) -> BlockOrExpr {
+fn cs_partial_cmp(
+    cx: &mut ExtCtxt<'_>,
+    span: Span,
+    substr: &Substructure<'_>,
+    tag_then_data: bool,
+) -> BlockOrExpr {
     let test_id = Ident::new(sym::cmp, span);
     let equal_path = cx.path_global(span, cx.std_path(&[sym::cmp, sym::Ordering, sym::Equal]));
     let partial_cmp_path = cx.std_path(&[sym::cmp, sym::PartialOrd, sym::partial_cmp]);
@@ -71,15 +98,53 @@ pub fn cs_partial_cmp(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_
                 let [other_expr] = &field.other_selflike_exprs[..] else {
                         cx.span_bug(field.span, "not exactly 2 arguments in `derive(Ord)`");
                     };
-                let args = vec![field.self_expr.clone(), other_expr.clone()];
+                let args = thin_vec![field.self_expr.clone(), other_expr.clone()];
                 cx.expr_call_global(field.span, partial_cmp_path.clone(), args)
             }
-            CsFold::Combine(span, expr1, expr2) => {
-                let eq_arm =
-                    cx.arm(span, cx.pat_some(span, cx.pat_path(span, equal_path.clone())), expr1);
-                let neq_arm =
-                    cx.arm(span, cx.pat_ident(span, test_id), cx.expr_ident(span, test_id));
-                cx.expr_match(span, expr2, vec![eq_arm, neq_arm])
+            CsFold::Combine(span, mut expr1, expr2) => {
+                // When the item is an enum, this expands to
+                // ```
+                // match (expr2) {
+                //     Some(Ordering::Equal) => expr1,
+                //     cmp => cmp
+                // }
+                // ```
+                // where `expr2` is `partial_cmp(self_tag, other_tag)`, and `expr1` is a `match`
+                //  against the enum variants. This means that we begin by comparing the enum tags,
+                // before either inspecting their contents (if they match), or returning
+                // the `cmp::Ordering` of comparing the enum tags.
+                // ```
+                // match partial_cmp(self_tag, other_tag) {
+                //     Some(Ordering::Equal) => match (self, other)  {
+                //         (Self::A(self_0), Self::A(other_0)) => partial_cmp(self_0, other_0),
+                //         (Self::B(self_0), Self::B(other_0)) => partial_cmp(self_0, other_0),
+                //         _ => Some(Ordering::Equal)
+                //     }
+                //     cmp => cmp
+                // }
+                // ```
+                // If we have any certain enum layouts, flipping this results in better codegen
+                // ```
+                // match (self, other) {
+                //     (Self::A(self_0), Self::A(other_0)) => partial_cmp(self_0, other_0),
+                //     _ => partial_cmp(self_tag, other_tag)
+                // }
+                // ```
+                // Reference: https://github.com/rust-lang/rust/pull/103659#issuecomment-1328126354
+
+                if !tag_then_data
+                    && let ExprKind::Match(_, arms) = &mut expr1.kind
+                    && let Some(last) = arms.last_mut()
+                    && let PatKind::Wild = last.pat.kind {
+                        last.body = expr2;
+                        expr1
+                } else {
+                    let eq_arm =
+                        cx.arm(span, cx.pat_some(span, cx.pat_path(span, equal_path.clone())), expr1);
+                    let neq_arm =
+                        cx.arm(span, cx.pat_ident(span, test_id), cx.expr_ident(span, test_id));
+                    cx.expr_match(span, expr2, thin_vec![eq_arm, neq_arm])
+                }
             }
             CsFold::Fieldless => cx.expr_some(span, cx.expr_path(equal_path.clone())),
         },
diff --git a/compiler/rustc_builtin_macros/src/deriving/debug.rs b/compiler/rustc_builtin_macros/src/deriving/debug.rs
index e0f487e86..809f9838d 100644
--- a/compiler/rustc_builtin_macros/src/deriving/debug.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/debug.rs
@@ -7,6 +7,7 @@ use rustc_ast::{self as ast, MetaItem};
 use rustc_expand::base::{Annotatable, ExtCtxt};
 use rustc_span::symbol::{sym, Ident, Symbol};
 use rustc_span::Span;
+use thin_vec::{thin_vec, ThinVec};
 
 pub fn expand_deriving_debug(
     cx: &mut ExtCtxt<'_>,
@@ -23,6 +24,7 @@ pub fn expand_deriving_debug(
         span,
         path: path_std!(fmt::Debug),
         skip_path_as_bound: false,
+        needs_copy_as_bound_if_packed: true,
         additional_bounds: Vec::new(),
         supports_unions: false,
         methods: vec![MethodDef {
@@ -75,10 +77,25 @@ fn show_substructure(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>
     // The number of fields that can be handled without an array.
     const CUTOFF: usize = 5;
 
+    fn expr_for_field(
+        cx: &ExtCtxt<'_>,
+        field: &FieldInfo,
+        index: usize,
+        len: usize,
+    ) -> ast::ptr::P<ast::Expr> {
+        if index < len - 1 {
+            field.self_expr.clone()
+        } else {
+            // Unsized types need an extra indirection, but only the last field
+            // may be unsized.
+            cx.expr_addr_of(field.span, field.self_expr.clone())
+        }
+    }
+
     if fields.is_empty() {
         // Special case for no fields.
         let fn_path_write_str = cx.std_path(&[sym::fmt, sym::Formatter, sym::write_str]);
-        let expr = cx.expr_call_global(span, fn_path_write_str, vec![fmt, name]);
+        let expr = cx.expr_call_global(span, fn_path_write_str, thin_vec![fmt, name]);
         BlockOrExpr::new_expr(expr)
     } else if fields.len() <= CUTOFF {
         // Few enough fields that we can use a specific-length method.
@@ -89,7 +106,7 @@ fn show_substructure(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>
         };
         let fn_path_debug = cx.std_path(&[sym::fmt, sym::Formatter, Symbol::intern(&debug)]);
 
-        let mut args = Vec::with_capacity(2 + fields.len() * args_per_field);
+        let mut args = ThinVec::with_capacity(2 + fields.len() * args_per_field);
         args.extend([fmt, name]);
         for i in 0..fields.len() {
             let field = &fields[i];
@@ -97,47 +114,48 @@ fn show_substructure(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>
                 let name = cx.expr_str(field.span, field.name.unwrap().name);
                 args.push(name);
             }
-            // Use an extra indirection to make sure this works for unsized types.
-            let field = cx.expr_addr_of(field.span, field.self_expr.clone());
+
+            let field = expr_for_field(cx, field, i, fields.len());
             args.push(field);
         }
         let expr = cx.expr_call_global(span, fn_path_debug, args);
         BlockOrExpr::new_expr(expr)
     } else {
         // Enough fields that we must use the any-length method.
-        let mut name_exprs = Vec::with_capacity(fields.len());
-        let mut value_exprs = Vec::with_capacity(fields.len());
+        let mut name_exprs = ThinVec::with_capacity(fields.len());
+        let mut value_exprs = ThinVec::with_capacity(fields.len());
 
-        for field in fields {
+        for i in 0..fields.len() {
+            let field = &fields[i];
             if is_struct {
                 name_exprs.push(cx.expr_str(field.span, field.name.unwrap().name));
             }
 
-            // Use an extra indirection to make sure this works for unsized types.
-            let field = cx.expr_addr_of(field.span, field.self_expr.clone());
+            let field = expr_for_field(cx, field, i, fields.len());
             value_exprs.push(field);
         }
 
         // `let names: &'static _ = &["field1", "field2"];`
-        let names_let = if is_struct {
+        let names_let = is_struct.then(|| {
             let lt_static = Some(cx.lifetime_static(span));
             let ty_static_ref = cx.ty_ref(span, cx.ty_infer(span), lt_static, ast::Mutability::Not);
-            Some(cx.stmt_let_ty(
+            cx.stmt_let_ty(
                 span,
                 false,
                 Ident::new(sym::names, span),
                 Some(ty_static_ref),
                 cx.expr_array_ref(span, name_exprs),
-            ))
-        } else {
-            None
-        };
+            )
+        });
 
         // `let values: &[&dyn Debug] = &[&&self.field1, &&self.field2];`
         let path_debug = cx.path_global(span, cx.std_path(&[sym::fmt, sym::Debug]));
         let ty_dyn_debug = cx.ty(
             span,
-            ast::TyKind::TraitObject(vec![cx.trait_bound(path_debug)], ast::TraitObjectSyntax::Dyn),
+            ast::TyKind::TraitObject(
+                vec![cx.trait_bound(path_debug, false)],
+                ast::TraitObjectSyntax::Dyn,
+            ),
         );
         let ty_slice = cx.ty(
             span,
@@ -160,7 +178,7 @@ fn show_substructure(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>
         };
         let fn_path_debug_internal = cx.std_path(&[sym::fmt, sym::Formatter, sym_debug]);
 
-        let mut args = Vec::with_capacity(4);
+        let mut args = ThinVec::with_capacity(4);
         args.push(fmt);
         args.push(name);
         if is_struct {
@@ -169,7 +187,7 @@ fn show_substructure(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>
         args.push(cx.expr_ident(span, Ident::new(sym::values, span)));
         let expr = cx.expr_call_global(span, fn_path_debug_internal, args);
 
-        let mut stmts = Vec::with_capacity(3);
+        let mut stmts = ThinVec::with_capacity(2);
         if is_struct {
             stmts.push(names_let.unwrap());
         }
@@ -206,18 +224,18 @@ fn show_fieldless_enum(
             let pat = match &v.data {
                 ast::VariantData::Tuple(fields, _) => {
                     debug_assert!(fields.is_empty());
-                    cx.pat_tuple_struct(span, variant_path, vec![])
+                    cx.pat_tuple_struct(span, variant_path, ThinVec::new())
                 }
                 ast::VariantData::Struct(fields, _) => {
                     debug_assert!(fields.is_empty());
-                    cx.pat_struct(span, variant_path, vec![])
+                    cx.pat_struct(span, variant_path, ThinVec::new())
                 }
                 ast::VariantData::Unit(_) => cx.pat_path(span, variant_path),
             };
             cx.arm(span, pat, cx.expr_str(span, v.ident.name))
         })
-        .collect::<Vec<_>>();
+        .collect::<ThinVec<_>>();
     let name = cx.expr_match(span, cx.expr_self(span), arms);
     let fn_path_write_str = cx.std_path(&[sym::fmt, sym::Formatter, sym::write_str]);
-    BlockOrExpr::new_expr(cx.expr_call_global(span, fn_path_write_str, vec![fmt, name]))
+    BlockOrExpr::new_expr(cx.expr_call_global(span, fn_path_write_str, thin_vec![fmt, name]))
 }
diff --git a/compiler/rustc_builtin_macros/src/deriving/decodable.rs b/compiler/rustc_builtin_macros/src/deriving/decodable.rs
index 5f9519dad..3921533c8 100644
--- a/compiler/rustc_builtin_macros/src/deriving/decodable.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/decodable.rs
@@ -3,12 +3,12 @@
 use crate::deriving::generic::ty::*;
 use crate::deriving::generic::*;
 use crate::deriving::pathvec_std;
-
 use rustc_ast::ptr::P;
 use rustc_ast::{self as ast, Expr, MetaItem, Mutability};
 use rustc_expand::base::{Annotatable, ExtCtxt};
 use rustc_span::symbol::{sym, Ident, Symbol};
 use rustc_span::Span;
+use thin_vec::{thin_vec, ThinVec};
 
 pub fn expand_deriving_rustc_decodable(
     cx: &mut ExtCtxt<'_>,
@@ -25,6 +25,7 @@ pub fn expand_deriving_rustc_decodable(
         span,
         path: Path::new_(vec![krate, sym::Decodable], vec![], PathKind::Global),
         skip_path_as_bound: false,
+        needs_copy_as_bound_if_packed: true,
         additional_bounds: Vec::new(),
         supports_unions: false,
         methods: vec![MethodDef {
@@ -95,7 +96,7 @@ fn decodable_substructure(
                         cx.expr_call_global(
                             span,
                             fn_read_struct_field_path.clone(),
-                            vec![
+                            thin_vec![
                                 blkdecoder.clone(),
                                 cx.expr_str(span, name),
                                 cx.expr_usize(span, field),
@@ -111,7 +112,7 @@ fn decodable_substructure(
             cx.expr_call_global(
                 trait_span,
                 fn_read_struct_path,
-                vec![
+                thin_vec![
                     decoder,
                     cx.expr_str(trait_span, substr.type_ident.name),
                     cx.expr_usize(trait_span, nfields),
@@ -122,8 +123,8 @@ fn decodable_substructure(
         StaticEnum(_, fields) => {
             let variant = Ident::new(sym::i, trait_span);
 
-            let mut arms = Vec::with_capacity(fields.len() + 1);
-            let mut variants = Vec::with_capacity(fields.len());
+            let mut arms = ThinVec::with_capacity(fields.len() + 1);
+            let mut variants = ThinVec::with_capacity(fields.len());
 
             let fn_read_enum_variant_arg_path: Vec<_> =
                 cx.def_site_path(&[sym::rustc_serialize, sym::Decoder, sym::read_enum_variant_arg]);
@@ -140,7 +141,7 @@ fn decodable_substructure(
                             cx.expr_call_global(
                                 span,
                                 fn_read_enum_variant_arg_path.clone(),
-                                vec![blkdecoder.clone(), idx, exprdecode.clone()],
+                                thin_vec![blkdecoder.clone(), idx, exprdecode.clone()],
                             ),
                         )
                     });
@@ -161,7 +162,7 @@ fn decodable_substructure(
             let result = cx.expr_call_global(
                 trait_span,
                 fn_read_enum_variant_path,
-                vec![blkdecoder, variant_array_ref, lambda],
+                thin_vec![blkdecoder, variant_array_ref, lambda],
             );
             let fn_read_enum_path: Vec<_> =
                 cx.def_site_path(&[sym::rustc_serialize, sym::Decoder, sym::read_enum]);
@@ -169,7 +170,7 @@ fn decodable_substructure(
             cx.expr_call_global(
                 trait_span,
                 fn_read_enum_path,
-                vec![
+                thin_vec![
                     decoder,
                     cx.expr_str(trait_span, substr.type_ident.name),
                     cx.lambda1(trait_span, result, blkarg),
diff --git a/compiler/rustc_builtin_macros/src/deriving/default.rs b/compiler/rustc_builtin_macros/src/deriving/default.rs
index 182707472..4d753a2ed 100644
--- a/compiler/rustc_builtin_macros/src/deriving/default.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/default.rs
@@ -8,7 +8,7 @@ use rustc_span::symbol::Ident;
 use rustc_span::symbol::{kw, sym};
 use rustc_span::Span;
 use smallvec::SmallVec;
-use thin_vec::thin_vec;
+use thin_vec::{thin_vec, ThinVec};
 
 pub fn expand_deriving_default(
     cx: &mut ExtCtxt<'_>,
@@ -25,6 +25,7 @@ pub fn expand_deriving_default(
         span,
         path: Path::new(vec![kw::Default, sym::Default]),
         skip_path_as_bound: has_a_default_variant(item),
+        needs_copy_as_bound_if_packed: false,
         additional_bounds: Vec::new(),
         supports_unions: false,
         methods: vec![MethodDef {
@@ -59,7 +60,7 @@ fn default_struct_substructure(
 ) -> BlockOrExpr {
     // Note that `kw::Default` is "default" and `sym::Default` is "Default"!
     let default_ident = cx.std_path(&[kw::Default, sym::Default, kw::Default]);
-    let default_call = |span| cx.expr_call_global(span, default_ident.clone(), Vec::new());
+    let default_call = |span| cx.expr_call_global(span, default_ident.clone(), ThinVec::new());
 
     let expr = match summary {
         Unnamed(_, false) => cx.expr_ident(trait_span, substr.type_ident),
diff --git a/compiler/rustc_builtin_macros/src/deriving/encodable.rs b/compiler/rustc_builtin_macros/src/deriving/encodable.rs
index 2afeed927..a3b11309d 100644
--- a/compiler/rustc_builtin_macros/src/deriving/encodable.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/encodable.rs
@@ -88,11 +88,11 @@
 use crate::deriving::generic::ty::*;
 use crate::deriving::generic::*;
 use crate::deriving::pathvec_std;
-
 use rustc_ast::{AttrVec, ExprKind, MetaItem, Mutability};
 use rustc_expand::base::{Annotatable, ExtCtxt};
 use rustc_span::symbol::{sym, Ident, Symbol};
 use rustc_span::Span;
+use thin_vec::{thin_vec, ThinVec};
 
 pub fn expand_deriving_rustc_encodable(
     cx: &mut ExtCtxt<'_>,
@@ -109,6 +109,7 @@ pub fn expand_deriving_rustc_encodable(
         span,
         path: Path::new_(vec![krate, sym::Encodable], vec![], PathKind::Global),
         skip_path_as_bound: false,
+        needs_copy_as_bound_if_packed: true,
         additional_bounds: Vec::new(),
         supports_unions: false,
         methods: vec![MethodDef {
@@ -168,19 +169,20 @@ fn encodable_substructure(
         Struct(_, fields) => {
             let fn_emit_struct_field_path =
                 cx.def_site_path(&[sym::rustc_serialize, sym::Encoder, sym::emit_struct_field]);
-            let mut stmts = Vec::new();
+            let mut stmts = ThinVec::new();
             for (i, &FieldInfo { name, ref self_expr, span, .. }) in fields.iter().enumerate() {
                 let name = match name {
                     Some(id) => id.name,
                     None => Symbol::intern(&format!("_field{}", i)),
                 };
                 let self_ref = cx.expr_addr_of(span, self_expr.clone());
-                let enc = cx.expr_call(span, fn_path.clone(), vec![self_ref, blkencoder.clone()]);
+                let enc =
+                    cx.expr_call(span, fn_path.clone(), thin_vec![self_ref, blkencoder.clone()]);
                 let lambda = cx.lambda1(span, enc, blkarg);
                 let call = cx.expr_call_global(
                     span,
                     fn_emit_struct_field_path.clone(),
-                    vec![
+                    thin_vec![
                         blkencoder.clone(),
                         cx.expr_str(span, name),
                         cx.expr_usize(span, i),
@@ -202,7 +204,7 @@ fn encodable_substructure(
 
             // unit structs have no fields and need to return Ok()
             let blk = if stmts.is_empty() {
-                let ok = cx.expr_ok(trait_span, cx.expr_tuple(trait_span, vec![]));
+                let ok = cx.expr_ok(trait_span, cx.expr_tuple(trait_span, ThinVec::new()));
                 cx.lambda1(trait_span, ok, blkarg)
             } else {
                 cx.lambda_stmts_1(trait_span, stmts, blkarg)
@@ -214,7 +216,7 @@ fn encodable_substructure(
             let expr = cx.expr_call_global(
                 trait_span,
                 fn_emit_struct_path,
-                vec![
+                thin_vec![
                     encoder,
                     cx.expr_str(trait_span, substr.type_ident.name),
                     cx.expr_usize(trait_span, fields.len()),
@@ -235,19 +237,22 @@ fn encodable_substructure(
             let fn_emit_enum_variant_arg_path: Vec<_> =
                 cx.def_site_path(&[sym::rustc_serialize, sym::Encoder, sym::emit_enum_variant_arg]);
 
-            let mut stmts = Vec::new();
+            let mut stmts = ThinVec::new();
             if !fields.is_empty() {
                 let last = fields.len() - 1;
                 for (i, &FieldInfo { ref self_expr, span, .. }) in fields.iter().enumerate() {
                     let self_ref = cx.expr_addr_of(span, self_expr.clone());
-                    let enc =
-                        cx.expr_call(span, fn_path.clone(), vec![self_ref, blkencoder.clone()]);
+                    let enc = cx.expr_call(
+                        span,
+                        fn_path.clone(),
+                        thin_vec![self_ref, blkencoder.clone()],
+                    );
                     let lambda = cx.lambda1(span, enc, blkarg);
 
                     let call = cx.expr_call_global(
                         span,
                         fn_emit_enum_variant_arg_path.clone(),
-                        vec![blkencoder.clone(), cx.expr_usize(span, i), lambda],
+                        thin_vec![blkencoder.clone(), cx.expr_usize(span, i), lambda],
                     );
                     let call = if i != last {
                         cx.expr_try(span, call)
@@ -257,7 +262,7 @@ fn encodable_substructure(
                     stmts.push(cx.stmt_expr(call));
                 }
             } else {
-                let ok = cx.expr_ok(trait_span, cx.expr_tuple(trait_span, vec![]));
+                let ok = cx.expr_ok(trait_span, cx.expr_tuple(trait_span, ThinVec::new()));
                 let ret_ok = cx.expr(trait_span, ExprKind::Ret(Some(ok)));
                 stmts.push(cx.stmt_expr(ret_ok));
             }
@@ -271,7 +276,7 @@ fn encodable_substructure(
             let call = cx.expr_call_global(
                 trait_span,
                 fn_emit_enum_variant_path,
-                vec![
+                thin_vec![
                     blkencoder,
                     name,
                     cx.expr_usize(trait_span, *idx),
@@ -286,9 +291,9 @@ fn encodable_substructure(
             let expr = cx.expr_call_global(
                 trait_span,
                 fn_emit_enum_path,
-                vec![encoder, cx.expr_str(trait_span, substr.type_ident.name), blk],
+                thin_vec![encoder, cx.expr_str(trait_span, substr.type_ident.name), blk],
             );
-            BlockOrExpr::new_mixed(vec![me], Some(expr))
+            BlockOrExpr::new_mixed(thin_vec![me], Some(expr))
         }
 
         _ => cx.bug("expected Struct or EnumMatching in derive(Encodable)"),
diff --git a/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs b/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs
index 17b7ac0eb..1f819beeb 100644
--- a/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs
@@ -165,18 +165,19 @@ pub use SubstructureFields::*;
 use crate::deriving;
 use rustc_ast::ptr::P;
 use rustc_ast::{
-    self as ast, BindingAnnotation, ByRef, EnumDef, Expr, Generics, Mutability, PatKind,
+    self as ast, BindingAnnotation, ByRef, EnumDef, Expr, GenericArg, GenericParamKind, Generics,
+    Mutability, PatKind, TyKind, VariantData,
 };
-use rustc_ast::{GenericArg, GenericParamKind, VariantData};
 use rustc_attr as attr;
 use rustc_expand::base::{Annotatable, ExtCtxt};
+use rustc_session::lint::builtin::BYTE_SLICE_IN_PACKED_STRUCT_WITH_DERIVE;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
 use std::cell::RefCell;
 use std::iter;
 use std::ops::Not;
 use std::vec;
-use thin_vec::thin_vec;
+use thin_vec::{thin_vec, ThinVec};
 use ty::{Bounds, Path, Ref, Self_, Ty};
 
 pub mod ty;
@@ -191,6 +192,9 @@ pub struct TraitDef<'a> {
     /// Whether to skip adding the current trait as a bound to the type parameters of the type.
     pub skip_path_as_bound: bool,
 
+    /// Whether `Copy` is needed as an additional bound on type parameters in a packed struct.
+    pub needs_copy_as_bound_if_packed: bool,
+
     /// Additional bounds required of any type parameters of the type,
     /// other than the current trait
     pub additional_bounds: Vec<Ty>,
@@ -314,7 +318,7 @@ pub fn combine_substructure(
 }
 
 struct TypeParameter {
-    bound_generic_params: Vec<ast::GenericParam>,
+    bound_generic_params: ThinVec<ast::GenericParam>,
     ty: P<ast::Ty>,
 }
 
@@ -324,18 +328,18 @@ struct TypeParameter {
 /// avoiding the insertion of any unnecessary blocks.
 ///
 /// The statements come before the expression.
-pub struct BlockOrExpr(Vec<ast::Stmt>, Option<P<Expr>>);
+pub struct BlockOrExpr(ThinVec<ast::Stmt>, Option<P<Expr>>);
 
 impl BlockOrExpr {
-    pub fn new_stmts(stmts: Vec<ast::Stmt>) -> BlockOrExpr {
+    pub fn new_stmts(stmts: ThinVec<ast::Stmt>) -> BlockOrExpr {
         BlockOrExpr(stmts, None)
     }
 
     pub fn new_expr(expr: P<Expr>) -> BlockOrExpr {
-        BlockOrExpr(vec![], Some(expr))
+        BlockOrExpr(ThinVec::new(), Some(expr))
     }
 
-    pub fn new_mixed(stmts: Vec<ast::Stmt>, expr: Option<P<Expr>>) -> BlockOrExpr {
+    pub fn new_mixed(stmts: ThinVec<ast::Stmt>, expr: Option<P<Expr>>) -> BlockOrExpr {
         BlockOrExpr(stmts, expr)
     }
 
@@ -351,7 +355,7 @@ impl BlockOrExpr {
     fn into_expr(self, cx: &ExtCtxt<'_>, span: Span) -> P<Expr> {
         if self.0.is_empty() {
             match self.1 {
-                None => cx.expr_block(cx.block(span, vec![])),
+                None => cx.expr_block(cx.block(span, ThinVec::new())),
                 Some(expr) => expr,
             }
         } else if self.0.len() == 1
@@ -381,7 +385,7 @@ fn find_type_parameters(
     struct Visitor<'a, 'b> {
         cx: &'a ExtCtxt<'b>,
         ty_param_names: &'a [Symbol],
-        bound_generic_params_stack: Vec<ast::GenericParam>,
+        bound_generic_params_stack: ThinVec<ast::GenericParam>,
         type_params: Vec<TypeParameter>,
     }
 
@@ -418,7 +422,7 @@ fn find_type_parameters(
     let mut visitor = Visitor {
         cx,
         ty_param_names,
-        bound_generic_params_stack: Vec::new(),
+        bound_generic_params_stack: ThinVec::new(),
         type_params: Vec::new(),
     };
     visit::Visitor::visit_ty(&mut visitor, ty);
@@ -455,18 +459,6 @@ impl<'a> TraitDef<'a> {
                     }
                     false
                 });
-                let has_no_type_params = match &item.kind {
-                    ast::ItemKind::Struct(_, generics)
-                    | ast::ItemKind::Enum(_, generics)
-                    | ast::ItemKind::Union(_, generics) => !generics
-                        .params
-                        .iter()
-                        .any(|param| matches!(param.kind, ast::GenericParamKind::Type { .. })),
-                    _ => unreachable!(),
-                };
-                let container_id = cx.current_expansion.id.expn_data().parent.expect_local();
-                let copy_fields =
-                    is_packed && has_no_type_params && cx.resolver.has_derive_copy(container_id);
 
                 let newitem = match &item.kind {
                     ast::ItemKind::Struct(struct_def, generics) => self.expand_struct_def(
@@ -475,7 +467,7 @@ impl<'a> TraitDef<'a> {
                         item.ident,
                         generics,
                         from_scratch,
-                        copy_fields,
+                        is_packed,
                     ),
                     ast::ItemKind::Enum(enum_def, generics) => {
                         // We ignore `is_packed` here, because `repr(packed)`
@@ -493,7 +485,7 @@ impl<'a> TraitDef<'a> {
                                 item.ident,
                                 generics,
                                 from_scratch,
-                                copy_fields,
+                                is_packed,
                             )
                         } else {
                             cx.span_err(mitem.span, "this trait cannot be derived for unions");
@@ -565,6 +557,7 @@ impl<'a> TraitDef<'a> {
         generics: &Generics,
         field_tys: Vec<P<ast::Ty>>,
         methods: Vec<P<ast::AssocItem>>,
+        is_packed: bool,
     ) -> P<ast::Item> {
         let trait_path = self.path.to_path(cx, self.span, type_ident, generics);
 
@@ -601,26 +594,46 @@ impl<'a> TraitDef<'a> {
         let span = generics.span.with_ctxt(ctxt);
 
         // Create the generic parameters
-        let params: Vec<_> = generics
+        let params: ThinVec<_> = generics
             .params
             .iter()
             .map(|param| match &param.kind {
                 GenericParamKind::Lifetime { .. } => param.clone(),
                 GenericParamKind::Type { .. } => {
-                    // I don't think this can be moved out of the loop, since
-                    // a GenericBound requires an ast id
-                    let bounds: Vec<_> =
-                    // extra restrictions on the generics parameters to the
-                    // type being derived upon
-                    self.additional_bounds.iter().map(|p| {
-                        cx.trait_bound(p.to_path(cx, self.span, type_ident, generics))
-                    }).chain(
-                        // require the current trait
-                        self.skip_path_as_bound.not().then(|| cx.trait_bound(trait_path.clone()))
-                    ).chain(
-                        // also add in any bounds from the declaration
-                        param.bounds.iter().cloned()
-                    ).collect();
+                    // Extra restrictions on the generics parameters to the
+                    // type being derived upon.
+                    let bounds: Vec<_> = self
+                        .additional_bounds
+                        .iter()
+                        .map(|p| {
+                            cx.trait_bound(
+                                p.to_path(cx, self.span, type_ident, generics),
+                                self.is_const,
+                            )
+                        })
+                        .chain(
+                            // Add a bound for the current trait.
+                            self.skip_path_as_bound
+                                .not()
+                                .then(|| cx.trait_bound(trait_path.clone(), self.is_const)),
+                        )
+                        .chain({
+                            // Add a `Copy` bound if required.
+                            if is_packed && self.needs_copy_as_bound_if_packed {
+                                let p = deriving::path_std!(marker::Copy);
+                                Some(cx.trait_bound(
+                                    p.to_path(cx, self.span, type_ident, generics),
+                                    self.is_const,
+                                ))
+                            } else {
+                                None
+                            }
+                        })
+                        .chain(
+                            // Also add in any bounds from the declaration.
+                            param.bounds.iter().cloned(),
+                        )
+                        .collect();
 
                     cx.typaram(param.ident.span.with_ctxt(ctxt), param.ident, bounds, None)
                 }
@@ -689,11 +702,25 @@ impl<'a> TraitDef<'a> {
                         let mut bounds: Vec<_> = self
                             .additional_bounds
                             .iter()
-                            .map(|p| cx.trait_bound(p.to_path(cx, self.span, type_ident, generics)))
+                            .map(|p| {
+                                cx.trait_bound(
+                                    p.to_path(cx, self.span, type_ident, generics),
+                                    self.is_const,
+                                )
+                            })
                             .collect();
 
-                        // require the current trait
-                        bounds.push(cx.trait_bound(trait_path.clone()));
+                        // Require the current trait.
+                        bounds.push(cx.trait_bound(trait_path.clone(), self.is_const));
+
+                        // Add a `Copy` bound if required.
+                        if is_packed && self.needs_copy_as_bound_if_packed {
+                            let p = deriving::path_std!(marker::Copy);
+                            bounds.push(cx.trait_bound(
+                                p.to_path(cx, self.span, type_ident, generics),
+                                self.is_const,
+                            ));
+                        }
 
                         let predicate = ast::WhereBoundPredicate {
                             span: self.span,
@@ -734,8 +761,7 @@ impl<'a> TraitDef<'a> {
         let path = cx.path_all(self.span, false, vec![type_ident], self_params);
         let self_type = cx.ty_path(path);
 
-        let attr = cx.attr_word(sym::automatically_derived, self.span);
-        let attrs = thin_vec![attr];
+        let attrs = thin_vec![cx.attr_word(sym::automatically_derived, self.span),];
         let opt_trait_ref = Some(trait_ref);
 
         cx.item(
@@ -762,7 +788,7 @@ impl<'a> TraitDef<'a> {
         type_ident: Ident,
         generics: &Generics,
         from_scratch: bool,
-        copy_fields: bool,
+        is_packed: bool,
     ) -> P<ast::Item> {
         let field_tys: Vec<P<ast::Ty>> =
             struct_def.fields().iter().map(|field| field.ty.clone()).collect();
@@ -790,7 +816,7 @@ impl<'a> TraitDef<'a> {
                         type_ident,
                         &selflike_args,
                         &nonselflike_args,
-                        copy_fields,
+                        is_packed,
                     )
                 };
 
@@ -806,7 +832,7 @@ impl<'a> TraitDef<'a> {
             })
             .collect();
 
-        self.create_derived_impl(cx, type_ident, generics, field_tys, methods)
+        self.create_derived_impl(cx, type_ident, generics, field_tys, methods, is_packed)
     }
 
     fn expand_enum_def(
@@ -861,7 +887,8 @@ impl<'a> TraitDef<'a> {
             })
             .collect();
 
-        self.create_derived_impl(cx, type_ident, generics, field_tys, methods)
+        let is_packed = false; // enums are never packed
+        self.create_derived_impl(cx, type_ident, generics, field_tys, methods, is_packed)
     }
 }
 
@@ -908,19 +935,17 @@ impl<'a> MethodDef<'a> {
         trait_: &TraitDef<'_>,
         type_ident: Ident,
         generics: &Generics,
-    ) -> (Option<ast::ExplicitSelf>, Vec<P<Expr>>, Vec<P<Expr>>, Vec<(Ident, P<ast::Ty>)>) {
-        let mut selflike_args = Vec::new();
+    ) -> (Option<ast::ExplicitSelf>, ThinVec<P<Expr>>, Vec<P<Expr>>, Vec<(Ident, P<ast::Ty>)>) {
+        let mut selflike_args = ThinVec::new();
         let mut nonselflike_args = Vec::new();
         let mut nonself_arg_tys = Vec::new();
         let span = trait_.span;
 
-        let explicit_self = if self.explicit_self {
+        let explicit_self = self.explicit_self.then(|| {
             let (self_expr, explicit_self) = ty::get_explicit_self(cx, span);
             selflike_args.push(self_expr);
-            Some(explicit_self)
-        } else {
-            None
-        };
+            explicit_self
+        });
 
         for (ty, name) in self.nonself_args.iter() {
             let ast_ty = ty.to_ty(cx, span, type_ident, generics);
@@ -1011,8 +1036,8 @@ impl<'a> MethodDef<'a> {
     /// ```
     /// But if the struct is `repr(packed)`, we can't use something like
     /// `&self.x` because that might cause an unaligned ref. So for any trait
-    /// method that takes a reference, if the struct impls `Copy` then we use a
-    /// local block to force a copy:
+    /// method that takes a reference, we use a local block to force a copy.
+    /// This requires that the field impl `Copy`.
     /// ```
     /// # struct A { x: u8, y: u8 }
     /// impl PartialEq for A {
@@ -1027,10 +1052,6 @@ impl<'a> MethodDef<'a> {
     ///         ::core::hash::Hash::hash(&{ self.y }, state)
     ///     }
     /// }
-    /// ```
-    /// If the struct doesn't impl `Copy`, we use the normal `&self.x`. This
-    /// only works if the fields match the alignment required by the
-    /// `packed(N)` attribute. (We'll get errors later on if not.)
     fn expand_struct_method_body<'b>(
         &self,
         cx: &mut ExtCtxt<'_>,
@@ -1039,12 +1060,12 @@ impl<'a> MethodDef<'a> {
         type_ident: Ident,
         selflike_args: &[P<Expr>],
         nonselflike_args: &[P<Expr>],
-        copy_fields: bool,
+        is_packed: bool,
     ) -> BlockOrExpr {
         assert!(selflike_args.len() == 1 || selflike_args.len() == 2);
 
         let selflike_fields =
-            trait_.create_struct_field_access_fields(cx, selflike_args, struct_def, copy_fields);
+            trait_.create_struct_field_access_fields(cx, selflike_args, struct_def, is_packed);
         self.call_substructure_method(
             cx,
             trait_,
@@ -1112,7 +1133,7 @@ impl<'a> MethodDef<'a> {
         trait_: &TraitDef<'b>,
         enum_def: &'b EnumDef,
         type_ident: Ident,
-        selflike_args: Vec<P<Expr>>,
+        selflike_args: ThinVec<P<Expr>>,
         nonselflike_args: &[P<Expr>],
     ) -> BlockOrExpr {
         let span = trait_.span;
@@ -1125,7 +1146,7 @@ impl<'a> MethodDef<'a> {
         // There is no sensible code to be generated for *any* deriving on a
         // zero-variant enum. So we just generate a failing expression.
         if variants.is_empty() {
-            return BlockOrExpr(vec![], Some(deriving::call_unreachable(cx, span)));
+            return BlockOrExpr(ThinVec::new(), Some(deriving::call_unreachable(cx, span)));
         }
 
         let prefixes = iter::once("__self".to_string())
@@ -1161,13 +1182,13 @@ impl<'a> MethodDef<'a> {
             let other_selflike_exprs = tag_exprs;
             let tag_field = FieldInfo { span, name: None, self_expr, other_selflike_exprs };
 
-            let tag_let_stmts: Vec<_> = iter::zip(&tag_idents, &selflike_args)
+            let tag_let_stmts: ThinVec<_> = iter::zip(&tag_idents, &selflike_args)
                 .map(|(&ident, selflike_arg)| {
                     let variant_value = deriving::call_intrinsic(
                         cx,
                         span,
                         sym::discriminant_value,
-                        vec![selflike_arg.clone()],
+                        thin_vec![selflike_arg.clone()],
                     );
                     cx.stmt_let(span, false, ident, variant_value)
                 })
@@ -1226,7 +1247,7 @@ impl<'a> MethodDef<'a> {
         // (Variant2, Variant2, ...) => Body2
         // ...
         // where each tuple has length = selflike_args.len()
-        let mut match_arms: Vec<ast::Arm> = variants
+        let mut match_arms: ThinVec<ast::Arm> = variants
             .iter()
             .enumerate()
             .filter(|&(_, v)| !(unify_fieldless_variants && v.data.fields().is_empty()))
@@ -1239,7 +1260,7 @@ impl<'a> MethodDef<'a> {
                 let sp = variant.span.with_ctxt(trait_.span.ctxt());
                 let variant_path = cx.path(sp, vec![type_ident, variant.ident]);
                 let by_ref = ByRef::No; // because enums can't be repr(packed)
-                let mut subpats: Vec<_> = trait_.create_struct_patterns(
+                let mut subpats = trait_.create_struct_patterns(
                     cx,
                     variant_path,
                     &variant.data,
@@ -1315,7 +1336,7 @@ impl<'a> MethodDef<'a> {
         //          ...
         //          _ => ::core::intrinsics::unreachable()
         //      }
-        let get_match_expr = |mut selflike_args: Vec<P<Expr>>| {
+        let get_match_expr = |mut selflike_args: ThinVec<P<Expr>>| {
             let match_arg = if selflike_args.len() == 1 {
                 selflike_args.pop().unwrap()
             } else {
@@ -1341,7 +1362,7 @@ impl<'a> MethodDef<'a> {
             tag_let_stmts.append(&mut tag_check_plus_match.0);
             BlockOrExpr(tag_let_stmts, tag_check_plus_match.1)
         } else {
-            BlockOrExpr(vec![], Some(get_match_expr(selflike_args)))
+            BlockOrExpr(ThinVec::new(), Some(get_match_expr(selflike_args)))
         }
     }
 
@@ -1406,7 +1427,7 @@ impl<'a> TraitDef<'a> {
         struct_def: &'a VariantData,
         prefixes: &[String],
         by_ref: ByRef,
-    ) -> Vec<P<ast::Pat>> {
+    ) -> ThinVec<P<ast::Pat>> {
         prefixes
             .iter()
             .map(|prefix| {
@@ -1514,7 +1535,7 @@ impl<'a> TraitDef<'a> {
         cx: &mut ExtCtxt<'_>,
         selflike_args: &[P<Expr>],
         struct_def: &'a VariantData,
-        copy_fields: bool,
+        is_packed: bool,
     ) -> Vec<FieldInfo> {
         self.create_fields(struct_def, |i, struct_field, sp| {
             selflike_args
@@ -1533,10 +1554,54 @@ impl<'a> TraitDef<'a> {
                             }),
                         ),
                     );
-                    if copy_fields {
-                        field_expr = cx.expr_block(
-                            cx.block(struct_field.span, vec![cx.stmt_expr(field_expr)]),
-                        );
+                    if is_packed {
+                        // In general, fields in packed structs are copied via a
+                        // block, e.g. `&{self.0}`. The two exceptions are `[u8]`
+                        // and `str` fields, which cannot be copied and also never
+                        // cause unaligned references. These exceptions are allowed
+                        // to handle the `FlexZeroSlice` type in the `zerovec`
+                        // crate within `icu4x-0.9.0`.
+                        //
+                        // Once use of `icu4x-0.9.0` has dropped sufficiently, this
+                        // exception should be removed.
+                        let is_simple_path = |ty: &P<ast::Ty>, sym| {
+                            if let TyKind::Path(None, ast::Path { segments, .. }) = &ty.kind &&
+                                let [seg] = segments.as_slice() &&
+                                seg.ident.name == sym && seg.args.is_none()
+                            {
+                                true
+                            } else {
+                                false
+                            }
+                        };
+
+                        let exception = if let TyKind::Slice(ty) = &struct_field.ty.kind &&
+                            is_simple_path(ty, sym::u8)
+                        {
+                            Some("byte")
+                        } else if is_simple_path(&struct_field.ty, sym::str) {
+                            Some("string")
+                        } else {
+                            None
+                        };
+
+                        if let Some(ty) = exception {
+                            cx.sess.parse_sess.buffer_lint_with_diagnostic(
+                                BYTE_SLICE_IN_PACKED_STRUCT_WITH_DERIVE,
+                                sp,
+                                ast::CRATE_NODE_ID,
+                                &format!(
+                                    "{} slice in a packed struct that derives a built-in trait",
+                                    ty
+                                ),
+                                rustc_lint_defs::BuiltinLintDiagnostics::ByteSliceInPackedStructWithDerive
+                            );
+                        } else {
+                            // Wrap the expression in `{...}`, causing a copy.
+                            field_expr = cx.expr_block(
+                                cx.block(struct_field.span, thin_vec![cx.stmt_expr(field_expr)]),
+                            );
+                        }
                     }
                     cx.expr_addr_of(sp, field_expr)
                 })
diff --git a/compiler/rustc_builtin_macros/src/deriving/generic/ty.rs b/compiler/rustc_builtin_macros/src/deriving/generic/ty.rs
index c6f5f5d08..26f91b714 100644
--- a/compiler/rustc_builtin_macros/src/deriving/generic/ty.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/generic/ty.rs
@@ -9,6 +9,7 @@ use rustc_expand::base::ExtCtxt;
 use rustc_span::source_map::{respan, DUMMY_SP};
 use rustc_span::symbol::{kw, Ident, Symbol};
 use rustc_span::Span;
+use thin_vec::ThinVec;
 
 /// A path, e.g., `::std::option::Option::<i32>` (global). Has support
 /// for type parameters.
@@ -102,7 +103,7 @@ impl Ty {
             Path(p) => p.to_ty(cx, span, self_ty, self_generics),
             Self_ => cx.ty_path(self.to_path(cx, span, self_ty, self_generics)),
             Unit => {
-                let ty = ast::TyKind::Tup(vec![]);
+                let ty = ast::TyKind::Tup(ThinVec::new());
                 cx.ty(span, ty)
             }
         }
@@ -154,7 +155,7 @@ fn mk_ty_param(
         .iter()
         .map(|b| {
             let path = b.to_path(cx, span, self_ident, self_generics);
-            cx.trait_bound(path)
+            cx.trait_bound(path, false)
         })
         .collect();
     cx.typaram(span, Ident::new(name, span), bounds, None)
@@ -185,7 +186,11 @@ impl Bounds {
 
         Generics {
             params,
-            where_clause: ast::WhereClause { has_where_token: false, predicates: Vec::new(), span },
+            where_clause: ast::WhereClause {
+                has_where_token: false,
+                predicates: ThinVec::new(),
+                span,
+            },
             span,
         }
     }
diff --git a/compiler/rustc_builtin_macros/src/deriving/hash.rs b/compiler/rustc_builtin_macros/src/deriving/hash.rs
index f8570d8f8..4eee573db 100644
--- a/compiler/rustc_builtin_macros/src/deriving/hash.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/hash.rs
@@ -1,11 +1,11 @@
 use crate::deriving::generic::ty::*;
 use crate::deriving::generic::*;
 use crate::deriving::{path_std, pathvec_std};
-
 use rustc_ast::{AttrVec, MetaItem, Mutability};
 use rustc_expand::base::{Annotatable, ExtCtxt};
 use rustc_span::symbol::sym;
 use rustc_span::Span;
+use thin_vec::thin_vec;
 
 pub fn expand_deriving_hash(
     cx: &mut ExtCtxt<'_>,
@@ -24,6 +24,7 @@ pub fn expand_deriving_hash(
         span,
         path,
         skip_path_as_bound: false,
+        needs_copy_as_bound_if_packed: true,
         additional_bounds: Vec::new(),
         supports_unions: false,
         methods: vec![MethodDef {
@@ -59,7 +60,7 @@ fn hash_substructure(
 
             cx.expr_path(cx.path_global(span, strs))
         };
-        let expr = cx.expr_call(span, hash_path, vec![expr, state_expr.clone()]);
+        let expr = cx.expr_call(span, hash_path, thin_vec![expr, state_expr.clone()]);
         cx.stmt_expr(expr)
     };
 
@@ -71,7 +72,7 @@ fn hash_substructure(
         }
         EnumTag(tag_field, match_expr) => {
             assert!(tag_field.other_selflike_exprs.is_empty());
-            let stmts = vec![call_hash(tag_field.span, tag_field.self_expr.clone())];
+            let stmts = thin_vec![call_hash(tag_field.span, tag_field.self_expr.clone())];
             (stmts, match_expr.clone())
         }
         _ => cx.span_bug(trait_span, "impossible substructure in `derive(Hash)`"),
diff --git a/compiler/rustc_builtin_macros/src/deriving/mod.rs b/compiler/rustc_builtin_macros/src/deriving/mod.rs
index de657e4e6..d34336e76 100644
--- a/compiler/rustc_builtin_macros/src/deriving/mod.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/mod.rs
@@ -6,6 +6,7 @@ use rustc_ast::{GenericArg, Impl, ItemKind, MetaItem};
 use rustc_expand::base::{Annotatable, ExpandResult, ExtCtxt, MultiItemModifier};
 use rustc_span::symbol::{sym, Ident, Symbol};
 use rustc_span::Span;
+use thin_vec::{thin_vec, ThinVec};
 
 macro path_local($x:ident) {
     generic::ty::Path::new_local(sym::$x)
@@ -92,7 +93,7 @@ fn call_intrinsic(
     cx: &ExtCtxt<'_>,
     span: Span,
     intrinsic: Symbol,
-    args: Vec<P<ast::Expr>>,
+    args: ThinVec<P<ast::Expr>>,
 ) -> P<ast::Expr> {
     let span = cx.with_def_site_ctxt(span);
     let path = cx.std_path(&[sym::intrinsics, intrinsic]);
@@ -103,10 +104,10 @@ fn call_intrinsic(
 fn call_unreachable(cx: &ExtCtxt<'_>, span: Span) -> P<ast::Expr> {
     let span = cx.with_def_site_ctxt(span);
     let path = cx.std_path(&[sym::intrinsics, sym::unreachable]);
-    let call = cx.expr_call_global(span, path, vec![]);
+    let call = cx.expr_call_global(span, path, ThinVec::new());
 
     cx.expr_block(P(ast::Block {
-        stmts: vec![cx.stmt_expr(call)],
+        stmts: thin_vec![cx.stmt_expr(call)],
         id: ast::DUMMY_NODE_ID,
         rules: ast::BlockCheckMode::Unsafe(ast::CompilerGenerated),
         span,
@@ -202,7 +203,7 @@ fn inject_impl_of_structural_trait(
             generics,
             of_trait: Some(trait_ref),
             self_ty: self_type,
-            items: Vec::new(),
+            items: ThinVec::new(),
         })),
     );
 
@@ -211,7 +212,7 @@ fn inject_impl_of_structural_trait(
 
 fn assert_ty_bounds(
     cx: &mut ExtCtxt<'_>,
-    stmts: &mut Vec<ast::Stmt>,
+    stmts: &mut ThinVec<ast::Stmt>,
     ty: P<ast::Ty>,
     span: Span,
     assert_path: &[Symbol],
diff --git a/compiler/rustc_builtin_macros/src/env.rs b/compiler/rustc_builtin_macros/src/env.rs
index e5a5e6069..f011cb754 100644
--- a/compiler/rustc_builtin_macros/src/env.rs
+++ b/compiler/rustc_builtin_macros/src/env.rs
@@ -8,8 +8,8 @@ use rustc_ast::{self as ast, GenericArg};
 use rustc_expand::base::{self, *};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::Span;
-
 use std::env;
+use thin_vec::thin_vec;
 
 pub fn expand_option_env<'cx>(
     cx: &'cx mut ExtCtxt<'_>,
@@ -41,7 +41,7 @@ pub fn expand_option_env<'cx>(
         Some(value) => cx.expr_call_global(
             sp,
             cx.std_path(&[sym::option, sym::Option, sym::Some]),
-            vec![cx.expr_str(sp, value)],
+            thin_vec![cx.expr_str(sp, value)],
         ),
     };
     MacEager::expr(e)
@@ -53,7 +53,7 @@ pub fn expand_env<'cx>(
     tts: TokenStream,
 ) -> Box<dyn base::MacResult + 'cx> {
     let mut exprs = match get_exprs_from_tts(cx, tts) {
-        Some(exprs) if exprs.is_empty() => {
+        Some(exprs) if exprs.is_empty() || exprs.len() > 2 => {
             cx.span_err(sp, "env! takes 1 or 2 arguments");
             return DummyResult::any(sp);
         }
@@ -64,28 +64,48 @@ pub fn expand_env<'cx>(
     let Some((var, _style)) = expr_to_string(cx, exprs.next().unwrap(), "expected string literal") else {
         return DummyResult::any(sp);
     };
-    let msg = match exprs.next() {
-        None => Symbol::intern(&format!("environment variable `{}` not defined", var)),
+
+    let custom_msg = match exprs.next() {
+        None => None,
         Some(second) => match expr_to_string(cx, second, "expected string literal") {
             None => return DummyResult::any(sp),
-            Some((s, _style)) => s,
+            Some((s, _style)) => Some(s),
         },
     };
 
-    if exprs.next().is_some() {
-        cx.span_err(sp, "env! takes 1 or 2 arguments");
-        return DummyResult::any(sp);
-    }
-
     let sp = cx.with_def_site_ctxt(sp);
     let value = env::var(var.as_str()).ok().as_deref().map(Symbol::intern);
     cx.sess.parse_sess.env_depinfo.borrow_mut().insert((var, value));
     let e = match value {
         None => {
-            cx.span_err(sp, msg.as_str());
+            let (msg, help) = match custom_msg {
+                None => (
+                    format!("environment variable `{var}` not defined at compile time"),
+                    Some(help_for_missing_env_var(var.as_str())),
+                ),
+                Some(s) => (s.to_string(), None),
+            };
+            let mut diag = cx.struct_span_err(sp, &msg);
+            if let Some(help) = help {
+                diag.help(help);
+            }
+            diag.emit();
             return DummyResult::any(sp);
         }
         Some(value) => cx.expr_str(sp, value),
     };
     MacEager::expr(e)
 }
+
+fn help_for_missing_env_var(var: &str) -> String {
+    if var.starts_with("CARGO_")
+        || var.starts_with("DEP_")
+        || matches!(var, "OUT_DIR" | "OPT_LEVEL" | "PROFILE" | "HOST" | "TARGET")
+    {
+        format!(
+            "Cargo sets build script variables at run time. Use `std::env::var(\"{var}\")` instead"
+        )
+    } else {
+        format!("Use `std::env::var(\"{var}\")` to read the variable at run time")
+    }
+}
diff --git a/compiler/rustc_builtin_macros/src/format.rs b/compiler/rustc_builtin_macros/src/format.rs
index 9f4bbbc62..e93a23394 100644
--- a/compiler/rustc_builtin_macros/src/format.rs
+++ b/compiler/rustc_builtin_macros/src/format.rs
@@ -1,7 +1,11 @@
 use rustc_ast::ptr::P;
 use rustc_ast::token;
 use rustc_ast::tokenstream::TokenStream;
-use rustc_ast::Expr;
+use rustc_ast::{
+    Expr, ExprKind, FormatAlignment, FormatArgPosition, FormatArgPositionKind, FormatArgs,
+    FormatArgsPiece, FormatArgument, FormatArgumentKind, FormatArguments, FormatCount,
+    FormatDebugHex, FormatOptions, FormatPlaceholder, FormatSign, FormatTrait,
+};
 use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::{pluralize, Applicability, MultiSpan, PResult};
 use rustc_expand::base::{self, *};
@@ -12,21 +16,15 @@ use rustc_span::{BytePos, InnerSpan, Span};
 use rustc_lint_defs::builtin::NAMED_ARGUMENTS_USED_POSITIONALLY;
 use rustc_lint_defs::{BufferedEarlyLint, BuiltinLintDiagnostics, LintId};
 
-mod ast;
-use ast::*;
-
-mod expand;
-use expand::expand_parsed_format_args;
-
 // The format_args!() macro is expanded in three steps:
 //  1. First, `parse_args` will parse the `(literal, arg, arg, name=arg, name=arg)` syntax,
 //     but doesn't parse the template (the literal) itself.
 //  2. Second, `make_format_args` will parse the template, the format options, resolve argument references,
-//     produce diagnostics, and turn the whole thing into a `FormatArgs` structure.
-//  3. Finally, `expand_parsed_format_args` will turn that `FormatArgs` structure
-//     into the expression that the macro expands to.
+//     produce diagnostics, and turn the whole thing into a `FormatArgs` AST node.
+//  3. Much later, in AST lowering (rustc_ast_lowering), that `FormatArgs` structure will be turned
+//     into the expression of type `core::fmt::Arguments`.
 
-// See format/ast.rs for the FormatArgs structure and glossary.
+// See rustc_ast/src/format.rs for the FormatArgs structure and glossary.
 
 // Only used in parse_args and report_invalid_references,
 // to indicate how a referred argument was used.
@@ -437,7 +435,16 @@ pub fn make_format_args(
                     format_options: FormatOptions {
                         fill: format.fill,
                         alignment,
-                        flags: format.flags,
+                        sign: format.sign.map(|s| match s {
+                            parse::Sign::Plus => FormatSign::Plus,
+                            parse::Sign::Minus => FormatSign::Minus,
+                        }),
+                        alternate: format.alternate,
+                        zero_pad: format.zero_pad,
+                        debug_hex: format.debug_hex.map(|s| match s {
+                            parse::DebugHex::Lower => FormatDebugHex::Lower,
+                            parse::DebugHex::Upper => FormatDebugHex::Upper,
+                        }),
                         precision,
                         width,
                     },
@@ -850,7 +857,7 @@ fn expand_format_args_impl<'cx>(
     match parse_args(ecx, sp, tts) {
         Ok((efmt, args)) => {
             if let Ok(format_args) = make_format_args(ecx, efmt, args, nl) {
-                MacEager::expr(expand_parsed_format_args(ecx, format_args))
+                MacEager::expr(ecx.expr(sp, ExprKind::FormatArgs(P(format_args))))
             } else {
                 MacEager::expr(DummyResult::raw_expr(sp, true))
             }
diff --git a/compiler/rustc_builtin_macros/src/format/expand.rs b/compiler/rustc_builtin_macros/src/format/expand.rs
deleted file mode 100644
index 9dde5efcb..000000000
--- a/compiler/rustc_builtin_macros/src/format/expand.rs
+++ /dev/null
@@ -1,353 +0,0 @@
-use super::*;
-use rustc_ast as ast;
-use rustc_ast::visit::{self, Visitor};
-use rustc_ast::{BlockCheckMode, UnsafeSource};
-use rustc_data_structures::fx::FxIndexSet;
-use rustc_span::{sym, symbol::kw};
-
-#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)]
-enum ArgumentType {
-    Format(FormatTrait),
-    Usize,
-}
-
-fn make_argument(ecx: &ExtCtxt<'_>, sp: Span, arg: P<ast::Expr>, ty: ArgumentType) -> P<ast::Expr> {
-    // Generate:
-    //     ::core::fmt::ArgumentV1::new_…(arg)
-    use ArgumentType::*;
-    use FormatTrait::*;
-    ecx.expr_call_global(
-        sp,
-        ecx.std_path(&[
-            sym::fmt,
-            sym::ArgumentV1,
-            match ty {
-                Format(Display) => sym::new_display,
-                Format(Debug) => sym::new_debug,
-                Format(LowerExp) => sym::new_lower_exp,
-                Format(UpperExp) => sym::new_upper_exp,
-                Format(Octal) => sym::new_octal,
-                Format(Pointer) => sym::new_pointer,
-                Format(Binary) => sym::new_binary,
-                Format(LowerHex) => sym::new_lower_hex,
-                Format(UpperHex) => sym::new_upper_hex,
-                Usize => sym::from_usize,
-            },
-        ]),
-        vec![arg],
-    )
-}
-
-fn make_count(
-    ecx: &ExtCtxt<'_>,
-    sp: Span,
-    count: &Option<FormatCount>,
-    argmap: &mut FxIndexSet<(usize, ArgumentType)>,
-) -> P<ast::Expr> {
-    // Generate:
-    //     ::core::fmt::rt::v1::Count::…(…)
-    match count {
-        Some(FormatCount::Literal(n)) => ecx.expr_call_global(
-            sp,
-            ecx.std_path(&[sym::fmt, sym::rt, sym::v1, sym::Count, sym::Is]),
-            vec![ecx.expr_usize(sp, *n)],
-        ),
-        Some(FormatCount::Argument(arg)) => {
-            if let Ok(arg_index) = arg.index {
-                let (i, _) = argmap.insert_full((arg_index, ArgumentType::Usize));
-                ecx.expr_call_global(
-                    sp,
-                    ecx.std_path(&[sym::fmt, sym::rt, sym::v1, sym::Count, sym::Param]),
-                    vec![ecx.expr_usize(sp, i)],
-                )
-            } else {
-                DummyResult::raw_expr(sp, true)
-            }
-        }
-        None => ecx.expr_path(ecx.path_global(
-            sp,
-            ecx.std_path(&[sym::fmt, sym::rt, sym::v1, sym::Count, sym::Implied]),
-        )),
-    }
-}
-
-fn make_format_spec(
-    ecx: &ExtCtxt<'_>,
-    sp: Span,
-    placeholder: &FormatPlaceholder,
-    argmap: &mut FxIndexSet<(usize, ArgumentType)>,
-) -> P<ast::Expr> {
-    // Generate:
-    //     ::core::fmt::rt::v1::Argument {
-    //         position: 0usize,
-    //         format: ::core::fmt::rt::v1::FormatSpec {
-    //             fill: ' ',
-    //             align: ::core::fmt::rt::v1::Alignment::Unknown,
-    //             flags: 0u32,
-    //             precision: ::core::fmt::rt::v1::Count::Implied,
-    //             width: ::core::fmt::rt::v1::Count::Implied,
-    //         },
-    //     }
-    let position = match placeholder.argument.index {
-        Ok(arg_index) => {
-            let (i, _) =
-                argmap.insert_full((arg_index, ArgumentType::Format(placeholder.format_trait)));
-            ecx.expr_usize(sp, i)
-        }
-        Err(_) => DummyResult::raw_expr(sp, true),
-    };
-    let fill = ecx.expr_char(sp, placeholder.format_options.fill.unwrap_or(' '));
-    let align = ecx.expr_path(ecx.path_global(
-        sp,
-        ecx.std_path(&[
-            sym::fmt,
-            sym::rt,
-            sym::v1,
-            sym::Alignment,
-            match placeholder.format_options.alignment {
-                Some(FormatAlignment::Left) => sym::Left,
-                Some(FormatAlignment::Right) => sym::Right,
-                Some(FormatAlignment::Center) => sym::Center,
-                None => sym::Unknown,
-            },
-        ]),
-    ));
-    let flags = ecx.expr_u32(sp, placeholder.format_options.flags);
-    let prec = make_count(ecx, sp, &placeholder.format_options.precision, argmap);
-    let width = make_count(ecx, sp, &placeholder.format_options.width, argmap);
-    ecx.expr_struct(
-        sp,
-        ecx.path_global(sp, ecx.std_path(&[sym::fmt, sym::rt, sym::v1, sym::Argument])),
-        vec![
-            ecx.field_imm(sp, Ident::new(sym::position, sp), position),
-            ecx.field_imm(
-                sp,
-                Ident::new(sym::format, sp),
-                ecx.expr_struct(
-                    sp,
-                    ecx.path_global(
-                        sp,
-                        ecx.std_path(&[sym::fmt, sym::rt, sym::v1, sym::FormatSpec]),
-                    ),
-                    vec![
-                        ecx.field_imm(sp, Ident::new(sym::fill, sp), fill),
-                        ecx.field_imm(sp, Ident::new(sym::align, sp), align),
-                        ecx.field_imm(sp, Ident::new(sym::flags, sp), flags),
-                        ecx.field_imm(sp, Ident::new(sym::precision, sp), prec),
-                        ecx.field_imm(sp, Ident::new(sym::width, sp), width),
-                    ],
-                ),
-            ),
-        ],
-    )
-}
-
-pub fn expand_parsed_format_args(ecx: &mut ExtCtxt<'_>, fmt: FormatArgs) -> P<ast::Expr> {
-    let macsp = ecx.with_def_site_ctxt(ecx.call_site());
-
-    let lit_pieces = ecx.expr_array_ref(
-        fmt.span,
-        fmt.template
-            .iter()
-            .enumerate()
-            .filter_map(|(i, piece)| match piece {
-                &FormatArgsPiece::Literal(s) => Some(ecx.expr_str(fmt.span, s)),
-                &FormatArgsPiece::Placeholder(_) => {
-                    // Inject empty string before placeholders when not already preceded by a literal piece.
-                    if i == 0 || matches!(fmt.template[i - 1], FormatArgsPiece::Placeholder(_)) {
-                        Some(ecx.expr_str(fmt.span, kw::Empty))
-                    } else {
-                        None
-                    }
-                }
-            })
-            .collect(),
-    );
-
-    // Whether we'll use the `Arguments::new_v1_formatted` form (true),
-    // or the `Arguments::new_v1` form (false).
-    let mut use_format_options = false;
-
-    // Create a list of all _unique_ (argument, format trait) combinations.
-    // E.g. "{0} {0:x} {0} {1}" -> [(0, Display), (0, LowerHex), (1, Display)]
-    let mut argmap = FxIndexSet::default();
-    for piece in &fmt.template {
-        let FormatArgsPiece::Placeholder(placeholder) = piece else { continue };
-        if placeholder.format_options != Default::default() {
-            // Can't use basic form if there's any formatting options.
-            use_format_options = true;
-        }
-        if let Ok(index) = placeholder.argument.index {
-            if !argmap.insert((index, ArgumentType::Format(placeholder.format_trait))) {
-                // Duplicate (argument, format trait) combination,
-                // which we'll only put once in the args array.
-                use_format_options = true;
-            }
-        }
-    }
-
-    let format_options = use_format_options.then(|| {
-        // Generate:
-        //     &[format_spec_0, format_spec_1, format_spec_2]
-        ecx.expr_array_ref(
-            macsp,
-            fmt.template
-                .iter()
-                .filter_map(|piece| {
-                    let FormatArgsPiece::Placeholder(placeholder) = piece else { return None };
-                    Some(make_format_spec(ecx, macsp, placeholder, &mut argmap))
-                })
-                .collect(),
-        )
-    });
-
-    let arguments = fmt.arguments.into_vec();
-
-    // If the args array contains exactly all the original arguments once,
-    // in order, we can use a simple array instead of a `match` construction.
-    // However, if there's a yield point in any argument except the first one,
-    // we don't do this, because an ArgumentV1 cannot be kept across yield points.
-    let use_simple_array = argmap.len() == arguments.len()
-        && argmap.iter().enumerate().all(|(i, &(j, _))| i == j)
-        && arguments.iter().skip(1).all(|arg| !may_contain_yield_point(&arg.expr));
-
-    let args = if use_simple_array {
-        // Generate:
-        //     &[
-        //         ::core::fmt::ArgumentV1::new_display(&arg0),
-        //         ::core::fmt::ArgumentV1::new_lower_hex(&arg1),
-        //         ::core::fmt::ArgumentV1::new_debug(&arg2),
-        //     ]
-        ecx.expr_array_ref(
-            macsp,
-            arguments
-                .into_iter()
-                .zip(argmap)
-                .map(|(arg, (_, ty))| {
-                    let sp = arg.expr.span.with_ctxt(macsp.ctxt());
-                    make_argument(ecx, sp, ecx.expr_addr_of(sp, arg.expr), ty)
-                })
-                .collect(),
-        )
-    } else {
-        // Generate:
-        //     match (&arg0, &arg1, &arg2) {
-        //         args => &[
-        //             ::core::fmt::ArgumentV1::new_display(args.0),
-        //             ::core::fmt::ArgumentV1::new_lower_hex(args.1),
-        //             ::core::fmt::ArgumentV1::new_debug(args.0),
-        //         ]
-        //     }
-        let args_ident = Ident::new(sym::args, macsp);
-        let args = argmap
-            .iter()
-            .map(|&(arg_index, ty)| {
-                if let Some(arg) = arguments.get(arg_index) {
-                    let sp = arg.expr.span.with_ctxt(macsp.ctxt());
-                    make_argument(
-                        ecx,
-                        sp,
-                        ecx.expr_field(
-                            sp,
-                            ecx.expr_ident(macsp, args_ident),
-                            Ident::new(sym::integer(arg_index), macsp),
-                        ),
-                        ty,
-                    )
-                } else {
-                    DummyResult::raw_expr(macsp, true)
-                }
-            })
-            .collect();
-        ecx.expr_addr_of(
-            macsp,
-            ecx.expr_match(
-                macsp,
-                ecx.expr_tuple(
-                    macsp,
-                    arguments
-                        .into_iter()
-                        .map(|arg| {
-                            ecx.expr_addr_of(arg.expr.span.with_ctxt(macsp.ctxt()), arg.expr)
-                        })
-                        .collect(),
-                ),
-                vec![ecx.arm(macsp, ecx.pat_ident(macsp, args_ident), ecx.expr_array(macsp, args))],
-            ),
-        )
-    };
-
-    if let Some(format_options) = format_options {
-        // Generate:
-        //     ::core::fmt::Arguments::new_v1_formatted(
-        //         lit_pieces,
-        //         args,
-        //         format_options,
-        //         unsafe { ::core::fmt::UnsafeArg::new() }
-        //     )
-        ecx.expr_call_global(
-            macsp,
-            ecx.std_path(&[sym::fmt, sym::Arguments, sym::new_v1_formatted]),
-            vec![
-                lit_pieces,
-                args,
-                format_options,
-                ecx.expr_block(P(ast::Block {
-                    stmts: vec![ecx.stmt_expr(ecx.expr_call_global(
-                        macsp,
-                        ecx.std_path(&[sym::fmt, sym::UnsafeArg, sym::new]),
-                        Vec::new(),
-                    ))],
-                    id: ast::DUMMY_NODE_ID,
-                    rules: BlockCheckMode::Unsafe(UnsafeSource::CompilerGenerated),
-                    span: macsp,
-                    tokens: None,
-                    could_be_bare_literal: false,
-                })),
-            ],
-        )
-    } else {
-        // Generate:
-        //     ::core::fmt::Arguments::new_v1(
-        //         lit_pieces,
-        //         args,
-        //     )
-        ecx.expr_call_global(
-            macsp,
-            ecx.std_path(&[sym::fmt, sym::Arguments, sym::new_v1]),
-            vec![lit_pieces, args],
-        )
-    }
-}
-
-fn may_contain_yield_point(e: &ast::Expr) -> bool {
-    struct MayContainYieldPoint(bool);
-
-    impl Visitor<'_> for MayContainYieldPoint {
-        fn visit_expr(&mut self, e: &ast::Expr) {
-            if let ast::ExprKind::Await(_) | ast::ExprKind::Yield(_) = e.kind {
-                self.0 = true;
-            } else {
-                visit::walk_expr(self, e);
-            }
-        }
-
-        fn visit_mac_call(&mut self, _: &ast::MacCall) {
-            self.0 = true;
-        }
-
-        fn visit_attribute(&mut self, _: &ast::Attribute) {
-            // Conservatively assume this may be a proc macro attribute in
-            // expression position.
-            self.0 = true;
-        }
-
-        fn visit_item(&mut self, _: &ast::Item) {
-            // Do not recurse into nested items.
-        }
-    }
-
-    let mut visitor = MayContainYieldPoint(false);
-    visitor.visit_expr(e);
-    visitor.0
-}
diff --git a/compiler/rustc_builtin_macros/src/global_allocator.rs b/compiler/rustc_builtin_macros/src/global_allocator.rs
index f8761653b..41b51bae7 100644
--- a/compiler/rustc_builtin_macros/src/global_allocator.rs
+++ b/compiler/rustc_builtin_macros/src/global_allocator.rs
@@ -9,7 +9,7 @@ use rustc_ast::{Fn, ItemKind, Mutability, Stmt, Ty, TyKind, Unsafe};
 use rustc_expand::base::{Annotatable, ExtCtxt};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::Span;
-use thin_vec::thin_vec;
+use thin_vec::{thin_vec, ThinVec};
 
 pub fn expand(
     ecx: &mut ExtCtxt<'_>,
@@ -47,7 +47,7 @@ pub fn expand(
     let stmts = ALLOCATOR_METHODS.iter().map(|method| f.allocator_fn(method)).collect();
 
     // Generate anonymous constant serving as container for the allocator methods.
-    let const_ty = ecx.ty(ty_span, TyKind::Tup(Vec::new()));
+    let const_ty = ecx.ty(ty_span, TyKind::Tup(ThinVec::new()));
     let const_body = ecx.expr_block(ecx.block(span, stmts));
     let const_item = ecx.item_const(span, Ident::new(kw::Underscore, span), const_ty, const_body);
     let const_item = if is_stmt {
@@ -70,7 +70,7 @@ struct AllocFnFactory<'a, 'b> {
 
 impl AllocFnFactory<'_, '_> {
     fn allocator_fn(&self, method: &AllocatorMethod) -> Stmt {
-        let mut abi_args = Vec::new();
+        let mut abi_args = ThinVec::new();
         let mut i = 0;
         let mut mk = || {
             let name = Ident::from_str_and_span(&format!("arg{}", i), self.span);
@@ -99,7 +99,7 @@ impl AllocFnFactory<'_, '_> {
         self.cx.stmt_item(self.ty_span, item)
     }
 
-    fn call_allocator(&self, method: Symbol, mut args: Vec<P<Expr>>) -> P<Expr> {
+    fn call_allocator(&self, method: Symbol, mut args: ThinVec<P<Expr>>) -> P<Expr> {
         let method = self.cx.std_path(&[sym::alloc, sym::GlobalAlloc, method]);
         let method = self.cx.expr_path(self.cx.path(self.ty_span, method));
         let allocator = self.cx.path_ident(self.ty_span, self.global);
@@ -117,7 +117,7 @@ impl AllocFnFactory<'_, '_> {
     fn arg_ty(
         &self,
         ty: &AllocatorTy,
-        args: &mut Vec<Param>,
+        args: &mut ThinVec<Param>,
         ident: &mut dyn FnMut() -> Ident,
     ) -> P<Expr> {
         match *ty {
@@ -134,7 +134,7 @@ impl AllocFnFactory<'_, '_> {
                 let layout_new = self.cx.expr_path(self.cx.path(self.span, layout_new));
                 let size = self.cx.expr_ident(self.span, size);
                 let align = self.cx.expr_ident(self.span, align);
-                let layout = self.cx.expr_call(self.span, layout_new, vec![size, align]);
+                let layout = self.cx.expr_call(self.span, layout_new, thin_vec![size, align]);
                 layout
             }
 
@@ -168,7 +168,7 @@ impl AllocFnFactory<'_, '_> {
                 (self.ptr_u8(), expr)
             }
 
-            AllocatorTy::Unit => (self.cx.ty(self.span, TyKind::Tup(Vec::new())), expr),
+            AllocatorTy::Unit => (self.cx.ty(self.span, TyKind::Tup(ThinVec::new())), expr),
 
             AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => {
                 panic!("can't convert `AllocatorTy` to an output")
diff --git a/compiler/rustc_builtin_macros/src/lib.rs b/compiler/rustc_builtin_macros/src/lib.rs
index 75cfac723..8afb6e560 100644
--- a/compiler/rustc_builtin_macros/src/lib.rs
+++ b/compiler/rustc_builtin_macros/src/lib.rs
@@ -21,8 +21,10 @@ extern crate tracing;
 
 use crate::deriving::*;
 
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
 use rustc_expand::base::{MacroExpanderFn, ResolverExpand, SyntaxExtensionKind};
 use rustc_expand::proc_macro::BangProcMacro;
+use rustc_macros::fluent_messages;
 use rustc_span::symbol::sym;
 
 mod alloc_error_handler;
@@ -54,6 +56,8 @@ pub mod proc_macro_harness;
 pub mod standard_library_imports;
 pub mod test_harness;
 
+fluent_messages! { "../locales/en-US.ftl" }
+
 pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) {
     let mut register = |name, kind| resolver.register_builtin_macro(name, kind);
     macro register_bang($($name:ident: $f:expr,)*) {
diff --git a/compiler/rustc_builtin_macros/src/proc_macro_harness.rs b/compiler/rustc_builtin_macros/src/proc_macro_harness.rs
index 21c8caa65..bc513607d 100644
--- a/compiler/rustc_builtin_macros/src/proc_macro_harness.rs
+++ b/compiler/rustc_builtin_macros/src/proc_macro_harness.rs
@@ -11,6 +11,7 @@ use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
 use smallvec::smallvec;
 use std::mem;
+use thin_vec::{thin_vec, ThinVec};
 
 struct ProcMacroDerive {
     id: NodeId,
@@ -314,11 +315,14 @@ fn mk_decls(cx: &mut ExtCtxt<'_>, macros: &[ProcMacro]) -> P<ast::Item> {
                     cx.expr_call(
                         span,
                         proc_macro_ty_method_path(cx, custom_derive),
-                        vec![
+                        thin_vec![
                             cx.expr_str(span, cd.trait_name),
                             cx.expr_array_ref(
                                 span,
-                                cd.attrs.iter().map(|&s| cx.expr_str(span, s)).collect::<Vec<_>>(),
+                                cd.attrs
+                                    .iter()
+                                    .map(|&s| cx.expr_str(span, s))
+                                    .collect::<ThinVec<_>>(),
                             ),
                             local_path(cx, cd.function_name),
                         ],
@@ -335,7 +339,7 @@ fn mk_decls(cx: &mut ExtCtxt<'_>, macros: &[ProcMacro]) -> P<ast::Item> {
                     cx.expr_call(
                         span,
                         proc_macro_ty_method_path(cx, ident),
-                        vec![
+                        thin_vec![
                             cx.expr_str(span, ca.function_name.name),
                             local_path(cx, ca.function_name),
                         ],
@@ -371,13 +375,13 @@ fn mk_decls(cx: &mut ExtCtxt<'_>, macros: &[ProcMacro]) -> P<ast::Item> {
         });
 
     let block = cx.expr_block(
-        cx.block(span, vec![cx.stmt_item(span, krate), cx.stmt_item(span, decls_static)]),
+        cx.block(span, thin_vec![cx.stmt_item(span, krate), cx.stmt_item(span, decls_static)]),
     );
 
     let anon_constant = cx.item_const(
         span,
         Ident::new(kw::Underscore, span),
-        cx.ty(span, ast::TyKind::Tup(Vec::new())),
+        cx.ty(span, ast::TyKind::Tup(ThinVec::new())),
         block,
     );
 
diff --git a/compiler/rustc_builtin_macros/src/standard_library_imports.rs b/compiler/rustc_builtin_macros/src/standard_library_imports.rs
index f73f20c84..e67c0dba6 100644
--- a/compiler/rustc_builtin_macros/src/standard_library_imports.rs
+++ b/compiler/rustc_builtin_macros/src/standard_library_imports.rs
@@ -62,7 +62,7 @@ pub fn inject(
     // the one with the prelude.
     let name = names[0];
 
-    let root = (edition == Edition2015).then(|| kw::PathRoot);
+    let root = (edition == Edition2015).then_some(kw::PathRoot);
 
     let import_path = root
         .iter()
diff --git a/compiler/rustc_builtin_macros/src/test.rs b/compiler/rustc_builtin_macros/src/test.rs
index 729ae4071..e02c7e6c0 100644
--- a/compiler/rustc_builtin_macros/src/test.rs
+++ b/compiler/rustc_builtin_macros/src/test.rs
@@ -10,7 +10,7 @@ use rustc_session::Session;
 use rustc_span::symbol::{sym, Ident, Symbol};
 use rustc_span::Span;
 use std::iter;
-use thin_vec::thin_vec;
+use thin_vec::{thin_vec, ThinVec};
 
 /// #[test_case] is used by custom test authors to mark tests
 /// When building for test, it needs to make the item public and gensym the name
@@ -179,19 +179,19 @@ pub fn expand_test_or_bench(
         cx.expr_call(
             sp,
             cx.expr_path(test_path("StaticBenchFn")),
-            vec![
+            thin_vec![
                 // |b| self::test::assert_test_result(
                 cx.lambda1(
                     sp,
                     cx.expr_call(
                         sp,
                         cx.expr_path(test_path("assert_test_result")),
-                        vec![
+                        thin_vec![
                             // super::$test_fn(b)
                             cx.expr_call(
                                 ret_ty_sp,
                                 cx.expr_path(cx.path(sp, vec![item.ident])),
-                                vec![cx.expr_ident(sp, b)],
+                                thin_vec![cx.expr_ident(sp, b)],
                             ),
                         ],
                     ),
@@ -203,7 +203,7 @@ pub fn expand_test_or_bench(
         cx.expr_call(
             sp,
             cx.expr_path(test_path("StaticTestFn")),
-            vec![
+            thin_vec![
                 // || {
                 cx.lambda0(
                     sp,
@@ -211,12 +211,12 @@ pub fn expand_test_or_bench(
                     cx.expr_call(
                         sp,
                         cx.expr_path(test_path("assert_test_result")),
-                        vec![
+                        thin_vec![
                             // $test_fn()
                             cx.expr_call(
                                 ret_ty_sp,
                                 cx.expr_path(cx.path(sp, vec![item.ident])),
-                                vec![],
+                                ThinVec::new(),
                             ), // )
                         ],
                     ), // }
@@ -249,21 +249,21 @@ pub fn expand_test_or_bench(
                 cx.expr_struct(
                     sp,
                     test_path("TestDescAndFn"),
-                    vec![
+                    thin_vec![
                         // desc: test::TestDesc {
                         field(
                             "desc",
                             cx.expr_struct(
                                 sp,
                                 test_path("TestDesc"),
-                                vec![
+                                thin_vec![
                                     // name: "path::to::test"
                                     field(
                                         "name",
                                         cx.expr_call(
                                             sp,
                                             cx.expr_path(test_path("StaticTestName")),
-                                            vec![cx.expr_str(sp, test_path_symbol)],
+                                            thin_vec![cx.expr_str(sp, test_path_symbol)],
                                         ),
                                     ),
                                     // ignore: true | false
@@ -300,7 +300,7 @@ pub fn expand_test_or_bench(
                                             ShouldPanic::Yes(Some(sym)) => cx.expr_call(
                                                 sp,
                                                 cx.expr_path(should_panic_path("YesWithMessage")),
-                                                vec![cx.expr_str(sp, sym)],
+                                                thin_vec![cx.expr_str(sp, sym)],
                                             ),
                                         },
                                     ),
diff --git a/compiler/rustc_builtin_macros/src/test_harness.rs b/compiler/rustc_builtin_macros/src/test_harness.rs
index ad8871080..d8e3db9e8 100644
--- a/compiler/rustc_builtin_macros/src/test_harness.rs
+++ b/compiler/rustc_builtin_macros/src/test_harness.rs
@@ -14,7 +14,8 @@ use rustc_span::symbol::{sym, Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
 use rustc_target::spec::PanicStrategy;
 use smallvec::{smallvec, SmallVec};
-use thin_vec::thin_vec;
+use thin_vec::{thin_vec, ThinVec};
+use tracing::debug;
 
 use std::{iter, mem};
 
@@ -299,7 +300,7 @@ fn mk_main(cx: &mut TestCtxt<'_>) -> P<ast::Item> {
     test_runner.span = sp;
 
     let test_main_path_expr = ecx.expr_path(test_runner);
-    let call_test_main = ecx.expr_call(sp, test_main_path_expr, vec![mk_tests_slice(cx, sp)]);
+    let call_test_main = ecx.expr_call(sp, test_main_path_expr, thin_vec![mk_tests_slice(cx, sp)]);
     let call_test_main = ecx.stmt_expr(call_test_main);
 
     // extern crate test
@@ -312,16 +313,16 @@ fn mk_main(cx: &mut TestCtxt<'_>) -> P<ast::Item> {
     let main_attr = ecx.attr_word(sym::rustc_main, sp);
 
     // pub fn main() { ... }
-    let main_ret_ty = ecx.ty(sp, ast::TyKind::Tup(vec![]));
+    let main_ret_ty = ecx.ty(sp, ast::TyKind::Tup(ThinVec::new()));
 
     // If no test runner is provided we need to import the test crate
     let main_body = if cx.test_runner.is_none() {
-        ecx.block(sp, vec![test_extern_stmt, call_test_main])
+        ecx.block(sp, thin_vec![test_extern_stmt, call_test_main])
     } else {
-        ecx.block(sp, vec![call_test_main])
+        ecx.block(sp, thin_vec![call_test_main])
     };
 
-    let decl = ecx.fn_decl(vec![], ast::FnRetTy::Ty(main_ret_ty));
+    let decl = ecx.fn_decl(ThinVec::new(), ast::FnRetTy::Ty(main_ret_ty));
     let sig = ast::FnSig { decl, header: ast::FnHeader::default(), span: sp };
     let defaultness = ast::Defaultness::Final;
     let main = ast::ItemKind::Fn(Box::new(ast::Fn {
diff --git a/compiler/rustc_codegen_cranelift/.cirrus.yml b/compiler/rustc_codegen_cranelift/.cirrus.yml
index d627c2ee0..7886cae42 100644
--- a/compiler/rustc_codegen_cranelift/.cirrus.yml
+++ b/compiler/rustc_codegen_cranelift/.cirrus.yml
@@ -1,13 +1,11 @@
 task:
   name: freebsd
   freebsd_instance:
-    image: freebsd-12-1-release-amd64
+    image: freebsd-13-1-release-amd64
   setup_rust_script:
     - pkg install -y curl git bash
     - curl https://sh.rustup.rs -sSf --output rustup.sh
     - sh rustup.sh --default-toolchain none -y --profile=minimal
-  cargo_bin_cache:
-    folder: ~/.cargo/bin
   target_cache:
     folder: target
   prepare_script:
@@ -15,9 +13,4 @@ task:
     - ./y.rs prepare
   test_script:
     - . $HOME/.cargo/env
-    - # Enable backtraces for easier debugging
-    - export RUST_BACKTRACE=1
-    - # Reduce amount of benchmark runs as they are slow
-    - export COMPILE_RUNS=2
-    - export RUN_RUNS=2
     - ./y.rs test
diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml
index a6bb12a66..9d3ed3ac5 100644
--- a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml
+++ b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml
@@ -21,15 +21,20 @@ jobs:
         cargo fmt --check
         rustfmt --check build_system/mod.rs
 
+
   build:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
 
+    defaults:
+      run:
+        shell: bash
+
     strategy:
       fail-fast: false
       matrix:
         include:
-          - os: ubuntu-20.04 # FIXME switch to ubuntu-22.04 once #1303 is fixed
+          - os: ubuntu-latest
             env:
               TARGET_TRIPLE: x86_64-unknown-linux-gnu
           - os: macos-latest
@@ -46,36 +51,31 @@ jobs:
           - os: ubuntu-latest
             env:
               TARGET_TRIPLE: s390x-unknown-linux-gnu
+          - os: windows-latest
+            env:
+              TARGET_TRIPLE: x86_64-pc-windows-msvc
+          - os: windows-latest
+            env:
+              TARGET_TRIPLE: x86_64-pc-windows-gnu
 
     steps:
     - uses: actions/checkout@v3
 
-    - name: Cache cargo installed crates
-      uses: actions/cache@v3
-      with:
-        path: ~/.cargo/bin
-        key: ${{ runner.os }}-cargo-installed-crates
-
-    - name: Cache cargo registry and index
-      uses: actions/cache@v3
-      with:
-        path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-        key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
-
     - name: Cache cargo target dir
       uses: actions/cache@v3
       with:
         path: build/cg_clif
-        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+        key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+
+    - name: Set MinGW as the default toolchain
+      if: matrix.os == 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
+      run: rustup set default-host x86_64-pc-windows-gnu
 
     - name: Install MinGW toolchain and wine
       if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
       run: |
         sudo apt-get update
         sudo apt-get install -y gcc-mingw-w64-x86-64 wine-stable
-        rustup target add x86_64-pc-windows-gnu
 
     - name: Install AArch64 toolchain and qemu
       if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'aarch64-unknown-linux-gnu'
@@ -89,6 +89,13 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y gcc-s390x-linux-gnu qemu-user
 
+    - name: Use sparse cargo registry
+      run: |
+        cat >> ~/.cargo/config.toml <<EOF
+        [unstable]
+        sparse-registry = true
+        EOF
+
     - name: Prepare dependencies
       run: ./y.rs prepare
 
@@ -104,49 +111,30 @@ jobs:
     - name: Test
       env:
         TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
-      run: |
-        # Enable backtraces for easier debugging
-        export RUST_BACKTRACE=1
-
-        # Reduce amount of benchmark runs as they are slow
-        export COMPILE_RUNS=2
-        export RUN_RUNS=2
-
-        # Enable extra checks
-        export CG_CLIF_ENABLE_VERIFIER=1
-
-        ./y.rs test
-
-    - name: Package prebuilt cg_clif
-      run: tar cvfJ cg_clif.tar.xz dist
+      run: ./y.rs test
 
-    - name: Upload prebuilt cg_clif
-      if: matrix.env.TARGET_TRIPLE != 'x86_64-pc-windows-gnu'
-      uses: actions/upload-artifact@v2
-      with:
-        name: cg_clif-${{ matrix.env.TARGET_TRIPLE }}
-        path: cg_clif.tar.xz
 
-    - name: Upload prebuilt cg_clif (cross compile)
-      if: matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
-      uses: actions/upload-artifact@v3
-      with:
-        name: cg_clif-${{ runner.os }}-cross-x86_64-mingw
-        path: cg_clif.tar.xz
-
-  windows:
+  abi_cafe:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
 
+    defaults:
+      run:
+        shell: bash
+
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         include:
-          # Native Windows build with MSVC
+          - os: ubuntu-latest
+            env:
+              TARGET_TRIPLE: x86_64-unknown-linux-gnu
+          - os: macos-latest
+            env:
+              TARGET_TRIPLE: x86_64-apple-darwin
           - os: windows-latest
             env:
               TARGET_TRIPLE: x86_64-pc-windows-msvc
-          # cross-compile from Windows to Windows MinGW
           - os: windows-latest
             env:
               TARGET_TRIPLE: x86_64-pc-windows-gnu
@@ -154,20 +142,6 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Cache cargo installed crates
-      uses: actions/cache@v3
-      with:
-        path: ~/.cargo/bin
-        key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-installed-crates
-
-    - name: Cache cargo registry and index
-      uses: actions/cache@v3
-      with:
-        path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-        key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
-
     - name: Cache cargo target dir
       uses: actions/cache@v3
       with:
@@ -178,50 +152,146 @@ jobs:
       if: matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
       run: rustup set default-host x86_64-pc-windows-gnu
 
-    - name: Prepare dependencies
+    - name: Use sparse cargo registry
       run: |
-        git config --global core.autocrlf false
-        rustc y.rs -o y.exe -g
-        ./y.exe prepare
+        cat >> ~/.cargo/config.toml <<EOF
+        [unstable]
+        sparse-registry = true
+        EOF
 
-    - name: Build without unstable features
-      env:
-        TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
-      # This is the config rust-lang/rust uses for builds
-      run: ./y.rs build --no-unstable-features
+    - name: Prepare dependencies
+      run: ./y.rs prepare
 
     - name: Build
       run: ./y.rs build --sysroot none
 
-    - name: Test
+    - name: Test abi-cafe
+      env:
+        TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }}
+      run: ./y.rs abi-cafe
+
+
+  bench:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v3
+      with:
+        path: build/cg_clif
+        key: ${{ runner.os }}-x86_64-unknown-linux-gnu-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+
+    - name: Cache cargo bin dir
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/bin
+        key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-bin-dir-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+
+    - name: Use sparse cargo registry
       run: |
-        # Enable backtraces for easier debugging
-        $Env:RUST_BACKTRACE=1
+        cat >> ~/.cargo/config.toml <<EOF
+        [unstable]
+        sparse-registry = true
+        EOF
 
-        # Reduce amount of benchmark runs as they are slow
-        $Env:COMPILE_RUNS=2
-        $Env:RUN_RUNS=2
+    - name: Install hyperfine
+      run: cargo install hyperfine || true
 
-        # Enable extra checks
-        $Env:CG_CLIF_ENABLE_VERIFIER=1
+    - name: Prepare dependencies
+      run: ./y.rs prepare
+
+    - name: Build
+      run: CI_OPT=1 ./y.rs build --sysroot none
 
-        # WIP Disable some tests
+    - name: Benchmark
+      run: CI_OPT=1 ./y.rs bench
 
-        # This fails due to some weird argument handling by hyperfine, not an actual regression
-        # more of a build system issue
-        (Get-Content config.txt) -replace '(bench.simple-raytracer)', '# $1' |  Out-File config.txt
 
-        # This fails with a different output than expected
-        (Get-Content config.txt) -replace '(test.regex-shootout-regex-dna)', '# $1' |  Out-File config.txt
+  dist:
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
 
-        ./y.exe test
+    defaults:
+      run:
+        shell: bash
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # FIXME update at some point in the future once most distros use a newer glibc
+          - os: ubuntu-20.04
+            env:
+              TARGET_TRIPLE: x86_64-unknown-linux-gnu
+          - os: macos-latest
+            env:
+              TARGET_TRIPLE: x86_64-apple-darwin
+          # cross-compile from Linux to Windows using mingw
+          - os: ubuntu-latest
+            env:
+              TARGET_TRIPLE: x86_64-pc-windows-gnu
+          - os: windows-latest
+            env:
+              TARGET_TRIPLE: x86_64-pc-windows-msvc
+          - os: windows-latest
+            env:
+              TARGET_TRIPLE: x86_64-pc-windows-gnu
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v3
+      with:
+        path: build/cg_clif
+        key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-dist-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
+
+    - name: Set MinGW as the default toolchain
+      if: matrix.os == 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
+      run: rustup set default-host x86_64-pc-windows-gnu
+
+    - name: Install MinGW toolchain and wine
+      if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y gcc-mingw-w64-x86-64 wine-stable
+
+    - name: Use sparse cargo registry
+      run: |
+        cat >> ~/.cargo/config.toml <<EOF
+        [unstable]
+        sparse-registry = true
+        EOF
+
+    - name: Prepare dependencies
+      run: ./y.rs prepare
+
+    - name: Build backend
+      run: CI_OPT=1 ./y.rs build --sysroot none
+
+    - name: Build sysroot
+      run: CI_OPT=1 ./y.rs build
 
     - name: Package prebuilt cg_clif
-      # don't use compression as xzip isn't supported by tar on windows and bzip2 hangs
-      run: tar cvf cg_clif.tar dist
+      run: tar cvfJ cg_clif.tar.xz dist
 
     - name: Upload prebuilt cg_clif
+      if: matrix.os == 'windows-latest' || matrix.env.TARGET_TRIPLE != 'x86_64-pc-windows-gnu'
       uses: actions/upload-artifact@v3
       with:
         name: cg_clif-${{ matrix.env.TARGET_TRIPLE }}
-        path: cg_clif.tar
+        path: cg_clif.tar.xz
+
+    - name: Upload prebuilt cg_clif (cross compile)
+      if: matrix.os != 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
+      uses: actions/upload-artifact@v3
+      with:
+        name: cg_clif-${{ runner.os }}-cross-x86_64-mingw
+        path: cg_clif.tar.xz
diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/nightly-cranelift.yml b/compiler/rustc_codegen_cranelift/.github/workflows/nightly-cranelift.yml
deleted file mode 100644
index d0d58d2a7..000000000
--- a/compiler/rustc_codegen_cranelift/.github/workflows/nightly-cranelift.yml
+++ /dev/null
@@ -1,59 +0,0 @@
-name: Test nightly Cranelift
-
-on:
-  push:
-  schedule:
-    - cron: '17 1 * * *' # At 01:17 UTC every day.
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Cache cargo installed crates
-      uses: actions/cache@v3
-      with:
-        path: ~/.cargo/bin
-        key: ubuntu-latest-cargo-installed-crates
-
-    - name: Prepare dependencies
-      run: |
-        git config --global user.email "user@example.com"
-        git config --global user.name "User"
-        ./y.rs prepare
-
-    - name: Patch Cranelift
-      run: |
-        sed -i 's/cranelift-codegen = { version = "\w*.\w*.\w*", features = \["unwind", "all-arch"\] }/cranelift-codegen = { git = "https:\/\/github.com\/bytecodealliance\/wasmtime.git", features = ["unwind", "all-arch"] }/' Cargo.toml
-        sed -i 's/cranelift-frontend = "\w*.\w*.\w*"/cranelift-frontend = { git = "https:\/\/github.com\/bytecodealliance\/wasmtime.git" }/' Cargo.toml
-        sed -i 's/cranelift-module = "\w*.\w*.\w*"/cranelift-module = { git = "https:\/\/github.com\/bytecodealliance\/wasmtime.git" }/' Cargo.toml
-        sed -i 's/cranelift-native = "\w*.\w*.\w*"/cranelift-native = { git = "https:\/\/github.com\/bytecodealliance\/wasmtime.git" }/' Cargo.toml
-        sed -i 's/cranelift-jit = { version = "\w*.\w*.\w*", optional = true }/cranelift-jit = { git = "https:\/\/github.com\/bytecodealliance\/wasmtime.git", optional = true }/' Cargo.toml
-        sed -i 's/cranelift-object = "\w*.\w*.\w*"/cranelift-object = { git = "https:\/\/github.com\/bytecodealliance\/wasmtime.git" }/' Cargo.toml
-
-        sed -i 's/object = { version = "0.27.0"/object = { version = "0.28.0"/' Cargo.toml
-
-        cat Cargo.toml
-
-    - name: Build without unstable features
-      # This is the config rust-lang/rust uses for builds
-      run: ./y.rs build --no-unstable-features
-
-    - name: Build
-      run: ./y.rs build --sysroot none
-    - name: Test
-      run: |
-        # Enable backtraces for easier debugging
-        export RUST_BACKTRACE=1
-
-        # Reduce amount of benchmark runs as they are slow
-        export COMPILE_RUNS=2
-        export RUN_RUNS=2
-
-        # Enable extra checks
-        export CG_CLIF_ENABLE_VERIFIER=1
-
-        ./test.sh
diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml b/compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml
index bef806318..5faa8f054 100644
--- a/compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml
+++ b/compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml
@@ -10,73 +10,45 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Cache cargo installed crates
-      uses: actions/cache@v3
-      with:
-        path: ~/.cargo/bin
-        key: ${{ runner.os }}-cargo-installed-crates
-
-    - name: Cache cargo registry and index
-      uses: actions/cache@v3
-      with:
-        path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-        key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
-
     - name: Cache cargo target dir
       uses: actions/cache@v3
       with:
         path: build/cg_clif
         key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
 
-    - name: Prepare dependencies
+    - name: Use sparse cargo registry
       run: |
-        git config --global user.email "user@example.com"
-        git config --global user.name "User"
-        ./y.rs prepare
+        cat >> ~/.cargo/config.toml <<EOF
+        [unstable]
+        sparse-registry = true
+        EOF
 
-    - name: Test
-      run: |
-        # Enable backtraces for easier debugging
-        export RUST_BACKTRACE=1
+    - name: Prepare dependencies
+      run: ./y.rs prepare
 
-        ./scripts/test_bootstrap.sh
+    - name: Test
+      run: ./scripts/test_bootstrap.sh
   rustc_test_suite:
     runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v3
 
-    - name: Cache cargo installed crates
-      uses: actions/cache@v3
-      with:
-        path: ~/.cargo/bin
-        key: ${{ runner.os }}-cargo-installed-crates
-
-    - name: Cache cargo registry and index
-      uses: actions/cache@v3
-      with:
-        path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-        key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
-
     - name: Cache cargo target dir
       uses: actions/cache@v3
       with:
         path: build/cg_clif
         key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
 
-    - name: Prepare dependencies
+    - name: Use sparse cargo registry
       run: |
-        git config --global user.email "user@example.com"
-        git config --global user.name "User"
-        ./y.rs prepare
+        cat >> ~/.cargo/config.toml <<EOF
+        [unstable]
+        sparse-registry = true
+        EOF
 
-    - name: Test
-      run: |
-        # Enable backtraces for easier debugging
-        export RUST_BACKTRACE=1
+    - name: Prepare dependencies
+      run: ./y.rs prepare
 
-        ./scripts/test_rustc_tests.sh
+    - name: Test
+      run: ./scripts/test_rustc_tests.sh
diff --git a/compiler/rustc_codegen_cranelift/.vscode/settings.json b/compiler/rustc_codegen_cranelift/.vscode/settings.json
index bc914e37d..7c8703cba 100644
--- a/compiler/rustc_codegen_cranelift/.vscode/settings.json
+++ b/compiler/rustc_codegen_cranelift/.vscode/settings.json
@@ -1,4 +1,6 @@
 {
+    "editor.formatOnSave": true,
+
     // source for rustc_* is not included in the rust-src component; disable the errors about this
     "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "unresolved-macro-call"],
     "rust-analyzer.imports.granularity.enforce": true,
@@ -30,7 +32,7 @@
             ]
         },
         {
-            "sysroot_src": "./build_sysroot/sysroot_src/library",
+            "sysroot_src": "./download/sysroot/sysroot_src/library",
             "crates": [
                 {
                     "root_module": "./example/std_example.rs",
diff --git a/compiler/rustc_codegen_cranelift/Cargo.lock b/compiler/rustc_codegen_cranelift/Cargo.lock
index e4d3e9ca5..50249ea1b 100644
--- a/compiler/rustc_codegen_cranelift/Cargo.lock
+++ b/compiler/rustc_codegen_cranelift/Cargo.lock
@@ -57,28 +57,28 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.90.1"
+version = "0.92.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b62c772976416112fa4484cbd688cb6fb35fd430005c1c586224fc014018abad"
+checksum = "2f3d54eab028f5805ae3b26fd60eca3f3a9cfb76b989d9bab173be3f61356cc3"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.90.1"
+version = "0.92.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b40ed2dd13c2ac7e24f88a3090c68ad3414eb1d066a95f8f1f7b3b819cb4e46"
+checksum = "2be1d5f2c3cca1efb691844bc1988b89c77291f13f778499a3f3c0cf49c0ed61"
 dependencies = [
  "arrayvec",
  "bumpalo",
  "cranelift-bforest",
  "cranelift-codegen-meta",
  "cranelift-codegen-shared",
- "cranelift-egraph",
  "cranelift-entity",
  "cranelift-isle",
  "gimli",
+ "hashbrown",
  "log",
  "regalloc2",
  "smallvec",
@@ -87,44 +87,30 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.90.1"
+version = "0.92.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb927a8f1c27c34ee3759b6b0ffa528d2330405d5cc4511f0cab33fe2279f4b5"
+checksum = "3f9b1b1089750ce4005893af7ee00bb08a2cf1c9779999c0f7164cbc8ad2e0d2"
 dependencies = [
  "cranelift-codegen-shared",
 ]
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.90.1"
+version = "0.92.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43dfa417b884a9ab488d95fd6b93b25e959321fe7bfd7a0a960ba5d7fb7ab927"
-
-[[package]]
-name = "cranelift-egraph"
-version = "0.90.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0a66b39785efd8513d2cca967ede56d6cc57c8d7986a595c7c47d0c78de8dce"
-dependencies = [
- "cranelift-entity",
- "fxhash",
- "hashbrown",
- "indexmap",
- "log",
- "smallvec",
-]
+checksum = "cc5fbaec51de47297fd7304986fd53c8c0030abbe69728a60d72e1c63559318d"
 
 [[package]]
 name = "cranelift-entity"
-version = "0.90.1"
+version = "0.92.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0637ffde963cb5d759bc4d454cfa364b6509e6c74cdaa21298add0ed9276f346"
+checksum = "dab984c94593f876090fae92e984bdcc74d9b1acf740ab5f79036001c65cba13"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.90.1"
+version = "0.92.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb72b8342685e850cb037350418f62cc4fc55d6c2eb9c7ca01b82f9f1a6f3d56"
+checksum = "6e0cb3102d21a2fe5f3210af608748ddd0cd09825ac12d42dc56ed5ed8725fe0"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -134,15 +120,15 @@ dependencies = [
 
 [[package]]
 name = "cranelift-isle"
-version = "0.90.1"
+version = "0.92.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "850579cb9e4b448f7c301f1e6e6cbad99abe3f1f1d878a4994cb66e33c6db8cd"
+checksum = "72101dd1f441d629735143c41e00b3428f9267738176983ef588ff43382af0a0"
 
 [[package]]
 name = "cranelift-jit"
-version = "0.90.1"
+version = "0.92.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9add822ad66dcbe152b5ab57de10240a2df4505099f2f6c27159acb711890bd4"
+checksum = "6557f8ce44d498777f2495aa58d9692a4a37d6f84aa445750d666cef770b6a5c"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -159,9 +145,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.90.1"
+version = "0.92.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "406b772626fc2664864cf947f3895a23b619895c7fff635f3622e2d857f4492f"
+checksum = "88807e1c0c47ec02fe433333ccbe56b480425418b1470e333205e11650697d72"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -169,9 +155,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.90.1"
+version = "0.92.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d0a279e5bcba3e0466c734d8d8eb6bfc1ad29e95c37f3e4955b492b5616335e"
+checksum = "c22b0d9fcbe3fc5a1af9e7021b44ce42b930bcefac446ce22e02e8f9a0d67120"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -180,9 +166,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.90.1"
+version = "0.92.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39793c550f0c1d7db96c2fc1324583670c8143befe6edbfbaf1c68aba53be983"
+checksum = "341375758d7c3fedc0b5315f552e6f0feac46baf87c450a15e9455ef47c2b261"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -317,9 +303,9 @@ checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
 
 [[package]]
 name = "regalloc2"
-version = "0.4.2"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91b2eab54204ea0117fe9a060537e0b07a4e72f7c7d182361ecc346cab2240e5"
+checksum = "300d4fbfb40c1c66a78ba3ddd41c1110247cf52f97b87d0f2fc9209bd49b030c"
 dependencies = [
  "fxhash",
  "log",
@@ -347,7 +333,6 @@ dependencies = [
  "cranelift-frontend",
  "cranelift-jit",
  "cranelift-module",
- "cranelift-native",
  "cranelift-object",
  "gimli",
  "indexmap",
@@ -396,9 +381,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasmtime-jit-icache-coherence"
-version = "2.0.1"
+version = "5.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6bbabb309c06cc238ee91b1455b748c45f0bdcab0dda2c2db85b0a1e69fcb66"
+checksum = "08fcba5ebd96da2a9f0747ab6337fe9788adfb3f63fa2c180520d665562d257e"
 dependencies = [
  "cfg-if",
  "libc",
@@ -429,43 +414,57 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
 name = "windows-sys"
-version = "0.36.1"
+version = "0.42.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
+checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
 dependencies = [
+ "windows_aarch64_gnullvm",
  "windows_aarch64_msvc",
  "windows_i686_gnu",
  "windows_i686_msvc",
  "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
  "windows_x86_64_msvc",
 ]
 
 [[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e"
+
+[[package]]
 name = "windows_aarch64_msvc"
-version = "0.36.1"
+version = "0.42.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
+checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.36.1"
+version = "0.42.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
+checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.36.1"
+version = "0.42.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
+checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.36.1"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.42.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
+checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.36.1"
+version = "0.42.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
+checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5"
diff --git a/compiler/rustc_codegen_cranelift/Cargo.toml b/compiler/rustc_codegen_cranelift/Cargo.toml
index 2b216ca07..34117c288 100644
--- a/compiler/rustc_codegen_cranelift/Cargo.toml
+++ b/compiler/rustc_codegen_cranelift/Cargo.toml
@@ -15,12 +15,14 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { version = "0.90.1", features = ["unwind", "all-arch"] }
-cranelift-frontend = "0.90.1"
-cranelift-module = "0.90.1"
-cranelift-native = "0.90.1"
-cranelift-jit = { version = "0.90.1", optional = true }
-cranelift-object = "0.90.1"
+cranelift-codegen = { version = "0.92", features = ["unwind", "all-arch"] }
+cranelift-frontend = { version = "0.92" }
+cranelift-module = { version = "0.92" }
+# NOTE vendored as src/cranelift_native.rs
+# FIXME revert back to the external crate with Cranelift 0.93
+#cranelift-native = { version = "0.92" }
+cranelift-jit = { version = "0.92", optional = true }
+cranelift-object = { version = "0.92" }
 target-lexicon = "0.12.0"
 gimli = { version = "0.26.0", default-features = false, features = ["write"]}
 object = { version = "0.29.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
diff --git a/compiler/rustc_codegen_cranelift/Readme.md b/compiler/rustc_codegen_cranelift/Readme.md
index 0e9c77244..b87a9dc51 100644
--- a/compiler/rustc_codegen_cranelift/Readme.md
+++ b/compiler/rustc_codegen_cranelift/Readme.md
@@ -8,9 +8,9 @@ If not please open an issue.
 ## Building and testing
 
 ```bash
-$ git clone https://github.com/bjorn3/rustc_codegen_cranelift.git
+$ git clone https://github.com/bjorn3/rustc_codegen_cranelift
 $ cd rustc_codegen_cranelift
-$ ./y.rs prepare # download and patch sysroot src and install hyperfine for benchmarking
+$ ./y.rs prepare
 $ ./y.rs build
 ```
 
@@ -20,13 +20,12 @@ To run the test suite replace the last command with:
 $ ./test.sh
 ```
 
-This will implicitly build cg_clif too. Both `y.rs build` and `test.sh` accept a `--debug` argument to
-build in debug mode.
+For more docs on how to build and test see [build_system/usage.txt](build_system/usage.txt) or the help message of `./y.rs`.
 
-Alternatively you can download a pre built version from [GHA]. It is listed in the artifacts section
+Alternatively you can download a pre built version from [Github Actions]. It is listed in the artifacts section
 of workflow runs. Unfortunately due to GHA restrictions you need to be logged in to access it.
 
-[GHA]: https://github.com/bjorn3/rustc_codegen_cranelift/actions?query=branch%3Amaster+event%3Apush+is%3Asuccess
+[Github Actions]: https://github.com/bjorn3/rustc_codegen_cranelift/actions?query=branch%3Amaster+event%3Apush+is%3Asuccess
 
 ## Usage
 
@@ -53,7 +52,8 @@ configuration options.
 
 * Inline assembly ([no cranelift support](https://github.com/bytecodealliance/wasmtime/issues/1041))
     * On UNIX there is support for invoking an external assembler for `global_asm!` and `asm!`.
-* SIMD ([tracked here](https://github.com/bjorn3/rustc_codegen_cranelift/issues/171), some basic things work)
+* SIMD ([tracked here](https://github.com/bjorn3/rustc_codegen_cranelift/issues/171), `std::simd` fully works, `std::arch` is partially supported)
+* Unwinding on panics ([no cranelift support](https://github.com/bytecodealliance/wasmtime/issues/1677), `-Cpanic=abort` is enabled by default)
 
 ## License
 
diff --git a/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock
index bba321053..b7e0b68a2 100644
--- a/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock
+++ b/compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock
@@ -34,9 +34,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.77"
+version = "1.0.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4"
+checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
 
 [[package]]
 name = "cfg-if"
@@ -50,9 +50,9 @@ dependencies = [
 
 [[package]]
 name = "compiler_builtins"
-version = "0.1.85"
+version = "0.1.86"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13e81c6cd7ab79f51a0c927d22858d61ad12bd0b3865f0b13ece02a4486aeabb"
+checksum = "5dae98c88e576098d7ab13ebcb40cc43e5114b2beafe61a87cda9200649ff205"
 dependencies = [
  "rustc-std-workspace-core",
 ]
@@ -129,9 +129,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.138"
+version = "0.2.139"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8"
+checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
 dependencies = [
  "rustc-std-workspace-core",
 ]
diff --git a/compiler/rustc_codegen_cranelift/build_system/abi_cafe.rs b/compiler/rustc_codegen_cranelift/build_system/abi_cafe.rs
index a081fdaa1..0da27f529 100644
--- a/compiler/rustc_codegen_cranelift/build_system/abi_cafe.rs
+++ b/compiler/rustc_codegen_cranelift/build_system/abi_cafe.rs
@@ -1,13 +1,12 @@
 use std::path::Path;
 
 use super::build_sysroot;
-use super::config;
 use super::path::Dirs;
 use super::prepare::GitRepo;
 use super::utils::{spawn_and_wait, CargoProject, Compiler};
 use super::SysrootKind;
 
-pub(crate) static ABI_CAFE_REPO: GitRepo =
+static ABI_CAFE_REPO: GitRepo =
     GitRepo::github("Gankra", "abi-cafe", "4c6dc8c9c687e2b3a760ff2176ce236872b37212", "abi-cafe");
 
 static ABI_CAFE: CargoProject = CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe");
@@ -17,18 +16,10 @@ pub(crate) fn run(
     sysroot_kind: SysrootKind,
     dirs: &Dirs,
     cg_clif_dylib: &Path,
-    host_triple: &str,
-    target_triple: &str,
+    bootstrap_host_compiler: &Compiler,
 ) {
-    if !config::get_bool("testsuite.abi-cafe") {
-        eprintln!("[SKIP] abi-cafe");
-        return;
-    }
-
-    if host_triple != target_triple {
-        eprintln!("[SKIP] abi-cafe (cross-compilation not supported)");
-        return;
-    }
+    ABI_CAFE_REPO.fetch(dirs);
+    spawn_and_wait(ABI_CAFE.fetch("cargo", &bootstrap_host_compiler.rustc, dirs));
 
     eprintln!("Building sysroot for abi-cafe");
     build_sysroot::build_sysroot(
@@ -36,15 +27,15 @@ pub(crate) fn run(
         channel,
         sysroot_kind,
         cg_clif_dylib,
-        host_triple,
-        target_triple,
+        bootstrap_host_compiler,
+        bootstrap_host_compiler.triple.clone(),
     );
 
     eprintln!("Running abi-cafe");
 
     let pairs = ["rustc_calls_cgclif", "cgclif_calls_rustc", "cgclif_calls_cc", "cc_calls_cgclif"];
 
-    let mut cmd = ABI_CAFE.run(&Compiler::host(), dirs);
+    let mut cmd = ABI_CAFE.run(bootstrap_host_compiler, dirs);
     cmd.arg("--");
     cmd.arg("--pairs");
     cmd.args(pairs);
diff --git a/compiler/rustc_codegen_cranelift/build_system/bench.rs b/compiler/rustc_codegen_cranelift/build_system/bench.rs
new file mode 100644
index 000000000..a9a851d0a
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/build_system/bench.rs
@@ -0,0 +1,103 @@
+use std::env;
+use std::fs;
+use std::path::Path;
+
+use super::path::{Dirs, RelPath};
+use super::prepare::GitRepo;
+use super::rustc_info::get_file_name;
+use super::utils::{hyperfine_command, spawn_and_wait, CargoProject, Compiler};
+
+static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
+    "ebobby",
+    "simple-raytracer",
+    "804a7a21b9e673a482797aa289a18ed480e4d813",
+    "<none>",
+);
+
+// Use a separate target dir for the initial LLVM build to reduce unnecessary recompiles
+static SIMPLE_RAYTRACER_LLVM: CargoProject =
+    CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer_llvm");
+
+static SIMPLE_RAYTRACER: CargoProject =
+    CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer");
+
+pub(crate) fn benchmark(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
+    benchmark_simple_raytracer(dirs, bootstrap_host_compiler);
+}
+
+fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
+    if std::process::Command::new("hyperfine").output().is_err() {
+        eprintln!("Hyperfine not installed");
+        eprintln!("Hint: Try `cargo install hyperfine` to install hyperfine");
+        std::process::exit(1);
+    }
+
+    if !SIMPLE_RAYTRACER_REPO.source_dir().to_path(dirs).exists() {
+        SIMPLE_RAYTRACER_REPO.fetch(dirs);
+        spawn_and_wait(SIMPLE_RAYTRACER.fetch(
+            &bootstrap_host_compiler.cargo,
+            &bootstrap_host_compiler.rustc,
+            dirs,
+        ));
+    }
+
+    eprintln!("[LLVM BUILD] simple-raytracer");
+    let build_cmd = SIMPLE_RAYTRACER_LLVM.build(bootstrap_host_compiler, dirs);
+    spawn_and_wait(build_cmd);
+    fs::copy(
+        SIMPLE_RAYTRACER_LLVM
+            .target_dir(dirs)
+            .join(&bootstrap_host_compiler.triple)
+            .join("debug")
+            .join(get_file_name("main", "bin")),
+        RelPath::BUILD.to_path(dirs).join(get_file_name("raytracer_cg_llvm", "bin")),
+    )
+    .unwrap();
+
+    let bench_runs = env::var("BENCH_RUNS").unwrap_or_else(|_| "10".to_string()).parse().unwrap();
+
+    eprintln!("[BENCH COMPILE] ebobby/simple-raytracer");
+    let cargo_clif =
+        RelPath::DIST.to_path(dirs).join(get_file_name("cargo_clif", "bin").replace('_', "-"));
+    let manifest_path = SIMPLE_RAYTRACER.manifest_path(dirs);
+    let target_dir = SIMPLE_RAYTRACER.target_dir(dirs);
+
+    let clean_cmd = format!(
+        "RUSTC=rustc cargo clean --manifest-path {manifest_path} --target-dir {target_dir}",
+        manifest_path = manifest_path.display(),
+        target_dir = target_dir.display(),
+    );
+    let llvm_build_cmd = format!(
+        "RUSTC=rustc cargo build --manifest-path {manifest_path} --target-dir {target_dir}",
+        manifest_path = manifest_path.display(),
+        target_dir = target_dir.display(),
+    );
+    let clif_build_cmd = format!(
+        "RUSTC=rustc {cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir}",
+        cargo_clif = cargo_clif.display(),
+        manifest_path = manifest_path.display(),
+        target_dir = target_dir.display(),
+    );
+
+    let bench_compile =
+        hyperfine_command(1, bench_runs, Some(&clean_cmd), &llvm_build_cmd, &clif_build_cmd);
+
+    spawn_and_wait(bench_compile);
+
+    eprintln!("[BENCH RUN] ebobby/simple-raytracer");
+    fs::copy(
+        target_dir.join("debug").join(get_file_name("main", "bin")),
+        RelPath::BUILD.to_path(dirs).join(get_file_name("raytracer_cg_clif", "bin")),
+    )
+    .unwrap();
+
+    let mut bench_run = hyperfine_command(
+        0,
+        bench_runs,
+        None,
+        Path::new(".").join(get_file_name("raytracer_cg_llvm", "bin")).to_str().unwrap(),
+        Path::new(".").join(get_file_name("raytracer_cg_clif", "bin")).to_str().unwrap(),
+    );
+    bench_run.current_dir(RelPath::BUILD.to_path(dirs));
+    spawn_and_wait(bench_run);
+}
diff --git a/compiler/rustc_codegen_cranelift/build_system/build_backend.rs b/compiler/rustc_codegen_cranelift/build_system/build_backend.rs
index fde8ef424..4b740fa2d 100644
--- a/compiler/rustc_codegen_cranelift/build_system/build_backend.rs
+++ b/compiler/rustc_codegen_cranelift/build_system/build_backend.rs
@@ -3,17 +3,17 @@ use std::path::PathBuf;
 
 use super::path::{Dirs, RelPath};
 use super::rustc_info::get_file_name;
-use super::utils::{is_ci, CargoProject, Compiler};
+use super::utils::{is_ci, is_ci_opt, CargoProject, Compiler};
 
-static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif");
+pub(crate) static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif");
 
 pub(crate) fn build_backend(
     dirs: &Dirs,
     channel: &str,
-    host_triple: &str,
+    bootstrap_host_compiler: &Compiler,
     use_unstable_features: bool,
 ) -> PathBuf {
-    let mut cmd = CG_CLIF.build(&Compiler::host(), dirs);
+    let mut cmd = CG_CLIF.build(&bootstrap_host_compiler, dirs);
 
     cmd.env("CARGO_BUILD_INCREMENTAL", "true"); // Force incr comp even in release mode
 
@@ -25,6 +25,10 @@ pub(crate) fn build_backend(
 
         // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway
         cmd.env("CARGO_BUILD_INCREMENTAL", "false");
+
+        if !is_ci_opt() {
+            cmd.env("CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS", "true");
+        }
     }
 
     if use_unstable_features {
@@ -46,7 +50,7 @@ pub(crate) fn build_backend(
 
     CG_CLIF
         .target_dir(dirs)
-        .join(host_triple)
+        .join(&bootstrap_host_compiler.triple)
         .join(channel)
         .join(get_file_name("rustc_codegen_cranelift", "dylib"))
 }
diff --git a/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs b/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs
index cbbf09b9b..2e04f2c68 100644
--- a/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs
+++ b/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs
@@ -1,31 +1,32 @@
 use std::fs;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::process::{self, Command};
 
 use super::path::{Dirs, RelPath};
-use super::rustc_info::{get_file_name, get_rustc_version, get_wrapper_file_name};
-use super::utils::{spawn_and_wait, try_hard_link, CargoProject, Compiler};
+use super::rustc_info::{get_file_name, get_rustc_version, get_toolchain_name};
+use super::utils::{remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler};
 use super::SysrootKind;
 
 static DIST_DIR: RelPath = RelPath::DIST;
 static BIN_DIR: RelPath = RelPath::DIST.join("bin");
 static LIB_DIR: RelPath = RelPath::DIST.join("lib");
-static RUSTLIB_DIR: RelPath = LIB_DIR.join("rustlib");
 
 pub(crate) fn build_sysroot(
     dirs: &Dirs,
     channel: &str,
     sysroot_kind: SysrootKind,
     cg_clif_dylib_src: &Path,
-    host_triple: &str,
-    target_triple: &str,
-) {
+    bootstrap_host_compiler: &Compiler,
+    target_triple: String,
+) -> Compiler {
     eprintln!("[BUILD] sysroot {:?}", sysroot_kind);
 
     DIST_DIR.ensure_fresh(dirs);
     BIN_DIR.ensure_exists(dirs);
     LIB_DIR.ensure_exists(dirs);
 
+    let is_native = bootstrap_host_compiler.triple == target_triple;
+
     // Copy the backend
     let cg_clif_dylib_path = if cfg!(windows) {
         // Windows doesn't have rpath support, so the cg_clif dylib needs to be next to the
@@ -35,129 +36,169 @@ pub(crate) fn build_sysroot(
         LIB_DIR
     }
     .to_path(dirs)
-    .join(get_file_name("rustc_codegen_cranelift", "dylib"));
+    .join(cg_clif_dylib_src.file_name().unwrap());
     try_hard_link(cg_clif_dylib_src, &cg_clif_dylib_path);
 
     // Build and copy rustc and cargo wrappers
+    let wrapper_base_name = get_file_name("____", "bin");
+    let toolchain_name = get_toolchain_name();
     for wrapper in ["rustc-clif", "rustdoc-clif", "cargo-clif"] {
-        let wrapper_name = get_wrapper_file_name(wrapper, "bin");
+        let wrapper_name = wrapper_base_name.replace("____", wrapper);
 
-        let mut build_cargo_wrapper_cmd = Command::new("rustc");
+        let mut build_cargo_wrapper_cmd = Command::new(&bootstrap_host_compiler.rustc);
         build_cargo_wrapper_cmd
+            .env("TOOLCHAIN_NAME", toolchain_name.clone())
             .arg(RelPath::SCRIPTS.to_path(dirs).join(&format!("{wrapper}.rs")))
             .arg("-o")
             .arg(DIST_DIR.to_path(dirs).join(wrapper_name))
-            .arg("-g");
+            .arg("-Cstrip=debuginfo");
         spawn_and_wait(build_cargo_wrapper_cmd);
     }
 
-    let default_sysroot = super::rustc_info::get_default_sysroot();
+    let host = build_sysroot_for_triple(
+        dirs,
+        channel,
+        bootstrap_host_compiler.clone(),
+        &cg_clif_dylib_path,
+        sysroot_kind,
+    );
+    host.install_into_sysroot(&DIST_DIR.to_path(dirs));
 
-    let host_rustlib_lib = RUSTLIB_DIR.to_path(dirs).join(host_triple).join("lib");
-    let target_rustlib_lib = RUSTLIB_DIR.to_path(dirs).join(target_triple).join("lib");
-    fs::create_dir_all(&host_rustlib_lib).unwrap();
-    fs::create_dir_all(&target_rustlib_lib).unwrap();
+    if !is_native {
+        build_sysroot_for_triple(
+            dirs,
+            channel,
+            {
+                let mut bootstrap_target_compiler = bootstrap_host_compiler.clone();
+                bootstrap_target_compiler.triple = target_triple.clone();
+                bootstrap_target_compiler.set_cross_linker_and_runner();
+                bootstrap_target_compiler
+            },
+            &cg_clif_dylib_path,
+            sysroot_kind,
+        )
+        .install_into_sysroot(&DIST_DIR.to_path(dirs));
+    }
 
-    if target_triple == "x86_64-pc-windows-gnu" {
-        if !default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib").exists() {
-            eprintln!(
-                "The x86_64-pc-windows-gnu target needs to be installed first before it is possible \
-                to compile a sysroot for it.",
-            );
-            process::exit(1);
+    // Copy std for the host to the lib dir. This is necessary for the jit mode to find
+    // libstd.
+    for lib in host.libs {
+        let filename = lib.file_name().unwrap().to_str().unwrap();
+        if filename.contains("std-") && !filename.contains(".rlib") {
+            try_hard_link(&lib, LIB_DIR.to_path(dirs).join(lib.file_name().unwrap()));
         }
-        for file in fs::read_dir(
-            default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib"),
-        )
-        .unwrap()
-        {
-            let file = file.unwrap().path();
-            if file.extension().map_or(true, |ext| ext.to_str().unwrap() != "o") {
-                continue; // only copy object files
-            }
-            try_hard_link(&file, target_rustlib_lib.join(file.file_name().unwrap()));
+    }
+
+    let mut target_compiler = {
+        let dirs: &Dirs = &dirs;
+        let rustc_clif =
+            RelPath::DIST.to_path(&dirs).join(wrapper_base_name.replace("____", "rustc-clif"));
+        let rustdoc_clif =
+            RelPath::DIST.to_path(&dirs).join(wrapper_base_name.replace("____", "rustdoc-clif"));
+
+        Compiler {
+            cargo: bootstrap_host_compiler.cargo.clone(),
+            rustc: rustc_clif.clone(),
+            rustdoc: rustdoc_clif.clone(),
+            rustflags: String::new(),
+            rustdocflags: String::new(),
+            triple: target_triple,
+            runner: vec![],
         }
+    };
+    if !is_native {
+        target_compiler.set_cross_linker_and_runner();
     }
+    target_compiler
+}
 
-    match sysroot_kind {
-        SysrootKind::None => {} // Nothing to do
-        SysrootKind::Llvm => {
-            for file in fs::read_dir(
-                default_sysroot.join("lib").join("rustlib").join(host_triple).join("lib"),
-            )
-            .unwrap()
-            {
-                let file = file.unwrap().path();
-                let file_name_str = file.file_name().unwrap().to_str().unwrap();
-                if (file_name_str.contains("rustc_")
-                    && !file_name_str.contains("rustc_std_workspace_")
-                    && !file_name_str.contains("rustc_demangle"))
-                    || file_name_str.contains("chalk")
-                    || file_name_str.contains("tracing")
-                    || file_name_str.contains("regex")
-                {
-                    // These are large crates that are part of the rustc-dev component and are not
-                    // necessary to run regular programs.
-                    continue;
-                }
-                try_hard_link(&file, host_rustlib_lib.join(file.file_name().unwrap()));
-            }
+struct SysrootTarget {
+    triple: String,
+    libs: Vec<PathBuf>,
+}
 
-            if target_triple != host_triple {
-                for file in fs::read_dir(
-                    default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib"),
-                )
-                .unwrap()
-                {
-                    let file = file.unwrap().path();
-                    try_hard_link(&file, target_rustlib_lib.join(file.file_name().unwrap()));
-                }
-            }
+impl SysrootTarget {
+    fn install_into_sysroot(&self, sysroot: &Path) {
+        if self.libs.is_empty() {
+            return;
         }
-        SysrootKind::Clif => {
-            build_clif_sysroot_for_triple(dirs, channel, host_triple, &cg_clif_dylib_path, None);
-
-            if host_triple != target_triple {
-                // When cross-compiling it is often necessary to manually pick the right linker
-                let linker = match target_triple {
-                    "aarch64-unknown-linux-gnu" => Some("aarch64-linux-gnu-gcc"),
-                    "s390x-unknown-linux-gnu" => Some("s390x-linux-gnu-gcc"),
-                    _ => None,
-                };
-                build_clif_sysroot_for_triple(
-                    dirs,
-                    channel,
-                    target_triple,
-                    &cg_clif_dylib_path,
-                    linker,
-                );
-            }
 
-            // Copy std for the host to the lib dir. This is necessary for the jit mode to find
-            // libstd.
-            for file in fs::read_dir(host_rustlib_lib).unwrap() {
-                let file = file.unwrap().path();
-                let filename = file.file_name().unwrap().to_str().unwrap();
-                if filename.contains("std-") && !filename.contains(".rlib") {
-                    try_hard_link(&file, LIB_DIR.to_path(dirs).join(file.file_name().unwrap()));
-                }
-            }
+        let target_rustlib_lib = sysroot.join("lib").join("rustlib").join(&self.triple).join("lib");
+        fs::create_dir_all(&target_rustlib_lib).unwrap();
+
+        for lib in &self.libs {
+            try_hard_link(lib, target_rustlib_lib.join(lib.file_name().unwrap()));
+        }
+    }
+}
+
+pub(crate) static ORIG_BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot");
+pub(crate) static BUILD_SYSROOT: RelPath = RelPath::DOWNLOAD.join("sysroot");
+pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = BUILD_SYSROOT.join("rustc_version");
+pub(crate) static SYSROOT_SRC: RelPath = BUILD_SYSROOT.join("sysroot_src");
+pub(crate) static STANDARD_LIBRARY: CargoProject =
+    CargoProject::new(&BUILD_SYSROOT, "build_sysroot");
+pub(crate) static RTSTARTUP_SYSROOT: RelPath = RelPath::BUILD.join("rtstartup");
+
+#[must_use]
+fn build_sysroot_for_triple(
+    dirs: &Dirs,
+    channel: &str,
+    compiler: Compiler,
+    cg_clif_dylib_path: &Path,
+    sysroot_kind: SysrootKind,
+) -> SysrootTarget {
+    match sysroot_kind {
+        SysrootKind::None => build_rtstartup(dirs, &compiler)
+            .unwrap_or(SysrootTarget { triple: compiler.triple, libs: vec![] }),
+        SysrootKind::Llvm => build_llvm_sysroot_for_triple(compiler),
+        SysrootKind::Clif => {
+            build_clif_sysroot_for_triple(dirs, channel, compiler, &cg_clif_dylib_path)
         }
     }
 }
 
-// FIXME move to download/ or dist/
-pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = RelPath::BUILD_SYSROOT.join("rustc_version");
-pub(crate) static SYSROOT_SRC: RelPath = RelPath::BUILD_SYSROOT.join("sysroot_src");
-static STANDARD_LIBRARY: CargoProject = CargoProject::new(&RelPath::BUILD_SYSROOT, "build_sysroot");
+#[must_use]
+fn build_llvm_sysroot_for_triple(compiler: Compiler) -> SysrootTarget {
+    let default_sysroot = super::rustc_info::get_default_sysroot(&compiler.rustc);
 
+    let mut target_libs = SysrootTarget { triple: compiler.triple, libs: vec![] };
+
+    for entry in fs::read_dir(
+        default_sysroot.join("lib").join("rustlib").join(&target_libs.triple).join("lib"),
+    )
+    .unwrap()
+    {
+        let entry = entry.unwrap();
+        if entry.file_type().unwrap().is_dir() {
+            continue;
+        }
+        let file = entry.path();
+        let file_name_str = file.file_name().unwrap().to_str().unwrap();
+        if (file_name_str.contains("rustc_")
+            && !file_name_str.contains("rustc_std_workspace_")
+            && !file_name_str.contains("rustc_demangle"))
+            || file_name_str.contains("chalk")
+            || file_name_str.contains("tracing")
+            || file_name_str.contains("regex")
+        {
+            // These are large crates that are part of the rustc-dev component and are not
+            // necessary to run regular programs.
+            continue;
+        }
+        target_libs.libs.push(file);
+    }
+
+    target_libs
+}
+
+#[must_use]
 fn build_clif_sysroot_for_triple(
     dirs: &Dirs,
     channel: &str,
-    triple: &str,
+    mut compiler: Compiler,
     cg_clif_dylib_path: &Path,
-    linker: Option<&str>,
-) {
+) -> SysrootTarget {
     match fs::read_to_string(SYSROOT_RUSTC_VERSION.to_path(dirs)) {
         Err(e) => {
             eprintln!("Failed to get rustc version for patched sysroot source: {}", e);
@@ -165,7 +206,7 @@ fn build_clif_sysroot_for_triple(
             process::exit(1);
         }
         Ok(source_version) => {
-            let rustc_version = get_rustc_version();
+            let rustc_version = get_rustc_version(&compiler.rustc);
             if source_version != rustc_version {
                 eprintln!("The patched sysroot source is outdated");
                 eprintln!("Source version: {}", source_version.trim());
@@ -176,37 +217,42 @@ fn build_clif_sysroot_for_triple(
         }
     }
 
-    let build_dir = STANDARD_LIBRARY.target_dir(dirs).join(triple).join(channel);
+    let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] };
+
+    if let Some(rtstartup_target_libs) = build_rtstartup(dirs, &compiler) {
+        rtstartup_target_libs.install_into_sysroot(&RTSTARTUP_SYSROOT.to_path(dirs));
+
+        target_libs.libs.extend(rtstartup_target_libs.libs);
+    }
+
+    let build_dir = STANDARD_LIBRARY.target_dir(dirs).join(&compiler.triple).join(channel);
 
     if !super::config::get_bool("keep_sysroot") {
         // Cleanup the deps dir, but keep build scripts and the incremental cache for faster
         // recompilation as they are not affected by changes in cg_clif.
-        if build_dir.join("deps").exists() {
-            fs::remove_dir_all(build_dir.join("deps")).unwrap();
-        }
+        remove_dir_if_exists(&build_dir.join("deps"));
     }
 
     // Build sysroot
-    let mut rustflags = "-Zforce-unstable-if-unmarked -Cpanic=abort".to_string();
+    let mut rustflags = " -Zforce-unstable-if-unmarked -Cpanic=abort".to_string();
     rustflags.push_str(&format!(" -Zcodegen-backend={}", cg_clif_dylib_path.to_str().unwrap()));
-    rustflags.push_str(&format!(" --sysroot={}", DIST_DIR.to_path(dirs).to_str().unwrap()));
+    // Necessary for MinGW to find rsbegin.o and rsend.o
+    rustflags
+        .push_str(&format!(" --sysroot={}", RTSTARTUP_SYSROOT.to_path(dirs).to_str().unwrap()));
     if channel == "release" {
         rustflags.push_str(" -Zmir-opt-level=3");
     }
-    if let Some(linker) = linker {
-        use std::fmt::Write;
-        write!(rustflags, " -Clinker={}", linker).unwrap();
-    }
-    let mut compiler = Compiler::with_triple(triple.to_owned());
-    compiler.rustflags = rustflags;
+    compiler.rustflags += &rustflags;
     let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs);
     if channel == "release" {
         build_cmd.arg("--release");
     }
     build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif");
+    if compiler.triple.contains("apple") {
+        build_cmd.env("CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO", "packed");
+    }
     spawn_and_wait(build_cmd);
 
-    // Copy all relevant files to the sysroot
     for entry in fs::read_dir(build_dir.join("deps")).unwrap() {
         let entry = entry.unwrap();
         if let Some(ext) = entry.path().extension() {
@@ -216,9 +262,35 @@ fn build_clif_sysroot_for_triple(
         } else {
             continue;
         };
-        try_hard_link(
-            entry.path(),
-            RUSTLIB_DIR.to_path(dirs).join(triple).join("lib").join(entry.file_name()),
-        );
+        target_libs.libs.push(entry.path());
     }
+
+    target_libs
+}
+
+fn build_rtstartup(dirs: &Dirs, compiler: &Compiler) -> Option<SysrootTarget> {
+    if !compiler.triple.ends_with("windows-gnu") {
+        return None;
+    }
+
+    RTSTARTUP_SYSROOT.ensure_fresh(dirs);
+
+    let rtstartup_src = SYSROOT_SRC.to_path(dirs).join("library").join("rtstartup");
+    let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] };
+
+    for file in ["rsbegin", "rsend"] {
+        let obj = RTSTARTUP_SYSROOT.to_path(dirs).join(format!("{file}.o"));
+        let mut build_rtstartup_cmd = Command::new(&compiler.rustc);
+        build_rtstartup_cmd
+            .arg("--target")
+            .arg(&compiler.triple)
+            .arg("--emit=obj")
+            .arg("-o")
+            .arg(&obj)
+            .arg(rtstartup_src.join(format!("{file}.rs")));
+        spawn_and_wait(build_rtstartup_cmd);
+        target_libs.libs.push(obj.clone());
+    }
+
+    Some(target_libs)
 }
diff --git a/compiler/rustc_codegen_cranelift/build_system/mod.rs b/compiler/rustc_codegen_cranelift/build_system/mod.rs
index 1afc9a55c..8a53acdf7 100644
--- a/compiler/rustc_codegen_cranelift/build_system/mod.rs
+++ b/compiler/rustc_codegen_cranelift/build_system/mod.rs
@@ -2,9 +2,10 @@ use std::env;
 use std::path::PathBuf;
 use std::process;
 
-use self::utils::is_ci;
+use self::utils::{is_ci, is_ci_opt, Compiler};
 
 mod abi_cafe;
+mod bench;
 mod build_backend;
 mod build_sysroot;
 mod config;
@@ -14,31 +15,8 @@ mod rustc_info;
 mod tests;
 mod utils;
 
-const USAGE: &str = r#"The build system of cg_clif.
-
-USAGE:
-    ./y.rs prepare [--out-dir DIR]
-    ./y.rs build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
-    ./y.rs test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
-
-OPTIONS:
-    --sysroot none|clif|llvm
-            Which sysroot libraries to use:
-            `none` will not include any standard library in the sysroot.
-            `clif` will build the standard library using Cranelift.
-            `llvm` will use the pre-compiled standard library of rustc which is compiled with LLVM.
-
-    --out-dir DIR
-            Specify the directory in which the download, build and dist directories are stored.
-            By default this is the working directory.
-
-    --no-unstable-features
-            fSome features are not yet ready for production usage. This option will disable these
-            features. This includes the JIT mode and inline assembly support.
-"#;
-
 fn usage() {
-    eprintln!("{USAGE}");
+    eprintln!("{}", include_str!("usage.txt"));
 }
 
 macro_rules! arg_error {
@@ -54,6 +32,8 @@ enum Command {
     Prepare,
     Build,
     Test,
+    AbiCafe,
+    Bench,
 }
 
 #[derive(Copy, Clone, Debug)]
@@ -64,12 +44,19 @@ pub(crate) enum SysrootKind {
 }
 
 pub fn main() {
-    env::set_var("CG_CLIF_DISPLAY_CG_TIME", "1");
+    if env::var("RUST_BACKTRACE").is_err() {
+        env::set_var("RUST_BACKTRACE", "1");
+    }
     env::set_var("CG_CLIF_DISABLE_INCR_CACHE", "1");
 
     if is_ci() {
         // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway
         env::set_var("CARGO_BUILD_INCREMENTAL", "false");
+
+        if !is_ci_opt() {
+            // Enable the Cranelift verifier
+            env::set_var("CG_CLIF_ENABLE_VERIFIER", "1");
+        }
     }
 
     let mut args = env::args().skip(1);
@@ -77,6 +64,8 @@ pub fn main() {
         Some("prepare") => Command::Prepare,
         Some("build") => Command::Build,
         Some("test") => Command::Test,
+        Some("abi-cafe") => Command::AbiCafe,
+        Some("bench") => Command::Bench,
         Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag),
         Some(command) => arg_error!("Unknown command {}", command),
         None => {
@@ -112,24 +101,16 @@ pub fn main() {
         }
     }
 
-    let host_triple = if let Ok(host_triple) = std::env::var("HOST_TRIPLE") {
-        host_triple
-    } else if let Some(host_triple) = config::get_value("host") {
-        host_triple
-    } else {
-        rustc_info::get_host_triple()
-    };
-    let target_triple = if let Ok(target_triple) = std::env::var("TARGET_TRIPLE") {
-        if target_triple != "" {
-            target_triple
-        } else {
-            host_triple.clone() // Empty target triple can happen on GHA
-        }
-    } else if let Some(target_triple) = config::get_value("target") {
-        target_triple
-    } else {
-        host_triple.clone()
-    };
+    let bootstrap_host_compiler = Compiler::bootstrap_with_triple(
+        std::env::var("HOST_TRIPLE")
+            .ok()
+            .or_else(|| config::get_value("host"))
+            .unwrap_or_else(|| rustc_info::get_host_triple()),
+    );
+    let target_triple = std::env::var("TARGET_TRIPLE")
+        .ok()
+        .or_else(|| config::get_value("target"))
+        .unwrap_or_else(|| bootstrap_host_compiler.triple.clone());
 
     // FIXME allow changing the location of these dirs using cli arguments
     let current_dir = std::env::current_dir().unwrap();
@@ -157,8 +138,15 @@ pub fn main() {
         process::exit(0);
     }
 
-    let cg_clif_dylib =
-        build_backend::build_backend(&dirs, channel, &host_triple, use_unstable_features);
+    env::set_var("RUSTC", "rustc_should_be_set_explicitly");
+    env::set_var("RUSTDOC", "rustdoc_should_be_set_explicitly");
+
+    let cg_clif_dylib = build_backend::build_backend(
+        &dirs,
+        channel,
+        &bootstrap_host_compiler,
+        use_unstable_features,
+    );
     match command {
         Command::Prepare => {
             // Handled above
@@ -169,28 +157,37 @@ pub fn main() {
                 channel,
                 sysroot_kind,
                 &cg_clif_dylib,
-                &host_triple,
-                &target_triple,
+                &bootstrap_host_compiler,
+                target_triple.clone(),
             );
-
-            abi_cafe::run(
+        }
+        Command::AbiCafe => {
+            if bootstrap_host_compiler.triple != target_triple {
+                eprintln!("Abi-cafe doesn't support cross-compilation");
+                process::exit(1);
+            }
+            abi_cafe::run(channel, sysroot_kind, &dirs, &cg_clif_dylib, &bootstrap_host_compiler);
+        }
+        Command::Build => {
+            build_sysroot::build_sysroot(
+                &dirs,
                 channel,
                 sysroot_kind,
-                &dirs,
                 &cg_clif_dylib,
-                &host_triple,
-                &target_triple,
+                &bootstrap_host_compiler,
+                target_triple,
             );
         }
-        Command::Build => {
+        Command::Bench => {
             build_sysroot::build_sysroot(
                 &dirs,
                 channel,
                 sysroot_kind,
                 &cg_clif_dylib,
-                &host_triple,
-                &target_triple,
+                &bootstrap_host_compiler,
+                target_triple,
             );
+            bench::benchmark(&dirs, &bootstrap_host_compiler);
         }
     }
 }
diff --git a/compiler/rustc_codegen_cranelift/build_system/path.rs b/compiler/rustc_codegen_cranelift/build_system/path.rs
index e93981f1d..329072300 100644
--- a/compiler/rustc_codegen_cranelift/build_system/path.rs
+++ b/compiler/rustc_codegen_cranelift/build_system/path.rs
@@ -1,6 +1,8 @@
 use std::fs;
 use std::path::PathBuf;
 
+use super::utils::remove_dir_if_exists;
+
 #[derive(Debug, Clone)]
 pub(crate) struct Dirs {
     pub(crate) source_dir: PathBuf,
@@ -42,7 +44,6 @@ impl RelPath {
     pub(crate) const DIST: RelPath = RelPath::Base(PathBase::Dist);
 
     pub(crate) const SCRIPTS: RelPath = RelPath::SOURCE.join("scripts");
-    pub(crate) const BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot");
     pub(crate) const PATCHES: RelPath = RelPath::SOURCE.join("patches");
 
     pub(crate) const fn join(&'static self, suffix: &'static str) -> RelPath {
@@ -62,9 +63,7 @@ impl RelPath {
 
     pub(crate) fn ensure_fresh(&self, dirs: &Dirs) {
         let path = self.to_path(dirs);
-        if path.exists() {
-            fs::remove_dir_all(&path).unwrap();
-        }
+        remove_dir_if_exists(&path);
         fs::create_dir_all(path).unwrap();
     }
 }
diff --git a/compiler/rustc_codegen_cranelift/build_system/prepare.rs b/compiler/rustc_codegen_cranelift/build_system/prepare.rs
index 8ac67e8f9..50b1b7836 100644
--- a/compiler/rustc_codegen_cranelift/build_system/prepare.rs
+++ b/compiler/rustc_codegen_cranelift/build_system/prepare.rs
@@ -3,73 +3,51 @@ use std::fs;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
-use super::build_sysroot::{SYSROOT_RUSTC_VERSION, SYSROOT_SRC};
+use super::build_sysroot::{BUILD_SYSROOT, ORIG_BUILD_SYSROOT, SYSROOT_RUSTC_VERSION, SYSROOT_SRC};
 use super::path::{Dirs, RelPath};
-use super::rustc_info::{get_file_name, get_rustc_path, get_rustc_version};
-use super::utils::{copy_dir_recursively, spawn_and_wait, Compiler};
+use super::rustc_info::{get_default_sysroot, get_rustc_version};
+use super::utils::{copy_dir_recursively, git_command, retry_spawn_and_wait, spawn_and_wait};
 
 pub(crate) fn prepare(dirs: &Dirs) {
-    if RelPath::DOWNLOAD.to_path(dirs).exists() {
-        std::fs::remove_dir_all(RelPath::DOWNLOAD.to_path(dirs)).unwrap();
-    }
-    std::fs::create_dir_all(RelPath::DOWNLOAD.to_path(dirs)).unwrap();
+    RelPath::DOWNLOAD.ensure_fresh(dirs);
 
-    prepare_sysroot(dirs);
+    spawn_and_wait(super::build_backend::CG_CLIF.fetch("cargo", "rustc", dirs));
 
-    // FIXME maybe install this only locally?
-    eprintln!("[INSTALL] hyperfine");
-    Command::new("cargo")
-        .arg("install")
-        .arg("hyperfine")
-        .env_remove("CARGO_TARGET_DIR")
-        .spawn()
-        .unwrap()
-        .wait()
-        .unwrap();
+    prepare_sysroot(dirs);
+    spawn_and_wait(super::build_sysroot::STANDARD_LIBRARY.fetch("cargo", "rustc", dirs));
+    spawn_and_wait(super::tests::LIBCORE_TESTS.fetch("cargo", "rustc", dirs));
 
-    super::abi_cafe::ABI_CAFE_REPO.fetch(dirs);
     super::tests::RAND_REPO.fetch(dirs);
+    spawn_and_wait(super::tests::RAND.fetch("cargo", "rustc", dirs));
     super::tests::REGEX_REPO.fetch(dirs);
+    spawn_and_wait(super::tests::REGEX.fetch("cargo", "rustc", dirs));
     super::tests::PORTABLE_SIMD_REPO.fetch(dirs);
-    super::tests::SIMPLE_RAYTRACER_REPO.fetch(dirs);
-
-    eprintln!("[LLVM BUILD] simple-raytracer");
-    let host_compiler = Compiler::host();
-    let build_cmd = super::tests::SIMPLE_RAYTRACER.build(&host_compiler, dirs);
-    spawn_and_wait(build_cmd);
-    fs::copy(
-        super::tests::SIMPLE_RAYTRACER
-            .target_dir(dirs)
-            .join(&host_compiler.triple)
-            .join("debug")
-            .join(get_file_name("main", "bin")),
-        RelPath::BUILD.to_path(dirs).join(get_file_name("raytracer_cg_llvm", "bin")),
-    )
-    .unwrap();
+    spawn_and_wait(super::tests::PORTABLE_SIMD.fetch("cargo", "rustc", dirs));
 }
 
 fn prepare_sysroot(dirs: &Dirs) {
-    let rustc_path = get_rustc_path();
-    let sysroot_src_orig = rustc_path.parent().unwrap().join("../lib/rustlib/src/rust");
-    let sysroot_src = SYSROOT_SRC;
-
+    let sysroot_src_orig = get_default_sysroot(Path::new("rustc")).join("lib/rustlib/src/rust");
     assert!(sysroot_src_orig.exists());
 
-    sysroot_src.ensure_fresh(dirs);
-    fs::create_dir_all(sysroot_src.to_path(dirs).join("library")).unwrap();
     eprintln!("[COPY] sysroot src");
+
+    // FIXME ensure builds error out or update the copy if any of the files copied here change
+    BUILD_SYSROOT.ensure_fresh(dirs);
+    copy_dir_recursively(&ORIG_BUILD_SYSROOT.to_path(dirs), &BUILD_SYSROOT.to_path(dirs));
+
+    fs::create_dir_all(SYSROOT_SRC.to_path(dirs).join("library")).unwrap();
     copy_dir_recursively(
         &sysroot_src_orig.join("library"),
-        &sysroot_src.to_path(dirs).join("library"),
+        &SYSROOT_SRC.to_path(dirs).join("library"),
     );
 
-    let rustc_version = get_rustc_version();
+    let rustc_version = get_rustc_version(Path::new("rustc"));
     fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap();
 
     eprintln!("[GIT] init");
-    init_git_repo(&sysroot_src.to_path(dirs));
+    init_git_repo(&SYSROOT_SRC.to_path(dirs));
 
-    apply_patches(dirs, "sysroot", &sysroot_src.to_path(dirs));
+    apply_patches(dirs, "sysroot", &SYSROOT_SRC.to_path(dirs));
 }
 
 pub(crate) struct GitRepo {
@@ -98,7 +76,7 @@ impl GitRepo {
         }
     }
 
-    fn fetch(&self, dirs: &Dirs) {
+    pub(crate) fn fetch(&self, dirs: &Dirs) {
         match self.url {
             GitRepoUrl::Github { user, repo } => {
                 clone_repo_shallow_github(
@@ -118,14 +96,14 @@ impl GitRepo {
 fn clone_repo(download_dir: &Path, repo: &str, rev: &str) {
     eprintln!("[CLONE] {}", repo);
     // Ignore exit code as the repo may already have been checked out
-    Command::new("git").arg("clone").arg(repo).arg(&download_dir).spawn().unwrap().wait().unwrap();
+    git_command(None, "clone").arg(repo).arg(download_dir).spawn().unwrap().wait().unwrap();
 
-    let mut clean_cmd = Command::new("git");
-    clean_cmd.arg("checkout").arg("--").arg(".").current_dir(&download_dir);
+    let mut clean_cmd = git_command(download_dir, "checkout");
+    clean_cmd.arg("--").arg(".");
     spawn_and_wait(clean_cmd);
 
-    let mut checkout_cmd = Command::new("git");
-    checkout_cmd.arg("checkout").arg("-q").arg(rev).current_dir(download_dir);
+    let mut checkout_cmd = git_command(download_dir, "checkout");
+    checkout_cmd.arg("-q").arg(rev);
     spawn_and_wait(checkout_cmd);
 }
 
@@ -149,8 +127,22 @@ fn clone_repo_shallow_github(dirs: &Dirs, download_dir: &Path, user: &str, repo:
 
     // Download zip archive
     let mut download_cmd = Command::new("curl");
-    download_cmd.arg("--location").arg("--output").arg(&archive_file).arg(archive_url);
-    spawn_and_wait(download_cmd);
+    download_cmd
+        .arg("--max-time")
+        .arg("600")
+        .arg("-y")
+        .arg("30")
+        .arg("-Y")
+        .arg("10")
+        .arg("--connect-timeout")
+        .arg("30")
+        .arg("--continue-at")
+        .arg("-")
+        .arg("--location")
+        .arg("--output")
+        .arg(&archive_file)
+        .arg(archive_url);
+    retry_spawn_and_wait(5, download_cmd);
 
     // Unpack tar archive
     let mut unpack_cmd = Command::new("tar");
@@ -167,25 +159,16 @@ fn clone_repo_shallow_github(dirs: &Dirs, download_dir: &Path, user: &str, repo:
 }
 
 fn init_git_repo(repo_dir: &Path) {
-    let mut git_init_cmd = Command::new("git");
-    git_init_cmd.arg("init").arg("-q").current_dir(repo_dir);
+    let mut git_init_cmd = git_command(repo_dir, "init");
+    git_init_cmd.arg("-q");
     spawn_and_wait(git_init_cmd);
 
-    let mut git_add_cmd = Command::new("git");
-    git_add_cmd.arg("add").arg(".").current_dir(repo_dir);
+    let mut git_add_cmd = git_command(repo_dir, "add");
+    git_add_cmd.arg(".");
     spawn_and_wait(git_add_cmd);
 
-    let mut git_commit_cmd = Command::new("git");
-    git_commit_cmd
-        .arg("-c")
-        .arg("user.name=Dummy")
-        .arg("-c")
-        .arg("user.email=dummy@example.com")
-        .arg("commit")
-        .arg("-m")
-        .arg("Initial commit")
-        .arg("-q")
-        .current_dir(repo_dir);
+    let mut git_commit_cmd = git_command(repo_dir, "commit");
+    git_commit_cmd.arg("-m").arg("Initial commit").arg("-q");
     spawn_and_wait(git_commit_cmd);
 }
 
@@ -220,16 +203,8 @@ fn apply_patches(dirs: &Dirs, crate_name: &str, target_dir: &Path) {
             target_dir.file_name().unwrap(),
             patch.file_name().unwrap()
         );
-        let mut apply_patch_cmd = Command::new("git");
-        apply_patch_cmd
-            .arg("-c")
-            .arg("user.name=Dummy")
-            .arg("-c")
-            .arg("user.email=dummy@example.com")
-            .arg("am")
-            .arg(patch)
-            .arg("-q")
-            .current_dir(target_dir);
+        let mut apply_patch_cmd = git_command(target_dir, "am");
+        apply_patch_cmd.arg(patch).arg("-q");
         spawn_and_wait(apply_patch_cmd);
     }
 }
diff --git a/compiler/rustc_codegen_cranelift/build_system/rustc_info.rs b/compiler/rustc_codegen_cranelift/build_system/rustc_info.rs
index 8e5ab688e..a70453b44 100644
--- a/compiler/rustc_codegen_cranelift/build_system/rustc_info.rs
+++ b/compiler/rustc_codegen_cranelift/build_system/rustc_info.rs
@@ -1,9 +1,9 @@
 use std::path::{Path, PathBuf};
 use std::process::{Command, Stdio};
 
-pub(crate) fn get_rustc_version() -> String {
+pub(crate) fn get_rustc_version(rustc: &Path) -> String {
     let version_info =
-        Command::new("rustc").stderr(Stdio::inherit()).args(&["-V"]).output().unwrap().stdout;
+        Command::new(rustc).stderr(Stdio::inherit()).args(&["-V"]).output().unwrap().stdout;
     String::from_utf8(version_info).unwrap()
 }
 
@@ -23,6 +23,16 @@ pub(crate) fn get_host_triple() -> String {
         .to_owned()
 }
 
+pub(crate) fn get_toolchain_name() -> String {
+    let active_toolchain = Command::new("rustup")
+        .stderr(Stdio::inherit())
+        .args(&["show", "active-toolchain"])
+        .output()
+        .unwrap()
+        .stdout;
+    String::from_utf8(active_toolchain).unwrap().trim().split_once(' ').unwrap().0.to_owned()
+}
+
 pub(crate) fn get_cargo_path() -> PathBuf {
     let cargo_path = Command::new("rustup")
         .stderr(Stdio::inherit())
@@ -53,8 +63,8 @@ pub(crate) fn get_rustdoc_path() -> PathBuf {
     Path::new(String::from_utf8(rustc_path).unwrap().trim()).to_owned()
 }
 
-pub(crate) fn get_default_sysroot() -> PathBuf {
-    let default_sysroot = Command::new("rustc")
+pub(crate) fn get_default_sysroot(rustc: &Path) -> PathBuf {
+    let default_sysroot = Command::new(rustc)
         .stderr(Stdio::inherit())
         .args(&["--print", "sysroot"])
         .output()
@@ -83,12 +93,3 @@ pub(crate) fn get_file_name(crate_name: &str, crate_type: &str) -> String {
     assert!(file_name.contains(crate_name));
     file_name
 }
-
-/// Similar to `get_file_name`, but converts any dashes (`-`) in the `crate_name` to
-/// underscores (`_`). This is specially made for the rustc and cargo wrappers
-/// which have a dash in the name, and that is not allowed in a crate name.
-pub(crate) fn get_wrapper_file_name(crate_name: &str, crate_type: &str) -> String {
-    let crate_name = crate_name.replace('-', "_");
-    let wrapper_name = get_file_name(&crate_name, crate_type);
-    wrapper_name.replace('_', "-")
-}
diff --git a/compiler/rustc_codegen_cranelift/build_system/tests.rs b/compiler/rustc_codegen_cranelift/build_system/tests.rs
index 1c372736e..e9486888f 100644
--- a/compiler/rustc_codegen_cranelift/build_system/tests.rs
+++ b/compiler/rustc_codegen_cranelift/build_system/tests.rs
@@ -1,11 +1,9 @@
-use super::build_sysroot;
+use super::build_sysroot::{self, SYSROOT_SRC};
 use super::config;
 use super::path::{Dirs, RelPath};
 use super::prepare::GitRepo;
-use super::rustc_info::{get_cargo_path, get_wrapper_file_name};
-use super::utils::{
-    hyperfine_command, spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler,
-};
+use super::rustc_info::get_host_triple;
+use super::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler};
 use super::SysrootKind;
 use std::env;
 use std::ffi::OsStr;
@@ -17,256 +15,111 @@ static BUILD_EXAMPLE_OUT_DIR: RelPath = RelPath::BUILD.join("example");
 
 struct TestCase {
     config: &'static str,
-    func: &'static dyn Fn(&TestRunner),
+    cmd: TestCaseCmd,
+}
+
+enum TestCaseCmd {
+    Custom { func: &'static dyn Fn(&TestRunner) },
+    BuildLib { source: &'static str, crate_types: &'static str },
+    BuildBinAndRun { source: &'static str, args: &'static [&'static str] },
+    JitBin { source: &'static str, args: &'static str },
 }
 
 impl TestCase {
-    const fn new(config: &'static str, func: &'static dyn Fn(&TestRunner)) -> Self {
-        Self { config, func }
+    // FIXME reduce usage of custom test case commands
+    const fn custom(config: &'static str, func: &'static dyn Fn(&TestRunner)) -> Self {
+        Self { config, cmd: TestCaseCmd::Custom { func } }
+    }
+
+    const fn build_lib(
+        config: &'static str,
+        source: &'static str,
+        crate_types: &'static str,
+    ) -> Self {
+        Self { config, cmd: TestCaseCmd::BuildLib { source, crate_types } }
+    }
+
+    const fn build_bin_and_run(
+        config: &'static str,
+        source: &'static str,
+        args: &'static [&'static str],
+    ) -> Self {
+        Self { config, cmd: TestCaseCmd::BuildBinAndRun { source, args } }
+    }
+
+    const fn jit_bin(config: &'static str, source: &'static str, args: &'static str) -> Self {
+        Self { config, cmd: TestCaseCmd::JitBin { source, args } }
     }
 }
 
 const NO_SYSROOT_SUITE: &[TestCase] = &[
-    TestCase::new("build.mini_core", &|runner| {
-        runner.run_rustc([
-            "example/mini_core.rs",
-            "--crate-name",
-            "mini_core",
-            "--crate-type",
-            "lib,dylib",
-            "--target",
-            &runner.target_compiler.triple,
-        ]);
-    }),
-    TestCase::new("build.example", &|runner| {
-        runner.run_rustc([
-            "example/example.rs",
-            "--crate-type",
-            "lib",
-            "--target",
-            &runner.target_compiler.triple,
-        ]);
-    }),
-    TestCase::new("jit.mini_core_hello_world", &|runner| {
-        let mut jit_cmd = runner.rustc_command([
-            "-Zunstable-options",
-            "-Cllvm-args=mode=jit",
-            "-Cprefer-dynamic",
-            "example/mini_core_hello_world.rs",
-            "--cfg",
-            "jit",
-            "--target",
-            &runner.target_compiler.triple,
-        ]);
-        jit_cmd.env("CG_CLIF_JIT_ARGS", "abc bcd");
-        spawn_and_wait(jit_cmd);
-
-        eprintln!("[JIT-lazy] mini_core_hello_world");
-        let mut jit_cmd = runner.rustc_command([
-            "-Zunstable-options",
-            "-Cllvm-args=mode=jit-lazy",
-            "-Cprefer-dynamic",
-            "example/mini_core_hello_world.rs",
-            "--cfg",
-            "jit",
-            "--target",
-            &runner.target_compiler.triple,
-        ]);
-        jit_cmd.env("CG_CLIF_JIT_ARGS", "abc bcd");
-        spawn_and_wait(jit_cmd);
-    }),
-    TestCase::new("aot.mini_core_hello_world", &|runner| {
-        runner.run_rustc([
-            "example/mini_core_hello_world.rs",
-            "--crate-name",
-            "mini_core_hello_world",
-            "--crate-type",
-            "bin",
-            "-g",
-            "--target",
-            &runner.target_compiler.triple,
-        ]);
-        runner.run_out_command("mini_core_hello_world", ["abc", "bcd"]);
-    }),
+    TestCase::build_lib("build.mini_core", "example/mini_core.rs", "lib,dylib"),
+    TestCase::build_lib("build.example", "example/example.rs", "lib"),
+    TestCase::jit_bin("jit.mini_core_hello_world", "example/mini_core_hello_world.rs", "abc bcd"),
+    TestCase::build_bin_and_run(
+        "aot.mini_core_hello_world",
+        "example/mini_core_hello_world.rs",
+        &["abc", "bcd"],
+    ),
 ];
 
 const BASE_SYSROOT_SUITE: &[TestCase] = &[
-    TestCase::new("aot.arbitrary_self_types_pointers_and_wrappers", &|runner| {
-        runner.run_rustc([
-            "example/arbitrary_self_types_pointers_and_wrappers.rs",
-            "--crate-name",
-            "arbitrary_self_types_pointers_and_wrappers",
-            "--crate-type",
-            "bin",
-            "--target",
-            &runner.target_compiler.triple,
-        ]);
-        runner.run_out_command("arbitrary_self_types_pointers_and_wrappers", []);
-    }),
-    TestCase::new("aot.issue_91827_extern_types", &|runner| {
-        runner.run_rustc([
-            "example/issue-91827-extern-types.rs",
-            "--crate-name",
-            "issue_91827_extern_types",
-            "--crate-type",
-            "bin",
-            "--target",
-            &runner.target_compiler.triple,
-        ]);
-        runner.run_out_command("issue_91827_extern_types", []);
-    }),
-    TestCase::new("build.alloc_system", &|runner| {
-        runner.run_rustc([
-            "example/alloc_system.rs",
-            "--crate-type",
-            "lib",
-            "--target",
-            &runner.target_compiler.triple,
-        ]);
-    }),
-    TestCase::new("aot.alloc_example", &|runner| {
-        runner.run_rustc([
-            "example/alloc_example.rs",
-            "--crate-type",
-            "bin",
-            "--target",
-            &runner.target_compiler.triple,
-        ]);
-        runner.run_out_command("alloc_example", []);
-    }),
-    TestCase::new("jit.std_example", &|runner| {
-        runner.run_rustc([
-            "-Zunstable-options",
-            "-Cllvm-args=mode=jit",
-            "-Cprefer-dynamic",
-            "example/std_example.rs",
-            "--target",
-            &runner.target_compiler.triple,
-        ]);
-
-        eprintln!("[JIT-lazy] std_example");
-        runner.run_rustc([
-            "-Zunstable-options",
-            "-Cllvm-args=mode=jit-lazy",
-            "-Cprefer-dynamic",
-            "example/std_example.rs",
-            "--target",
-            &runner.target_compiler.triple,
-        ]);
-    }),
-    TestCase::new("aot.std_example", &|runner| {
-        runner.run_rustc([
-            "example/std_example.rs",
-            "--crate-type",
-            "bin",
-            "--target",
-            &runner.target_compiler.triple,
-        ]);
-        runner.run_out_command("std_example", ["arg"]);
-    }),
-    TestCase::new("aot.dst_field_align", &|runner| {
-        runner.run_rustc([
-            "example/dst-field-align.rs",
-            "--crate-name",
-            "dst_field_align",
-            "--crate-type",
-            "bin",
-            "--target",
-            &runner.target_compiler.triple,
-        ]);
-        runner.run_out_command("dst_field_align", []);
-    }),
-    TestCase::new("aot.subslice-patterns-const-eval", &|runner| {
-        runner.run_rustc([
-            "example/subslice-patterns-const-eval.rs",
-            "--crate-type",
-            "bin",
-            "-Cpanic=abort",
-            "--target",
-            &runner.target_compiler.triple,
-        ]);
-        runner.run_out_command("subslice-patterns-const-eval", []);
-    }),
-    TestCase::new("aot.track-caller-attribute", &|runner| {
-        runner.run_rustc([
-            "example/track-caller-attribute.rs",
-            "--crate-type",
-            "bin",
-            "-Cpanic=abort",
-            "--target",
-            &runner.target_compiler.triple,
-        ]);
-        runner.run_out_command("track-caller-attribute", []);
-    }),
-    TestCase::new("aot.float-minmax-pass", &|runner| {
-        runner.run_rustc([
-            "example/float-minmax-pass.rs",
-            "--crate-type",
-            "bin",
-            "-Cpanic=abort",
-            "--target",
-            &runner.target_compiler.triple,
-        ]);
-        runner.run_out_command("float-minmax-pass", []);
-    }),
-    TestCase::new("aot.mod_bench", &|runner| {
-        runner.run_rustc([
-            "example/mod_bench.rs",
-            "--crate-type",
-            "bin",
-            "--target",
-            &runner.target_compiler.triple,
-        ]);
-        runner.run_out_command("mod_bench", []);
-    }),
-    TestCase::new("aot.issue-72793", &|runner| {
-        runner.run_rustc([
-            "example/issue-72793.rs",
-            "--crate-type",
-            "bin",
-            "--target",
-            &runner.target_compiler.triple,
-        ]);
-        runner.run_out_command("issue-72793", []);
-    }),
+    TestCase::build_bin_and_run(
+        "aot.arbitrary_self_types_pointers_and_wrappers",
+        "example/arbitrary_self_types_pointers_and_wrappers.rs",
+        &[],
+    ),
+    TestCase::build_bin_and_run(
+        "aot.issue_91827_extern_types",
+        "example/issue-91827-extern-types.rs",
+        &[],
+    ),
+    TestCase::build_lib("build.alloc_system", "example/alloc_system.rs", "lib"),
+    TestCase::build_bin_and_run("aot.alloc_example", "example/alloc_example.rs", &[]),
+    TestCase::jit_bin("jit.std_example", "example/std_example.rs", ""),
+    TestCase::build_bin_and_run("aot.std_example", "example/std_example.rs", &["arg"]),
+    TestCase::build_bin_and_run("aot.dst_field_align", "example/dst-field-align.rs", &[]),
+    TestCase::build_bin_and_run(
+        "aot.subslice-patterns-const-eval",
+        "example/subslice-patterns-const-eval.rs",
+        &[],
+    ),
+    TestCase::build_bin_and_run(
+        "aot.track-caller-attribute",
+        "example/track-caller-attribute.rs",
+        &[],
+    ),
+    TestCase::build_bin_and_run("aot.float-minmax-pass", "example/float-minmax-pass.rs", &[]),
+    TestCase::build_bin_and_run("aot.mod_bench", "example/mod_bench.rs", &[]),
+    TestCase::build_bin_and_run("aot.issue-72793", "example/issue-72793.rs", &[]),
 ];
 
 pub(crate) static RAND_REPO: GitRepo =
     GitRepo::github("rust-random", "rand", "0f933f9c7176e53b2a3c7952ded484e1783f0bf1", "rand");
 
-static RAND: CargoProject = CargoProject::new(&RAND_REPO.source_dir(), "rand");
+pub(crate) static RAND: CargoProject = CargoProject::new(&RAND_REPO.source_dir(), "rand");
 
 pub(crate) static REGEX_REPO: GitRepo =
     GitRepo::github("rust-lang", "regex", "341f207c1071f7290e3f228c710817c280c8dca1", "regex");
 
-static REGEX: CargoProject = CargoProject::new(&REGEX_REPO.source_dir(), "regex");
+pub(crate) static REGEX: CargoProject = CargoProject::new(&REGEX_REPO.source_dir(), "regex");
 
 pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github(
     "rust-lang",
     "portable-simd",
-    "d5cd4a8112d958bd3a252327e0d069a6363249bd",
+    "582239ac3b32007613df04d7ffa78dc30f4c5645",
     "portable-simd",
 );
 
-static PORTABLE_SIMD: CargoProject =
+pub(crate) static PORTABLE_SIMD: CargoProject =
     CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable_simd");
 
-pub(crate) static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
-    "ebobby",
-    "simple-raytracer",
-    "804a7a21b9e673a482797aa289a18ed480e4d813",
-    "<none>",
-);
-
-pub(crate) static SIMPLE_RAYTRACER: CargoProject =
-    CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer");
-
-static LIBCORE_TESTS: CargoProject =
-    CargoProject::new(&RelPath::BUILD_SYSROOT.join("sysroot_src/library/core/tests"), "core_tests");
+pub(crate) static LIBCORE_TESTS: CargoProject =
+    CargoProject::new(&SYSROOT_SRC.join("library/core/tests"), "core_tests");
 
 const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
-    TestCase::new("test.rust-random/rand", &|runner| {
-        spawn_and_wait(RAND.clean(&runner.target_compiler.cargo, &runner.dirs));
+    TestCase::custom("test.rust-random/rand", &|runner| {
+        RAND.clean(&runner.dirs);
 
         if runner.is_native {
             eprintln!("[TEST] rust-random/rand");
@@ -280,60 +133,8 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
             spawn_and_wait(build_cmd);
         }
     }),
-    TestCase::new("bench.simple-raytracer", &|runner| {
-        let run_runs = env::var("RUN_RUNS").unwrap_or("10".to_string()).parse().unwrap();
-
-        if runner.is_native {
-            eprintln!("[BENCH COMPILE] ebobby/simple-raytracer");
-            let cargo_clif = RelPath::DIST
-                .to_path(&runner.dirs)
-                .join(get_wrapper_file_name("cargo-clif", "bin"));
-            let manifest_path = SIMPLE_RAYTRACER.manifest_path(&runner.dirs);
-            let target_dir = SIMPLE_RAYTRACER.target_dir(&runner.dirs);
-
-            let clean_cmd = format!(
-                "cargo clean --manifest-path {manifest_path} --target-dir {target_dir}",
-                manifest_path = manifest_path.display(),
-                target_dir = target_dir.display(),
-            );
-            let llvm_build_cmd = format!(
-                "cargo build --manifest-path {manifest_path} --target-dir {target_dir}",
-                manifest_path = manifest_path.display(),
-                target_dir = target_dir.display(),
-            );
-            let clif_build_cmd = format!(
-                "{cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir}",
-                cargo_clif = cargo_clif.display(),
-                manifest_path = manifest_path.display(),
-                target_dir = target_dir.display(),
-            );
-
-            let bench_compile =
-                hyperfine_command(1, run_runs, Some(&clean_cmd), &llvm_build_cmd, &clif_build_cmd);
-
-            spawn_and_wait(bench_compile);
-
-            eprintln!("[BENCH RUN] ebobby/simple-raytracer");
-            fs::copy(
-                target_dir.join("debug").join("main"),
-                RelPath::BUILD.to_path(&runner.dirs).join("raytracer_cg_clif"),
-            )
-            .unwrap();
-
-            let mut bench_run =
-                hyperfine_command(0, run_runs, None, "./raytracer_cg_llvm", "./raytracer_cg_clif");
-            bench_run.current_dir(RelPath::BUILD.to_path(&runner.dirs));
-            spawn_and_wait(bench_run);
-        } else {
-            spawn_and_wait(SIMPLE_RAYTRACER.clean(&runner.target_compiler.cargo, &runner.dirs));
-            eprintln!("[BENCH COMPILE] ebobby/simple-raytracer (skipped)");
-            eprintln!("[COMPILE] ebobby/simple-raytracer");
-            spawn_and_wait(SIMPLE_RAYTRACER.build(&runner.target_compiler, &runner.dirs));
-            eprintln!("[BENCH RUN] ebobby/simple-raytracer (skipped)");
-        }
-    }),
-    TestCase::new("test.libcore", &|runner| {
-        spawn_and_wait(LIBCORE_TESTS.clean(&runner.host_compiler.cargo, &runner.dirs));
+    TestCase::custom("test.libcore", &|runner| {
+        LIBCORE_TESTS.clean(&runner.dirs);
 
         if runner.is_native {
             spawn_and_wait(LIBCORE_TESTS.test(&runner.target_compiler, &runner.dirs));
@@ -344,8 +145,8 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
             spawn_and_wait(build_cmd);
         }
     }),
-    TestCase::new("test.regex-shootout-regex-dna", &|runner| {
-        spawn_and_wait(REGEX.clean(&runner.target_compiler.cargo, &runner.dirs));
+    TestCase::custom("test.regex-shootout-regex-dna", &|runner| {
+        REGEX.clean(&runner.dirs);
 
         // newer aho_corasick versions throw a deprecation warning
         let lint_rust_flags = format!("{} --cap-lints warn", runner.target_compiler.rustflags);
@@ -364,9 +165,10 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
                 REGEX.source_dir(&runner.dirs).join("examples").join("regexdna-input.txt"),
             )
             .unwrap();
-            let expected_path =
-                REGEX.source_dir(&runner.dirs).join("examples").join("regexdna-output.txt");
-            let expected = fs::read_to_string(&expected_path).unwrap();
+            let expected = fs::read_to_string(
+                REGEX.source_dir(&runner.dirs).join("examples").join("regexdna-output.txt"),
+            )
+            .unwrap();
 
             let output = spawn_and_wait_with_input(run_cmd, input);
             // Make sure `[codegen mono items] start` doesn't poison the diff
@@ -379,27 +181,16 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
 
             let output_matches = expected.lines().eq(output.lines());
             if !output_matches {
-                let res_path = REGEX.source_dir(&runner.dirs).join("res.txt");
-                fs::write(&res_path, &output).unwrap();
-
-                if cfg!(windows) {
-                    println!("Output files don't match!");
-                    println!("Expected Output:\n{}", expected);
-                    println!("Actual Output:\n{}", output);
-                } else {
-                    let mut diff = Command::new("diff");
-                    diff.arg("-u");
-                    diff.arg(res_path);
-                    diff.arg(expected_path);
-                    spawn_and_wait(diff);
-                }
+                println!("Output files don't match!");
+                println!("Expected Output:\n{}", expected);
+                println!("Actual Output:\n{}", output);
 
                 std::process::exit(1);
             }
         }
     }),
-    TestCase::new("test.regex", &|runner| {
-        spawn_and_wait(REGEX.clean(&runner.host_compiler.cargo, &runner.dirs));
+    TestCase::custom("test.regex", &|runner| {
+        REGEX.clean(&runner.dirs);
 
         // newer aho_corasick versions throw a deprecation warning
         let lint_rust_flags = format!("{} --cap-lints warn", runner.target_compiler.rustflags);
@@ -425,8 +216,8 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
             spawn_and_wait(build_cmd);
         }
     }),
-    TestCase::new("test.portable-simd", &|runner| {
-        spawn_and_wait(PORTABLE_SIMD.clean(&runner.host_compiler.cargo, &runner.dirs));
+    TestCase::custom("test.portable-simd", &|runner| {
+        PORTABLE_SIMD.clean(&runner.dirs);
 
         let mut build_cmd = PORTABLE_SIMD.build(&runner.target_compiler, &runner.dirs);
         build_cmd.arg("--all-targets");
@@ -445,21 +236,22 @@ pub(crate) fn run_tests(
     channel: &str,
     sysroot_kind: SysrootKind,
     cg_clif_dylib: &Path,
-    host_triple: &str,
-    target_triple: &str,
+    bootstrap_host_compiler: &Compiler,
+    target_triple: String,
 ) {
-    let runner = TestRunner::new(dirs.clone(), host_triple.to_string(), target_triple.to_string());
-
     if config::get_bool("testsuite.no_sysroot") {
-        build_sysroot::build_sysroot(
+        let target_compiler = build_sysroot::build_sysroot(
             dirs,
             channel,
             SysrootKind::None,
             cg_clif_dylib,
-            &host_triple,
-            &target_triple,
+            bootstrap_host_compiler,
+            target_triple.clone(),
         );
 
+        let runner =
+            TestRunner::new(dirs.clone(), target_compiler, get_host_triple() == target_triple);
+
         BUILD_EXAMPLE_OUT_DIR.ensure_fresh(dirs);
         runner.run_testsuite(NO_SYSROOT_SUITE);
     } else {
@@ -470,26 +262,29 @@ pub(crate) fn run_tests(
     let run_extended_sysroot = config::get_bool("testsuite.extended_sysroot");
 
     if run_base_sysroot || run_extended_sysroot {
-        build_sysroot::build_sysroot(
+        let target_compiler = build_sysroot::build_sysroot(
             dirs,
             channel,
             sysroot_kind,
             cg_clif_dylib,
-            &host_triple,
-            &target_triple,
+            bootstrap_host_compiler,
+            target_triple.clone(),
         );
-    }
 
-    if run_base_sysroot {
-        runner.run_testsuite(BASE_SYSROOT_SUITE);
-    } else {
-        eprintln!("[SKIP] base_sysroot tests");
-    }
+        let runner =
+            TestRunner::new(dirs.clone(), target_compiler, get_host_triple() == target_triple);
 
-    if run_extended_sysroot {
-        runner.run_testsuite(EXTENDED_SYSROOT_SUITE);
-    } else {
-        eprintln!("[SKIP] extended_sysroot tests");
+        if run_base_sysroot {
+            runner.run_testsuite(BASE_SYSROOT_SUITE);
+        } else {
+            eprintln!("[SKIP] base_sysroot tests");
+        }
+
+        if run_extended_sysroot {
+            runner.run_testsuite(EXTENDED_SYSROOT_SUITE);
+        } else {
+            eprintln!("[SKIP] extended_sysroot tests");
+        }
     }
 }
 
@@ -497,84 +292,34 @@ struct TestRunner {
     is_native: bool,
     jit_supported: bool,
     dirs: Dirs,
-    host_compiler: Compiler,
     target_compiler: Compiler,
 }
 
 impl TestRunner {
-    pub fn new(dirs: Dirs, host_triple: String, target_triple: String) -> Self {
-        let is_native = host_triple == target_triple;
-        let jit_supported =
-            target_triple.contains("x86_64") && is_native && !host_triple.contains("windows");
-
-        let rustc_clif =
-            RelPath::DIST.to_path(&dirs).join(get_wrapper_file_name("rustc-clif", "bin"));
-        let rustdoc_clif =
-            RelPath::DIST.to_path(&dirs).join(get_wrapper_file_name("rustdoc-clif", "bin"));
-
-        let mut rustflags = env::var("RUSTFLAGS").ok().unwrap_or("".to_string());
-        let mut runner = vec![];
-
-        if !is_native {
-            match target_triple.as_str() {
-                "aarch64-unknown-linux-gnu" => {
-                    // We are cross-compiling for aarch64. Use the correct linker and run tests in qemu.
-                    rustflags = format!("-Clinker=aarch64-linux-gnu-gcc{}", rustflags);
-                    runner = vec![
-                        "qemu-aarch64".to_owned(),
-                        "-L".to_owned(),
-                        "/usr/aarch64-linux-gnu".to_owned(),
-                    ];
-                }
-                "s390x-unknown-linux-gnu" => {
-                    // We are cross-compiling for s390x. Use the correct linker and run tests in qemu.
-                    rustflags = format!("-Clinker=s390x-linux-gnu-gcc{}", rustflags);
-                    runner = vec![
-                        "qemu-s390x".to_owned(),
-                        "-L".to_owned(),
-                        "/usr/s390x-linux-gnu".to_owned(),
-                    ];
-                }
-                "x86_64-pc-windows-gnu" => {
-                    // We are cross-compiling for Windows. Run tests in wine.
-                    runner = vec!["wine".to_owned()];
-                }
-                _ => {
-                    println!("Unknown non-native platform");
-                }
-            }
+    pub fn new(dirs: Dirs, mut target_compiler: Compiler, is_native: bool) -> Self {
+        if let Ok(rustflags) = env::var("RUSTFLAGS") {
+            target_compiler.rustflags.push(' ');
+            target_compiler.rustflags.push_str(&rustflags);
+        }
+        if let Ok(rustdocflags) = env::var("RUSTDOCFLAGS") {
+            target_compiler.rustdocflags.push(' ');
+            target_compiler.rustdocflags.push_str(&rustdocflags);
         }
 
         // FIXME fix `#[linkage = "extern_weak"]` without this
-        if target_triple.contains("darwin") {
-            rustflags = format!("{} -Clink-arg=-undefined -Clink-arg=dynamic_lookup", rustflags);
+        if target_compiler.triple.contains("darwin") {
+            target_compiler.rustflags.push_str(" -Clink-arg=-undefined -Clink-arg=dynamic_lookup");
         }
 
-        let host_compiler = Compiler {
-            cargo: get_cargo_path(),
-            rustc: rustc_clif.clone(),
-            rustdoc: rustdoc_clif.clone(),
-            rustflags: String::new(),
-            rustdocflags: String::new(),
-            triple: host_triple,
-            runner: vec![],
-        };
-
-        let target_compiler = Compiler {
-            cargo: get_cargo_path(),
-            rustc: rustc_clif,
-            rustdoc: rustdoc_clif,
-            rustflags: rustflags.clone(),
-            rustdocflags: rustflags,
-            triple: target_triple,
-            runner,
-        };
-
-        Self { is_native, jit_supported, dirs, host_compiler, target_compiler }
+        let jit_supported = is_native
+            && target_compiler.triple.contains("x86_64")
+            && !target_compiler.triple.contains("windows");
+
+        Self { is_native, jit_supported, dirs, target_compiler }
     }
 
     pub fn run_testsuite(&self, tests: &[TestCase]) {
-        for &TestCase { config, func } in tests {
+        for TestCase { config, cmd } in tests {
             let (tag, testname) = config.split_once('.').unwrap();
             let tag = tag.to_uppercase();
             let is_jit_test = tag == "JIT";
@@ -586,7 +331,47 @@ impl TestRunner {
                 eprintln!("[{tag}] {testname}");
             }
 
-            func(self);
+            match *cmd {
+                TestCaseCmd::Custom { func } => func(self),
+                TestCaseCmd::BuildLib { source, crate_types } => {
+                    self.run_rustc([source, "--crate-type", crate_types]);
+                }
+                TestCaseCmd::BuildBinAndRun { source, args } => {
+                    self.run_rustc([source]);
+                    self.run_out_command(
+                        source.split('/').last().unwrap().split('.').next().unwrap(),
+                        args,
+                    );
+                }
+                TestCaseCmd::JitBin { source, args } => {
+                    let mut jit_cmd = self.rustc_command([
+                        "-Zunstable-options",
+                        "-Cllvm-args=mode=jit",
+                        "-Cprefer-dynamic",
+                        source,
+                        "--cfg",
+                        "jit",
+                    ]);
+                    if !args.is_empty() {
+                        jit_cmd.env("CG_CLIF_JIT_ARGS", args);
+                    }
+                    spawn_and_wait(jit_cmd);
+
+                    eprintln!("[JIT-lazy] {testname}");
+                    let mut jit_cmd = self.rustc_command([
+                        "-Zunstable-options",
+                        "-Cllvm-args=mode=jit-lazy",
+                        "-Cprefer-dynamic",
+                        source,
+                        "--cfg",
+                        "jit",
+                    ]);
+                    if !args.is_empty() {
+                        jit_cmd.env("CG_CLIF_JIT_ARGS", args);
+                    }
+                    spawn_and_wait(jit_cmd);
+                }
+            }
         }
     }
 
@@ -603,6 +388,9 @@ impl TestRunner {
         cmd.arg("--out-dir");
         cmd.arg(format!("{}", BUILD_EXAMPLE_OUT_DIR.to_path(&self.dirs).display()));
         cmd.arg("-Cdebuginfo=2");
+        cmd.arg("--target");
+        cmd.arg(&self.target_compiler.triple);
+        cmd.arg("-Cpanic=abort");
         cmd.args(args);
         cmd
     }
@@ -615,10 +403,7 @@ impl TestRunner {
         spawn_and_wait(self.rustc_command(args));
     }
 
-    fn run_out_command<'a, I>(&self, name: &str, args: I)
-    where
-        I: IntoIterator<Item = &'a str>,
-    {
+    fn run_out_command<'a>(&self, name: &str, args: &[&str]) {
         let mut full_cmd = vec![];
 
         // Prepend the RUN_WRAPPER's
@@ -630,7 +415,7 @@ impl TestRunner {
             BUILD_EXAMPLE_OUT_DIR.to_path(&self.dirs).join(name).to_str().unwrap().to_string(),
         );
 
-        for arg in args.into_iter() {
+        for arg in args {
             full_cmd.push(arg.to_string());
         }
 
diff --git a/compiler/rustc_codegen_cranelift/build_system/usage.txt b/compiler/rustc_codegen_cranelift/build_system/usage.txt
new file mode 100644
index 000000000..ab98ccc35
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/build_system/usage.txt
@@ -0,0 +1,35 @@
+The build system of cg_clif.
+
+USAGE:
+    ./y.rs prepare [--out-dir DIR]
+    ./y.rs build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
+    ./y.rs test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
+    ./y.rs abi-cafe [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
+    ./y.rs bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
+
+OPTIONS:
+    --debug
+            Build cg_clif and the standard library in debug mode rather than release mode.
+            Warning: An unoptimized cg_clif is very slow.
+
+    --sysroot none|clif|llvm
+            Which sysroot libraries to use:
+            `none` will not include any standard library in the sysroot.
+            `clif` will build the standard library using Cranelift.
+            `llvm` will use the pre-compiled standard library of rustc which is compiled with LLVM.
+
+    --out-dir DIR
+            Specify the directory in which the download, build and dist directories are stored.
+            By default this is the working directory.
+
+    --no-unstable-features
+            Some features are not yet ready for production usage. This option will disable these
+            features. This includes the JIT mode and inline assembly support.
+
+REQUIREMENTS:
+    * Rustup: The build system has a hard coded dependency on rustup to install the right nightly
+      version and make sure it is used where necessary.
+    * Git: `./y.rs prepare` uses git for applying patches and on Windows for downloading test repos.
+    * Curl and tar (non-Windows only): Used by `./y.rs prepare` to download a single commit for
+      repos. Git will be used to clone the whole repo when using Windows.
+    * [Hyperfine](https://github.com/sharkdp/hyperfine/): Used for benchmarking with `./y.rs bench`.
diff --git a/compiler/rustc_codegen_cranelift/build_system/utils.rs b/compiler/rustc_codegen_cranelift/build_system/utils.rs
index 2be70e8e4..abc5bab49 100644
--- a/compiler/rustc_codegen_cranelift/build_system/utils.rs
+++ b/compiler/rustc_codegen_cranelift/build_system/utils.rs
@@ -1,12 +1,13 @@
 use std::env;
 use std::fs;
-use std::io::Write;
+use std::io::{self, Write};
 use std::path::{Path, PathBuf};
 use std::process::{self, Command, Stdio};
 
 use super::path::{Dirs, RelPath};
-use super::rustc_info::{get_cargo_path, get_host_triple, get_rustc_path, get_rustdoc_path};
+use super::rustc_info::{get_cargo_path, get_rustc_path, get_rustdoc_path};
 
+#[derive(Clone, Debug)]
 pub(crate) struct Compiler {
     pub(crate) cargo: PathBuf,
     pub(crate) rustc: PathBuf,
@@ -18,27 +19,47 @@ pub(crate) struct Compiler {
 }
 
 impl Compiler {
-    pub(crate) fn host() -> Compiler {
+    pub(crate) fn bootstrap_with_triple(triple: String) -> Compiler {
         Compiler {
             cargo: get_cargo_path(),
             rustc: get_rustc_path(),
             rustdoc: get_rustdoc_path(),
             rustflags: String::new(),
             rustdocflags: String::new(),
-            triple: get_host_triple(),
+            triple,
             runner: vec![],
         }
     }
 
-    pub(crate) fn with_triple(triple: String) -> Compiler {
-        Compiler {
-            cargo: get_cargo_path(),
-            rustc: get_rustc_path(),
-            rustdoc: get_rustdoc_path(),
-            rustflags: String::new(),
-            rustdocflags: String::new(),
-            triple,
-            runner: vec![],
+    pub(crate) fn set_cross_linker_and_runner(&mut self) {
+        match self.triple.as_str() {
+            "aarch64-unknown-linux-gnu" => {
+                // We are cross-compiling for aarch64. Use the correct linker and run tests in qemu.
+                self.rustflags += " -Clinker=aarch64-linux-gnu-gcc";
+                self.rustdocflags += " -Clinker=aarch64-linux-gnu-gcc";
+                self.runner = vec![
+                    "qemu-aarch64".to_owned(),
+                    "-L".to_owned(),
+                    "/usr/aarch64-linux-gnu".to_owned(),
+                ];
+            }
+            "s390x-unknown-linux-gnu" => {
+                // We are cross-compiling for s390x. Use the correct linker and run tests in qemu.
+                self.rustflags += " -Clinker=s390x-linux-gnu-gcc";
+                self.rustdocflags += " -Clinker=s390x-linux-gnu-gcc";
+                self.runner = vec![
+                    "qemu-s390x".to_owned(),
+                    "-L".to_owned(),
+                    "/usr/s390x-linux-gnu".to_owned(),
+                ];
+            }
+            "x86_64-pc-windows-gnu" => {
+                // We are cross-compiling for Windows. Run tests in wine.
+                self.runner = vec!["wine".to_owned()];
+            }
+            _ => {
+                println!("Unknown non-native platform");
+            }
         }
     }
 }
@@ -65,6 +86,7 @@ impl CargoProject {
         RelPath::BUILD.join(self.target).to_path(dirs)
     }
 
+    #[must_use]
     fn base_cmd(&self, command: &str, cargo: &Path, dirs: &Dirs) -> Command {
         let mut cmd = Command::new(cargo);
 
@@ -72,11 +94,13 @@ impl CargoProject {
             .arg("--manifest-path")
             .arg(self.manifest_path(dirs))
             .arg("--target-dir")
-            .arg(self.target_dir(dirs));
+            .arg(self.target_dir(dirs))
+            .arg("--frozen");
 
         cmd
     }
 
+    #[must_use]
     fn build_cmd(&self, command: &str, compiler: &Compiler, dirs: &Dirs) -> Command {
         let mut cmd = self.base_cmd(command, &compiler.cargo, dirs);
 
@@ -97,17 +121,24 @@ impl CargoProject {
     }
 
     #[must_use]
-    pub(crate) fn fetch(&self, cargo: impl AsRef<Path>, dirs: &Dirs) -> Command {
+    pub(crate) fn fetch(
+        &self,
+        cargo: impl AsRef<Path>,
+        rustc: impl AsRef<Path>,
+        dirs: &Dirs,
+    ) -> Command {
         let mut cmd = Command::new(cargo.as_ref());
 
-        cmd.arg("fetch").arg("--manifest-path").arg(self.manifest_path(dirs));
+        cmd.env("RUSTC", rustc.as_ref())
+            .arg("fetch")
+            .arg("--manifest-path")
+            .arg(self.manifest_path(dirs));
 
         cmd
     }
 
-    #[must_use]
-    pub(crate) fn clean(&self, cargo: &Path, dirs: &Dirs) -> Command {
-        self.base_cmd("clean", cargo, dirs)
+    pub(crate) fn clean(&self, dirs: &Dirs) {
+        let _ = fs::remove_dir_all(self.target_dir(dirs));
     }
 
     #[must_use]
@@ -153,6 +184,23 @@ pub(crate) fn hyperfine_command(
     bench
 }
 
+#[must_use]
+pub(crate) fn git_command<'a>(repo_dir: impl Into<Option<&'a Path>>, cmd: &str) -> Command {
+    let mut git_cmd = Command::new("git");
+    git_cmd
+        .arg("-c")
+        .arg("user.name=Dummy")
+        .arg("-c")
+        .arg("user.email=dummy@example.com")
+        .arg("-c")
+        .arg("core.autocrlf=false")
+        .arg(cmd);
+    if let Some(repo_dir) = repo_dir.into() {
+        git_cmd.current_dir(repo_dir);
+    }
+    git_cmd
+}
+
 #[track_caller]
 pub(crate) fn try_hard_link(src: impl AsRef<Path>, dst: impl AsRef<Path>) {
     let src = src.as_ref();
@@ -169,6 +217,22 @@ pub(crate) fn spawn_and_wait(mut cmd: Command) {
     }
 }
 
+// Based on the retry function in rust's src/ci/shared.sh
+#[track_caller]
+pub(crate) fn retry_spawn_and_wait(tries: u64, mut cmd: Command) {
+    for i in 1..tries + 1 {
+        if i != 1 {
+            println!("Command failed. Attempt {i}/{tries}:");
+        }
+        if cmd.spawn().unwrap().wait().unwrap().success() {
+            return;
+        }
+        std::thread::sleep(std::time::Duration::from_secs(i * 5));
+    }
+    println!("The command has failed after {tries} attempts.");
+    process::exit(1);
+}
+
 #[track_caller]
 pub(crate) fn spawn_and_wait_with_input(mut cmd: Command, input: String) -> String {
     let mut child = cmd
@@ -190,6 +254,14 @@ pub(crate) fn spawn_and_wait_with_input(mut cmd: Command, input: String) -> Stri
     String::from_utf8(output.stdout).unwrap()
 }
 
+pub(crate) fn remove_dir_if_exists(path: &Path) {
+    match fs::remove_dir_all(&path) {
+        Ok(()) => {}
+        Err(err) if err.kind() == io::ErrorKind::NotFound => {}
+        Err(err) => panic!("Failed to remove {path}: {err}", path = path.display()),
+    }
+}
+
 pub(crate) fn copy_dir_recursively(from: &Path, to: &Path) {
     for entry in fs::read_dir(from).unwrap() {
         let entry = entry.unwrap();
@@ -207,5 +279,9 @@ pub(crate) fn copy_dir_recursively(from: &Path, to: &Path) {
 }
 
 pub(crate) fn is_ci() -> bool {
-    env::var("CI").as_deref() == Ok("true")
+    env::var("CI").is_ok()
+}
+
+pub(crate) fn is_ci_opt() -> bool {
+    env::var("CI_OPT").is_ok()
 }
diff --git a/compiler/rustc_codegen_cranelift/clean_all.sh b/compiler/rustc_codegen_cranelift/clean_all.sh
index 1760e5836..cdfc2e143 100755
--- a/compiler/rustc_codegen_cranelift/clean_all.sh
+++ b/compiler/rustc_codegen_cranelift/clean_all.sh
@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
 set -e
 
-rm -rf build_sysroot/{sysroot_src/,target/,compiler-builtins/,rustc_version}
-rm -rf target/ build/ dist/ perf.data{,.old} y.bin
-rm -rf download/
+rm -rf target/ download/ build/ dist/ y.bin y.bin.dSYM y.exe y.pdb
 
 # Kept for now in case someone updates their checkout of cg_clif before running clean_all.sh
 # FIXME remove at some point in the future
 rm -rf rand/ regex/ simple-raytracer/ portable-simd/ abi-checker/ abi-cafe/
+rm -rf build_sysroot/{sysroot_src/,target/,compiler-builtins/,rustc_version}
diff --git a/compiler/rustc_codegen_cranelift/config.txt b/compiler/rustc_codegen_cranelift/config.txt
index 258b67e93..d6e3924a2 100644
--- a/compiler/rustc_codegen_cranelift/config.txt
+++ b/compiler/rustc_codegen_cranelift/config.txt
@@ -44,10 +44,7 @@ aot.issue-72793
 
 testsuite.extended_sysroot
 test.rust-random/rand
-bench.simple-raytracer
 test.libcore
 test.regex-shootout-regex-dna
 test.regex
 test.portable-simd
-
-testsuite.abi-cafe
diff --git a/compiler/rustc_codegen_cranelift/patches/0001-portable-simd-Disable-unsupported-tests.patch b/compiler/rustc_codegen_cranelift/patches/0001-portable-simd-Disable-unsupported-tests.patch
deleted file mode 100644
index 89e2b61c1..000000000
--- a/compiler/rustc_codegen_cranelift/patches/0001-portable-simd-Disable-unsupported-tests.patch
+++ /dev/null
@@ -1,35 +0,0 @@
-From b742f03694b920cc14400727d54424e8e1b60928 Mon Sep 17 00:00:00 2001
-From: bjorn3 <bjorn3@users.noreply.github.com>
-Date: Thu, 18 Nov 2021 19:28:40 +0100
-Subject: [PATCH] Disable unsupported tests
-
----
- crates/core_simd/src/elements/int.rs     | 8 ++++++++
- crates/core_simd/src/elements/uint.rs    | 4 ++++
- crates/core_simd/src/masks/full_masks.rs | 6 ++++++
- crates/core_simd/src/vector.rs           | 2 ++
- crates/core_simd/tests/masks.rs          | 3 ---
- 5 files changed, 20 insertions(+), 3 deletions(-)
-
-diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs
-index e8e8f68..7173c24 100644
---- a/crates/core_simd/src/vector.rs
-+++ b/crates/core_simd/src/vector.rs
-@@ -250,6 +250,7 @@ where
-         unsafe { intrinsics::simd_cast(self) }
-     }
- 
-+    /*
-     /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector.
-     /// If an index is out-of-bounds, the lane is instead selected from the `or` vector.
-     ///
-@@ -473,6 +474,7 @@ where
-             // Cleared ☢️ *mut T Zone
-         }
-     }
-+    */
- }
- 
- impl<T, const LANES: usize> Copy for Simd<T, LANES>
--- 
-2.25.1
diff --git a/compiler/rustc_codegen_cranelift/patches/0022-sysroot-Disable-not-compiling-tests.patch b/compiler/rustc_codegen_cranelift/patches/0022-sysroot-Disable-not-compiling-tests.patch
index 8d9ee3f25..865aa833a 100644
--- a/compiler/rustc_codegen_cranelift/patches/0022-sysroot-Disable-not-compiling-tests.patch
+++ b/compiler/rustc_codegen_cranelift/patches/0022-sysroot-Disable-not-compiling-tests.patch
@@ -18,7 +18,7 @@ new file mode 100644
 index 0000000..46fd999
 --- /dev/null
 +++ b/library/core/tests/Cargo.toml
-@@ -0,0 +1,11 @@
+@@ -0,0 +1,12 @@
 +[package]
 +name = "core"
 +version = "0.0.0"
@@ -29,6 +29,7 @@ index 0000000..46fd999
 +path = "lib.rs"
 +
 +[dependencies]
-+rand = "0.7"
++rand = { version = "0.8.5", default-features = false }
++rand_xorshift = { version = "0.3.0", default-features = false }
 --
 2.21.0 (Apple Git-122)
diff --git a/compiler/rustc_codegen_cranelift/rust-toolchain b/compiler/rustc_codegen_cranelift/rust-toolchain
index d8f28dbcc..40fb54b91 100644
--- a/compiler/rustc_codegen_cranelift/rust-toolchain
+++ b/compiler/rustc_codegen_cranelift/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2022-12-13"
+channel = "nightly-2023-02-06"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]
diff --git a/compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs b/compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs
index 9362b47fa..c993430b8 100644
--- a/compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs
+++ b/compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs
@@ -26,7 +26,7 @@ fn main() {
     env::set_var("RUSTDOCFLAGS", env::var("RUSTDOCFLAGS").unwrap_or(String::new()) + &rustflags);
 
     // Ensure that the right toolchain is used
-    env::set_var("RUSTUP_TOOLCHAIN", env!("RUSTUP_TOOLCHAIN"));
+    env::set_var("RUSTUP_TOOLCHAIN", env!("TOOLCHAIN_NAME"));
 
     let args: Vec<_> = match env::args().nth(1).as_deref() {
         Some("jit") => {
diff --git a/compiler/rustc_codegen_cranelift/scripts/rustc-clif.rs b/compiler/rustc_codegen_cranelift/scripts/rustc-clif.rs
index 3abfcd8dd..c187f54a6 100644
--- a/compiler/rustc_codegen_cranelift/scripts/rustc-clif.rs
+++ b/compiler/rustc_codegen_cranelift/scripts/rustc-clif.rs
@@ -24,7 +24,7 @@ fn main() {
     }
 
     // Ensure that the right toolchain is used
-    env::set_var("RUSTUP_TOOLCHAIN", env!("RUSTUP_TOOLCHAIN"));
+    env::set_var("RUSTUP_TOOLCHAIN", env!("TOOLCHAIN_NAME"));
 
     #[cfg(unix)]
     Command::new("rustc").args(args).exec();
diff --git a/compiler/rustc_codegen_cranelift/scripts/rustdoc-clif.rs b/compiler/rustc_codegen_cranelift/scripts/rustdoc-clif.rs
index a19d72acf..a6528ac41 100644
--- a/compiler/rustc_codegen_cranelift/scripts/rustdoc-clif.rs
+++ b/compiler/rustc_codegen_cranelift/scripts/rustdoc-clif.rs
@@ -24,7 +24,7 @@ fn main() {
     }
 
     // Ensure that the right toolchain is used
-    env::set_var("RUSTUP_TOOLCHAIN", env!("RUSTUP_TOOLCHAIN"));
+    env::set_var("RUSTUP_TOOLCHAIN", env!("TOOLCHAIN_NAME"));
 
     #[cfg(unix)]
     Command::new("rustdoc").args(args).exec();
diff --git a/compiler/rustc_codegen_cranelift/scripts/rustup.sh b/compiler/rustc_codegen_cranelift/scripts/rustup.sh
index bc4c06ed7..34e3981b5 100755
--- a/compiler/rustc_codegen_cranelift/scripts/rustup.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/rustup.sh
@@ -17,10 +17,10 @@ case $1 in
         done
 
         ./clean_all.sh
-        ./y.rs prepare
 
-        (cd build_sysroot && cargo update)
+        ./y.rs prepare
 
+        (cd download/sysroot && cargo update && cargo fetch && cp Cargo.lock ../../build_sysroot/)
         ;;
     "commit")
         git add rust-toolchain build_sysroot/Cargo.lock
diff --git a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh
index 6c64b7de7..a08e80dd1 100644
--- a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh
@@ -10,7 +10,7 @@ git fetch
 git checkout -- .
 git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')"
 
-git am ../patches/*-sysroot-*.patch
+git -c user.name=Dummy -c user.email=dummy@example.com am ../patches/*-sysroot-*.patch
 
 git apply - <<EOF
 diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml
@@ -25,8 +25,8 @@ index d95b5b7f17f..00b6f0e3635 100644
 +compiler_builtins = { version = "0.1.66", features = ['rustc-dep-of-std', 'no-asm'] }
 
  [dev-dependencies]
- rand = "0.7"
- rand_xorshift = "0.2"
+ rand = { version = "0.8.5", default-features = false, features = ["alloc"] }
+ rand_xorshift = "0.3.0"
 EOF
 
 cat > config.toml <<EOF
@@ -51,7 +51,7 @@ popd
 # FIXME remove once inline asm is fully supported
 export RUSTFLAGS="$RUSTFLAGS --cfg=rustix_use_libc"
 
-export CFG_VIRTUAL_RUST_SOURCE_BASE_DIR="$(cd build_sysroot/sysroot_src; pwd)"
+export CFG_VIRTUAL_RUST_SOURCE_BASE_DIR="$(cd download/sysroot/sysroot_src; pwd)"
 
 # Allow the testsuite to use llvm tools
 host_triple=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ")
diff --git a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
index 12ecb8cf4..e14a129db 100755
--- a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
@@ -11,7 +11,7 @@ pushd rust
 command -v rg >/dev/null 2>&1 || cargo install ripgrep
 
 rm -r tests/ui/{extern/,unsized-locals/,lto/,linkage*} || true
-for test in $(rg --files-with-matches "lto|// needs-asm-support|// needs-unwind" tests/{ui,incremental}); do
+for test in $(rg --files-with-matches "lto|// needs-asm-support|// needs-unwind" tests/{codegen-units,ui,incremental}); do
   rm $test
 done
 
@@ -32,20 +32,13 @@ rm tests/incremental/issue-80691-bad-eval-cache.rs # -Cpanic=abort causes abort
 # requires compiling with -Cpanic=unwind
 rm -r tests/ui/macros/rfc-2011-nicer-assert-messages/
 rm -r tests/run-make/test-benches
+rm tests/ui/test-attrs/test-type.rs
 
 # vendor intrinsics
 rm tests/ui/sse2.rs # cpuid not supported, so sse2 not detected
 rm tests/ui/intrinsics/const-eval-select-x86_64.rs # requires x86_64 vendor intrinsics
 rm tests/ui/simd/array-type.rs # "Index argument for `simd_insert` is not a constant"
-rm tests/ui/simd/intrinsic/generic-bitmask-pass.rs # simd_bitmask unimplemented
-rm tests/ui/simd/intrinsic/generic-as.rs # simd_as unimplemented
-rm tests/ui/simd/intrinsic/generic-arithmetic-saturating-pass.rs # simd_saturating_add unimplemented
 rm tests/ui/simd/intrinsic/float-math-pass.rs # simd_fcos unimplemented
-rm tests/ui/simd/intrinsic/generic-gather-pass.rs # simd_gather unimplemented
-rm tests/ui/simd/intrinsic/generic-select-pass.rs # simd_select_bitmask unimplemented
-rm tests/ui/simd/issue-85915-simd-ptrs.rs # simd_gather unimplemented
-rm tests/ui/simd/issue-89193.rs # simd_gather unimplemented
-rm tests/ui/simd/simd-bitmask.rs # simd_bitmask unimplemented
 
 # exotic linkages
 rm tests/ui/issues/issue-33992.rs # unsupported linkages
@@ -64,10 +57,7 @@ rm tests/ui/intrinsics/intrinsic-nearby.rs # unimplemented nearbyintf32 and near
 rm tests/ui/target-feature/missing-plusminus.rs # error not implemented
 rm tests/ui/fn/dyn-fn-alignment.rs # wants a 256 byte alignment
 rm -r tests/run-make/emit-named-files # requires full --emit support
-rm tests/ui/abi/stack-probes.rs # stack probes not yet implemented
-rm tests/ui/simd/intrinsic/ptr-cast.rs # simd_expose_addr intrinsic unimplemented
 rm -r tests/run-make/repr128-dwarf # debuginfo test
-rm tests/codegen-units/item-collection/asm-sym.rs # requires support for sym in asm!()
 
 # optimization tests
 # ==================
@@ -88,6 +78,21 @@ rm tests/ui/mir/mir_raw_fat_ptr.rs # same
 rm tests/ui/consts/issue-33537.rs # same
 rm tests/ui/layout/valid_range_oob.rs # different ICE message
 
+rm tests/ui/consts/issue-miri-1910.rs # different error message
+rm tests/ui/consts/offset_ub.rs # same
+rm tests/ui/intrinsics/panic-uninitialized-zeroed.rs # same
+rm tests/ui/lint/lint-const-item-mutation.rs # same
+rm tests/ui/pattern/usefulness/doc-hidden-non-exhaustive.rs # same
+rm tests/ui/suggestions/derive-trait-for-method-call.rs # same
+rm tests/ui/typeck/issue-46112.rs # same
+
+rm tests/ui/proc-macro/crt-static.rs # extra warning about -Cpanic=abort for proc macros
+rm tests/ui/proc-macro/proc-macro-deprecated-attr.rs # same
+rm tests/ui/proc-macro/quote-debug.rs # same
+rm tests/ui/proc-macro/no-missing-docs.rs # same
+rm tests/ui/rust-2018/proc-macro-crate-in-paths.rs # same
+rm tests/ui/proc-macro/allowed-signatures.rs # same
+
 # doesn't work due to the way the rustc test suite is invoked.
 # should work when using ./x.py test the way it is intended
 # ============================================================
@@ -102,23 +107,20 @@ rm -r tests/ui/consts/missing_span_in_backtrace.rs # expects sysroot source to b
 # ============
 rm tests/incremental/spike-neg1.rs # errors out for some reason
 rm tests/incremental/spike-neg2.rs # same
-rm tests/ui/issues/issue-74564-if-expr-stack-overflow.rs # gives a stackoverflow before the backend runs
-rm tests/ui/mir/ssa-analysis-regression-50041.rs # produces ICE
-rm tests/ui/type-alias-impl-trait/assoc-projection-ice.rs # produces ICE
 
 rm tests/ui/simd/intrinsic/generic-reduction-pass.rs # simd_reduce_add_unordered doesn't accept an accumulator for integer vectors
 
-rm tests/ui/runtime/out-of-stack.rs # SIGSEGV instead of SIGABRT for some reason (#1301)
+rm tests/ui/simd/intrinsic/generic-as.rs # crash when accessing vector type filed (#1318)
+rm tests/ui/simd/simd-bitmask.rs # crash
+
+rm tests/ui/dyn-star/dyn-star-to-dyn.rs
+rm tests/ui/dyn-star/dispatch-on-pin-mut.rs
 
 # bugs in the test suite
 # ======================
 rm tests/ui/backtrace.rs # TODO warning
 rm tests/ui/simple_global_asm.rs # TODO add needs-asm-support
-rm tests/ui/test-attrs/test-type.rs # TODO panic message on stderr. correct stdout
-# not sure if this is actually a bug in the test suite, but the symbol list shows the function without leading _ for some reason
-rm -r tests/run-make/native-link-modifier-bundle
 rm tests/ui/process/nofile-limit.rs # TODO some AArch64 linking issue
-rm tests/ui/dyn-star/dispatch-on-pin-mut.rs # TODO failed assertion in vtable::get_ptr_and_method_ref
 
 rm tests/ui/stdio-is-blocking.rs # really slow with unoptimized libstd
 
diff --git a/compiler/rustc_codegen_cranelift/src/abi/comments.rs b/compiler/rustc_codegen_cranelift/src/abi/comments.rs
index 7f4619b5c..abf63e33c 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/comments.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/comments.rs
@@ -98,12 +98,12 @@ pub(super) fn add_local_place_comments<'tcx>(
         }
         CPlaceInner::VarPair(place_local, var1, var2) => {
             assert_eq!(local, place_local);
-            ("ssa", Cow::Owned(format!(",var=({}, {})", var1.index(), var2.index())))
+            ("ssa", Cow::Owned(format!("var=({}, {})", var1.index(), var2.index())))
         }
         CPlaceInner::VarLane(_local, _var, _lane) => unreachable!(),
         CPlaceInner::Addr(ptr, meta) => {
             let meta = if let Some(meta) = meta {
-                Cow::Owned(format!(",meta={}", meta))
+                Cow::Owned(format!("meta={}", meta))
             } else {
                 Cow::Borrowed("")
             };
diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
index 65cc6b437..74396a66f 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
@@ -7,6 +7,7 @@ mod returning;
 use cranelift_module::ModuleError;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::ty::layout::FnAbiOf;
+use rustc_session::Session;
 use rustc_target::abi::call::{Conv, FnAbi};
 use rustc_target::spec::abi::Abi;
 
@@ -22,7 +23,7 @@ fn clif_sig_from_fn_abi<'tcx>(
     default_call_conv: CallConv,
     fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
 ) -> Signature {
-    let call_conv = conv_to_call_conv(fn_abi.conv, default_call_conv);
+    let call_conv = conv_to_call_conv(tcx.sess, fn_abi.conv, default_call_conv);
 
     let inputs = fn_abi.args.iter().map(|arg_abi| arg_abi.get_abi_param(tcx).into_iter()).flatten();
 
@@ -33,24 +34,32 @@ fn clif_sig_from_fn_abi<'tcx>(
     Signature { params, returns, call_conv }
 }
 
-pub(crate) fn conv_to_call_conv(c: Conv, default_call_conv: CallConv) -> CallConv {
+pub(crate) fn conv_to_call_conv(sess: &Session, c: Conv, default_call_conv: CallConv) -> CallConv {
     match c {
         Conv::Rust | Conv::C => default_call_conv,
         Conv::RustCold => CallConv::Cold,
         Conv::X86_64SysV => CallConv::SystemV,
         Conv::X86_64Win64 => CallConv::WindowsFastcall,
-        Conv::ArmAapcs
-        | Conv::CCmseNonSecureCall
-        | Conv::Msp430Intr
+
+        // Should already get a back compat warning
+        Conv::X86Fastcall | Conv::X86Stdcall | Conv::X86ThisCall | Conv::X86VectorCall => {
+            default_call_conv
+        }
+
+        Conv::X86Intr => sess.fatal("x86-interrupt call conv not yet implemented"),
+
+        Conv::ArmAapcs => sess.fatal("aapcs call conv not yet implemented"),
+        Conv::CCmseNonSecureCall => {
+            sess.fatal("C-cmse-nonsecure-call call conv is not yet implemented");
+        }
+
+        Conv::Msp430Intr
         | Conv::PtxKernel
-        | Conv::X86Fastcall
-        | Conv::X86Intr
-        | Conv::X86Stdcall
-        | Conv::X86ThisCall
-        | Conv::X86VectorCall
         | Conv::AmdGpuKernel
         | Conv::AvrInterrupt
-        | Conv::AvrNonBlockingInterrupt => todo!("{:?}", c),
+        | Conv::AvrNonBlockingInterrupt => {
+            unreachable!("tried to use {c:?} call conv which only exists on an unsupported target");
+        }
     }
 }
 
@@ -161,6 +170,12 @@ fn make_local_place<'tcx>(
     layout: TyAndLayout<'tcx>,
     is_ssa: bool,
 ) -> CPlace<'tcx> {
+    if layout.is_unsized() {
+        fx.tcx.sess.span_fatal(
+            fx.mir.local_decls[local].source_info.span,
+            "unsized locals are not yet supported",
+        );
+    }
     let place = if is_ssa {
         if let rustc_target::abi::Abi::ScalarPair(_, _) = layout.abi {
             CPlace::new_var_pair(fx, local, layout)
@@ -390,9 +405,9 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     };
 
     let extra_args = &args[fn_sig.inputs().skip_binder().len()..];
-    let extra_args = fx
-        .tcx
-        .mk_type_list(extra_args.iter().map(|op_arg| fx.monomorphize(op_arg.ty(fx.mir, fx.tcx))));
+    let extra_args = fx.tcx.mk_type_list_from_iter(
+        extra_args.iter().map(|op_arg| fx.monomorphize(op_arg.ty(fx.mir, fx.tcx))),
+    );
     let fn_abi = if let Some(instance) = instance {
         RevealAllLayoutCx(fx.tcx).fn_abi_of_instance(instance, extra_args)
     } else {
diff --git a/compiler/rustc_codegen_cranelift/src/allocator.rs b/compiler/rustc_codegen_cranelift/src/allocator.rs
index 850822717..1c73957ca 100644
--- a/compiler/rustc_codegen_cranelift/src/allocator.rs
+++ b/compiler/rustc_codegen_cranelift/src/allocator.rs
@@ -70,37 +70,13 @@ fn codegen_inner(
             params: arg_tys.iter().cloned().map(AbiParam::new).collect(),
             returns: output.into_iter().map(AbiParam::new).collect(),
         };
-
-        let caller_name = format!("__rust_{}", method.name);
-        let callee_name = kind.fn_name(method.name);
-
-        let func_id = module.declare_function(&caller_name, Linkage::Export, &sig).unwrap();
-
-        let callee_func_id = module.declare_function(&callee_name, Linkage::Import, &sig).unwrap();
-
-        let mut ctx = Context::new();
-        ctx.func.signature = sig.clone();
-        {
-            let mut func_ctx = FunctionBuilderContext::new();
-            let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);
-
-            let block = bcx.create_block();
-            bcx.switch_to_block(block);
-            let args = arg_tys
-                .into_iter()
-                .map(|ty| bcx.append_block_param(block, ty))
-                .collect::<Vec<Value>>();
-
-            let callee_func_ref = module.declare_func_in_func(callee_func_id, &mut bcx.func);
-            let call_inst = bcx.ins().call(callee_func_ref, &args);
-            let results = bcx.inst_results(call_inst).to_vec(); // Clone to prevent borrow error
-
-            bcx.ins().return_(&results);
-            bcx.seal_all_blocks();
-            bcx.finalize();
-        }
-        module.define_function(func_id, &mut ctx).unwrap();
-        unwind_context.add_function(func_id, &ctx, module.isa());
+        crate::common::create_wrapper_function(
+            module,
+            unwind_context,
+            sig,
+            &format!("__rust_{}", method.name),
+            &kind.fn_name(method.name),
+        );
     }
 
     let sig = Signature {
@@ -108,36 +84,13 @@ fn codegen_inner(
         params: vec![AbiParam::new(usize_ty), AbiParam::new(usize_ty)],
         returns: vec![],
     };
-
-    let callee_name = alloc_error_handler_kind.fn_name(sym::oom);
-
-    let func_id =
-        module.declare_function("__rust_alloc_error_handler", Linkage::Export, &sig).unwrap();
-
-    let callee_func_id = module.declare_function(&callee_name, Linkage::Import, &sig).unwrap();
-
-    let mut ctx = Context::new();
-    ctx.func.signature = sig;
-    {
-        let mut func_ctx = FunctionBuilderContext::new();
-        let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);
-
-        let block = bcx.create_block();
-        bcx.switch_to_block(block);
-        let args = (&[usize_ty, usize_ty])
-            .iter()
-            .map(|&ty| bcx.append_block_param(block, ty))
-            .collect::<Vec<Value>>();
-
-        let callee_func_ref = module.declare_func_in_func(callee_func_id, &mut bcx.func);
-        bcx.ins().call(callee_func_ref, &args);
-
-        bcx.ins().trap(TrapCode::UnreachableCodeReached);
-        bcx.seal_all_blocks();
-        bcx.finalize();
-    }
-    module.define_function(func_id, &mut ctx).unwrap();
-    unwind_context.add_function(func_id, &ctx, module.isa());
+    crate::common::create_wrapper_function(
+        module,
+        unwind_context,
+        sig,
+        "__rust_alloc_error_handler",
+        &alloc_error_handler_kind.fn_name(sym::oom),
+    );
 
     let data_id = module.declare_data(OomStrategy::SYMBOL, Linkage::Export, false, false).unwrap();
     let mut data_ctx = DataContext::new();
diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs
index 89d955e8b..7f857528c 100644
--- a/compiler/rustc_codegen_cranelift/src/base.rs
+++ b/compiler/rustc_codegen_cranelift/src/base.rs
@@ -21,23 +21,6 @@ pub(crate) struct CodegenedFunction {
     func_debug_cx: Option<FunctionDebugContext>,
 }
 
-#[cfg_attr(not(feature = "jit"), allow(dead_code))]
-pub(crate) fn codegen_and_compile_fn<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    cx: &mut crate::CodegenCx,
-    cached_context: &mut Context,
-    module: &mut dyn Module,
-    instance: Instance<'tcx>,
-) {
-    let _inst_guard =
-        crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name));
-
-    let cached_func = std::mem::replace(&mut cached_context.func, Function::new());
-    let codegened_func = codegen_fn(tcx, cx, cached_func, module, instance);
-
-    compile_fn(cx, cached_context, module, codegened_func);
-}
-
 pub(crate) fn codegen_fn<'tcx>(
     tcx: TyCtxt<'tcx>,
     cx: &mut crate::CodegenCx,
@@ -47,6 +30,9 @@ pub(crate) fn codegen_fn<'tcx>(
 ) -> CodegenedFunction {
     debug_assert!(!instance.substs.needs_infer());
 
+    let symbol_name = tcx.symbol_name(instance).name.to_string();
+    let _timer = tcx.prof.generic_activity_with_arg("codegen fn", &*symbol_name);
+
     let mir = tcx.instance_mir(instance.def);
     let _mir_guard = crate::PrintOnPanic(|| {
         let mut buf = Vec::new();
@@ -58,7 +44,6 @@ pub(crate) fn codegen_fn<'tcx>(
     });
 
     // Declare function
-    let symbol_name = tcx.symbol_name(instance).name.to_string();
     let sig = get_function_sig(tcx, module.target_config().default_call_conv, instance);
     let func_id = module.declare_function(&symbol_name, Linkage::Local, &sig).unwrap();
 
@@ -112,7 +97,9 @@ pub(crate) fn codegen_fn<'tcx>(
         next_ssa_var: 0,
     };
 
-    tcx.sess.time("codegen clif ir", || codegen_fn_body(&mut fx, start_block));
+    tcx.prof.generic_activity("codegen clif ir").run(|| codegen_fn_body(&mut fx, start_block));
+    fx.bcx.seal_all_blocks();
+    fx.bcx.finalize();
 
     // Recover all necessary data from fx, before accessing func will prevent future access to it.
     let symbol_name = fx.symbol_name;
@@ -144,6 +131,9 @@ pub(crate) fn compile_fn(
     module: &mut dyn Module,
     codegened_func: CodegenedFunction,
 ) {
+    let _timer =
+        cx.profiler.generic_activity_with_arg("compile function", &*codegened_func.symbol_name);
+
     let clif_comments = codegened_func.clif_comments;
 
     // Store function in context
@@ -189,9 +179,30 @@ pub(crate) fn compile_fn(
     };
 
     // Define function
-    cx.profiler.verbose_generic_activity("define function").run(|| {
+    cx.profiler.generic_activity("define function").run(|| {
         context.want_disasm = cx.should_write_ir;
         module.define_function(codegened_func.func_id, context).unwrap();
+
+        if cx.profiler.enabled() {
+            let mut recording_args = false;
+            cx.profiler
+                .generic_activity_with_arg_recorder(
+                    "define function (clif pass timings)",
+                    |recorder| {
+                        let pass_times = cranelift_codegen::timing::take_current();
+                        // Replace newlines with | as measureme doesn't allow control characters like
+                        // newlines inside strings.
+                        recorder.record_arg(format!("{}", pass_times).replace("\n", " | "));
+                        recording_args = true;
+                    },
+                )
+                .run(|| {
+                    if recording_args {
+                        // Wait a tiny bit to ensure chrome's profiler doesn't hide the event
+                        std::thread::sleep(std::time::Duration::from_nanos(2))
+                    }
+                });
+        }
     });
 
     if cx.should_write_ir {
@@ -218,7 +229,7 @@ pub(crate) fn compile_fn(
     let isa = module.isa();
     let debug_context = &mut cx.debug_context;
     let unwind_context = &mut cx.unwind_context;
-    cx.profiler.verbose_generic_activity("generate debug info").run(|| {
+    cx.profiler.generic_activity("generate debug info").run(|| {
         if let Some(debug_context) = debug_context {
             codegened_func.func_debug_cx.unwrap().finalize(
                 debug_context,
@@ -235,7 +246,7 @@ pub(crate) fn verify_func(
     writer: &crate::pretty_clif::CommentWriter,
     func: &Function,
 ) {
-    tcx.sess.time("verify clif ir", || {
+    tcx.prof.generic_activity("verify clif ir").run(|| {
         let flags = cranelift_codegen::settings::Flags::new(cranelift_codegen::settings::builder());
         match cranelift_codegen::verify_function(&func, &flags) {
             Ok(_) => {}
@@ -271,7 +282,10 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
         fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
         return;
     }
-    fx.tcx.sess.time("codegen prelude", || crate::abi::codegen_fn_prelude(fx, start_block));
+    fx.tcx
+        .prof
+        .generic_activity("codegen prelude")
+        .run(|| crate::abi::codegen_fn_prelude(fx, start_block));
 
     for (bb, bb_data) in fx.mir.basic_blocks.iter_enumerated() {
         let block = fx.get_block(bb);
@@ -303,6 +317,9 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
         let source_info = bb_data.terminator().source_info;
         fx.set_debug_loc(source_info);
 
+        let _print_guard =
+            crate::PrintOnPanic(|| format!("terminator {:?}", bb_data.terminator().kind));
+
         match &bb_data.terminator().kind {
             TerminatorKind::Goto { target } => {
                 if let TerminatorKind::Return = fx.mir[*target].terminator().kind {
@@ -330,7 +347,12 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
             }
             TerminatorKind::Assert { cond, expected, msg, target, cleanup: _ } => {
                 if !fx.tcx.sess.overflow_checks() {
-                    if let mir::AssertKind::OverflowNeg(_) = *msg {
+                    let overflow_not_to_check = match msg {
+                        AssertKind::OverflowNeg(..) => true,
+                        AssertKind::Overflow(op, ..) => op.is_checkable(),
+                        _ => false,
+                    };
+                    if overflow_not_to_check {
                         let target = fx.get_block(*target);
                         fx.bcx.ins().jump(target, &[]);
                         continue;
@@ -429,7 +451,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                 cleanup: _,
                 from_hir_call: _,
             } => {
-                fx.tcx.sess.time("codegen call", || {
+                fx.tcx.prof.generic_activity("codegen call").run(|| {
                     crate::abi::codegen_terminator_call(
                         fx,
                         mir::SourceInfo { span: *fn_span, ..source_info },
@@ -464,7 +486,10 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                     *destination,
                 );
             }
-            TerminatorKind::Resume | TerminatorKind::Abort => {
+            TerminatorKind::Abort => {
+                codegen_panic_cannot_unwind(fx, source_info);
+            }
+            TerminatorKind::Resume => {
                 // FIXME implement unwinding
                 fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
             }
@@ -487,9 +512,6 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
             }
         };
     }
-
-    fx.bcx.seal_all_blocks();
-    fx.bcx.finalize();
 }
 
 fn codegen_stmt<'tcx>(
@@ -550,15 +572,7 @@ fn codegen_stmt<'tcx>(
                     let lhs = codegen_operand(fx, &lhs_rhs.0);
                     let rhs = codegen_operand(fx, &lhs_rhs.1);
 
-                    let res = if !fx.tcx.sess.overflow_checks() {
-                        let val =
-                            crate::num::codegen_int_binop(fx, bin_op, lhs, rhs).load_scalar(fx);
-                        let is_overflow = fx.bcx.ins().iconst(types::I8, 0);
-                        CValue::by_val_pair(val, is_overflow, lval.layout())
-                    } else {
-                        crate::num::codegen_checked_int_binop(fx, bin_op, lhs, rhs)
-                    };
-
+                    let res = crate::num::codegen_checked_int_binop(fx, bin_op, lhs, rhs);
                     lval.write_cvalue(fx, res);
                 }
                 Rvalue::UnaryOp(un_op, ref operand) => {
@@ -773,22 +787,36 @@ fn codegen_stmt<'tcx>(
                     let val = CValue::const_val(fx, fx.layout_of(fx.tcx.types.usize), val.into());
                     lval.write_cvalue(fx, val);
                 }
-                Rvalue::Aggregate(ref kind, ref operands) => match kind.as_ref() {
-                    AggregateKind::Array(_ty) => {
-                        for (i, operand) in operands.iter().enumerate() {
-                            let operand = codegen_operand(fx, operand);
-                            let index = fx.bcx.ins().iconst(fx.pointer_type, i as i64);
-                            let to = lval.place_index(fx, index);
-                            to.write_cvalue(fx, operand);
+                Rvalue::Aggregate(ref kind, ref operands) => {
+                    let (variant_index, variant_dest, active_field_index) = match **kind {
+                        mir::AggregateKind::Adt(_, variant_index, _, _, active_field_index) => {
+                            let variant_dest = lval.downcast_variant(fx, variant_index);
+                            (variant_index, variant_dest, active_field_index)
                         }
+                        _ => (VariantIdx::from_u32(0), lval, None),
+                    };
+                    if active_field_index.is_some() {
+                        assert_eq!(operands.len(), 1);
                     }
-                    _ => unreachable!("shouldn't exist at codegen {:?}", to_place_and_rval.1),
-                },
+                    for (i, operand) in operands.iter().enumerate() {
+                        let operand = codegen_operand(fx, operand);
+                        let field_index = active_field_index.unwrap_or(i);
+                        let to = if let mir::AggregateKind::Array(_) = **kind {
+                            let index = fx.bcx.ins().iconst(fx.pointer_type, field_index as i64);
+                            variant_dest.place_index(fx, index)
+                        } else {
+                            variant_dest.place_field(fx, mir::Field::new(field_index))
+                        };
+                        to.write_cvalue(fx, operand);
+                    }
+                    crate::discriminant::codegen_set_discriminant(fx, lval, variant_index);
+                }
             }
         }
         StatementKind::StorageLive(_)
         | StatementKind::StorageDead(_)
         | StatementKind::Deinit(_)
+        | StatementKind::ConstEvalCounter
         | StatementKind::Nop
         | StatementKind::FakeRead(..)
         | StatementKind::Retag { .. }
@@ -826,7 +854,7 @@ fn codegen_stmt<'tcx>(
 fn codegen_array_len<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, place: CPlace<'tcx>) -> Value {
     match *place.layout().ty.kind() {
         ty::Array(_elem_ty, len) => {
-            let len = fx.monomorphize(len).eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64;
+            let len = fx.monomorphize(len).eval_target_usize(fx.tcx, ParamEnv::reveal_all()) as i64;
             fx.bcx.ins().iconst(fx.pointer_type, len)
         }
         ty::Slice(_elem_ty) => {
@@ -932,7 +960,28 @@ pub(crate) fn codegen_panic<'tcx>(
     codegen_panic_inner(fx, rustc_hir::LangItem::Panic, &args, source_info.span);
 }
 
-pub(crate) fn codegen_panic_inner<'tcx>(
+pub(crate) fn codegen_panic_nounwind<'tcx>(
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
+    msg_str: &str,
+    source_info: mir::SourceInfo,
+) {
+    let msg_ptr = fx.anonymous_str(msg_str);
+    let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap());
+    let args = [msg_ptr, msg_len];
+
+    codegen_panic_inner(fx, rustc_hir::LangItem::PanicNounwind, &args, source_info.span);
+}
+
+pub(crate) fn codegen_panic_cannot_unwind<'tcx>(
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
+    source_info: mir::SourceInfo,
+) {
+    let args = [];
+
+    codegen_panic_inner(fx, rustc_hir::LangItem::PanicCannotUnwind, &args, source_info.span);
+}
+
+fn codegen_panic_inner<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     lang_item: rustc_hir::LangItem,
     args: &[Value],
@@ -949,11 +998,7 @@ pub(crate) fn codegen_panic_inner<'tcx>(
 
     fx.lib_call(
         &*symbol_name,
-        vec![
-            AbiParam::new(fx.pointer_type),
-            AbiParam::new(fx.pointer_type),
-            AbiParam::new(fx.pointer_type),
-        ],
+        args.iter().map(|&arg| AbiParam::new(fx.bcx.func.dfg.value_type(arg))).collect(),
         vec![],
         args,
     );
diff --git a/compiler/rustc_codegen_cranelift/src/codegen_i128.rs b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs
index 638b2d573..40bfe7077 100644
--- a/compiler/rustc_codegen_cranelift/src/codegen_i128.rs
+++ b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs
@@ -56,7 +56,7 @@ pub(crate) fn maybe_codegen<'tcx>(
                     Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty))
                 }
             } else {
-                let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter());
+                let out_ty = fx.tcx.mk_tup(&[lhs.layout().ty, fx.tcx.types.bool]);
                 let oflow = CPlace::new_stack_slot(fx, fx.layout_of(fx.tcx.types.i32));
                 let lhs = lhs.load_scalar(fx);
                 let rhs = rhs.load_scalar(fx);
@@ -78,7 +78,7 @@ pub(crate) fn maybe_codegen<'tcx>(
         }
         BinOp::Add | BinOp::Sub | BinOp::Mul => {
             assert!(checked);
-            let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter());
+            let out_ty = fx.tcx.mk_tup(&[lhs.layout().ty, fx.tcx.types.bool]);
             let out_place = CPlace::new_stack_slot(fx, fx.layout_of(out_ty));
             let (param_types, args) = if fx.tcx.sess.target.is_like_windows {
                 let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs
index 2dcd42fbd..722e2754e 100644
--- a/compiler/rustc_codegen_cranelift/src/common.rs
+++ b/compiler/rustc_codegen_cranelift/src/common.rs
@@ -35,7 +35,8 @@ pub(crate) fn scalar_to_clif_type(tcx: TyCtxt<'_>, scalar: Scalar) -> Type {
         },
         Primitive::F32 => types::F32,
         Primitive::F64 => types::F64,
-        Primitive::Pointer => pointer_ty(tcx),
+        // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
+        Primitive::Pointer(_) => pointer_ty(tcx),
     }
 }
 
@@ -167,6 +168,15 @@ pub(crate) fn codegen_icmp_imm(
     }
 }
 
+pub(crate) fn codegen_bitcast(fx: &mut FunctionCx<'_, '_, '_>, dst_ty: Type, val: Value) -> Value {
+    let mut flags = MemFlags::new();
+    flags.set_endianness(match fx.tcx.data_layout.endian {
+        rustc_target::abi::Endian::Big => cranelift_codegen::ir::Endianness::Big,
+        rustc_target::abi::Endian::Little => cranelift_codegen::ir::Endianness::Little,
+    });
+    fx.bcx.ins().bitcast(dst_ty, flags, val)
+}
+
 pub(crate) fn type_zero_value(bcx: &mut FunctionBuilder<'_>, ty: Type) -> Value {
     if ty == types::I128 {
         let zero = bcx.ins().iconst(types::I64, 0);
@@ -244,6 +254,44 @@ pub(crate) fn type_sign(ty: Ty<'_>) -> bool {
     }
 }
 
+pub(crate) fn create_wrapper_function(
+    module: &mut dyn Module,
+    unwind_context: &mut UnwindContext,
+    sig: Signature,
+    wrapper_name: &str,
+    callee_name: &str,
+) {
+    let wrapper_func_id = module.declare_function(wrapper_name, Linkage::Export, &sig).unwrap();
+    let callee_func_id = module.declare_function(callee_name, Linkage::Import, &sig).unwrap();
+
+    let mut ctx = Context::new();
+    ctx.func.signature = sig;
+    {
+        let mut func_ctx = FunctionBuilderContext::new();
+        let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);
+
+        let block = bcx.create_block();
+        bcx.switch_to_block(block);
+        let func = &mut bcx.func.stencil;
+        let args = func
+            .signature
+            .params
+            .iter()
+            .map(|param| func.dfg.append_block_param(block, param.value_type))
+            .collect::<Vec<Value>>();
+
+        let callee_func_ref = module.declare_func_in_func(callee_func_id, &mut bcx.func);
+        let call_inst = bcx.ins().call(callee_func_ref, &args);
+        let results = bcx.inst_results(call_inst).to_vec(); // Clone to prevent borrow error
+
+        bcx.ins().return_(&results);
+        bcx.seal_all_blocks();
+        bcx.finalize();
+    }
+    module.define_function(wrapper_func_id, &mut ctx).unwrap();
+    unwind_context.add_function(wrapper_func_id, &ctx, module.isa());
+}
+
 pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
     pub(crate) cx: &'clif mut crate::CodegenCx,
     pub(crate) module: &'m mut dyn Module,
@@ -326,7 +374,7 @@ impl<'tcx> HasTargetSpec for FunctionCx<'_, '_, 'tcx> {
 impl<'tcx> FunctionCx<'_, '_, 'tcx> {
     pub(crate) fn monomorphize<T>(&self, value: T) -> T
     where
-        T: TypeFoldable<'tcx> + Copy,
+        T: TypeFoldable<TyCtxt<'tcx>> + Copy,
     {
         self.instance.subst_mir_and_normalize_erasing_regions(
             self.tcx,
diff --git a/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs b/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs
index c6a247cf5..8a53baa76 100644
--- a/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs
+++ b/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs
@@ -1,14 +1,33 @@
+#[cfg(all(unix, feature = "jit"))]
+use std::ffi::c_int;
+#[cfg(feature = "jit")]
+use std::ffi::c_void;
+
+// FIXME replace with core::ffi::c_size_t once stablized
+#[allow(non_camel_case_types)]
+#[cfg(feature = "jit")]
+type size_t = usize;
+
 macro_rules! builtin_functions {
-    ($register:ident; $(fn $name:ident($($arg_name:ident: $arg_ty:ty),*) -> $ret_ty:ty;)*) => {
+    (
+        $register:ident;
+        $(
+            $(#[$attr:meta])?
+            fn $name:ident($($arg_name:ident: $arg_ty:ty),*) -> $ret_ty:ty;
+        )*
+    ) => {
         #[cfg(feature = "jit")]
         #[allow(improper_ctypes)]
         extern "C" {
-            $(fn $name($($arg_name: $arg_ty),*) -> $ret_ty;)*
+            $(
+                $(#[$attr])?
+                fn $name($($arg_name: $arg_ty),*) -> $ret_ty;
+            )*
         }
 
         #[cfg(feature = "jit")]
         pub(crate) fn $register(builder: &mut cranelift_jit::JITBuilder) {
-            for (name, val) in [$((stringify!($name), $name as *const u8)),*] {
+            for (name, val) in [$($(#[$attr])? (stringify!($name), $name as *const u8)),*] {
                 builder.symbol(name, val);
             }
         }
@@ -40,4 +59,17 @@ builtin_functions! {
     fn __fixdfti(f: f64) -> i128;
     fn __fixunssfti(f: f32) -> u128;
     fn __fixunsdfti(f: f64) -> u128;
+
+    // allocator
+    // NOTE: These need to be mentioned here despite not being part of compiler_builtins because
+    // newer glibc resolve dlsym("malloc") to libc.so despite the override in the rustc binary to
+    // use jemalloc. Libraries opened with dlopen still get the jemalloc version, causing multiple
+    // allocators to be mixed, resulting in a crash.
+    fn calloc(nobj: size_t, size: size_t) -> *mut c_void;
+    #[cfg(unix)]
+    fn posix_memalign(memptr: *mut *mut c_void, align: size_t, size: size_t) -> c_int;
+    fn malloc(size: size_t) -> *mut c_void;
+    fn realloc(p: *mut c_void, size: size_t) -> *mut c_void;
+    fn free(p: *mut c_void) -> ();
+
 }
diff --git a/compiler/rustc_codegen_cranelift/src/config.rs b/compiler/rustc_codegen_cranelift/src/config.rs
index 45522fb1a..263401e1c 100644
--- a/compiler/rustc_codegen_cranelift/src/config.rs
+++ b/compiler/rustc_codegen_cranelift/src/config.rs
@@ -42,12 +42,6 @@ pub struct BackendConfig {
     /// Defaults to the value of `CG_CLIF_JIT_ARGS`.
     pub jit_args: Vec<String>,
 
-    /// Display the time it took to perform codegen for a crate.
-    ///
-    /// Defaults to true when the `CG_CLIF_DISPLAY_CG_TIME` env var is set to 1 or false otherwise.
-    /// Can be set using `-Cllvm-args=display_cg_time=...`.
-    pub display_cg_time: bool,
-
     /// Enable the Cranelift ir verifier for all compilation passes. If not set it will only run
     /// once before passing the clif ir to Cranelift for compilation.
     ///
@@ -73,7 +67,6 @@ impl Default for BackendConfig {
                 let args = std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new());
                 args.split(' ').map(|arg| arg.to_string()).collect()
             },
-            display_cg_time: bool_env_var("CG_CLIF_DISPLAY_CG_TIME"),
             enable_verifier: cfg!(debug_assertions) || bool_env_var("CG_CLIF_ENABLE_VERIFIER"),
             disable_incr_cache: bool_env_var("CG_CLIF_DISABLE_INCR_CACHE"),
         }
@@ -92,7 +85,6 @@ impl BackendConfig {
             if let Some((name, value)) = opt.split_once('=') {
                 match name {
                     "mode" => config.codegen_mode = value.parse()?,
-                    "display_cg_time" => config.display_cg_time = parse_bool(name, value)?,
                     "enable_verifier" => config.enable_verifier = parse_bool(name, value)?,
                     "disable_incr_cache" => config.disable_incr_cache = parse_bool(name, value)?,
                     _ => return Err(format!("Unknown option `{}`", name)),
diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs
index 51450897b..49c4f1aaa 100644
--- a/compiler/rustc_codegen_cranelift/src/constant.rs
+++ b/compiler/rustc_codegen_cranelift/src/constant.rs
@@ -530,6 +530,7 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
                         | StatementKind::Retag(_, _)
                         | StatementKind::AscribeUserType(_, _)
                         | StatementKind::Coverage(_)
+                        | StatementKind::ConstEvalCounter
                         | StatementKind::Nop => {}
                     }
                 }
diff --git a/compiler/rustc_codegen_cranelift/src/cranelift_native.rs b/compiler/rustc_codegen_cranelift/src/cranelift_native.rs
new file mode 100644
index 000000000..6c4efca44
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/src/cranelift_native.rs
@@ -0,0 +1,248 @@
+// Vendored from https://github.com/bytecodealliance/wasmtime/blob/b58a197d33f044193c3d608010f5e6ec394ac07e/cranelift/native/src/lib.rs
+// which is licensed as
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// unlike rustc_codegen_cranelift itself. Also applies a small change to remove #![cfg_attr] that
+// rust's CI complains about and to fix formatting to match rustc.
+// FIXME revert back to the external crate with Cranelift 0.93
+#![allow(warnings)]
+
+//! Performs autodetection of the host for the purposes of running
+//! Cranelift to generate code to run on the same machine.
+
+#![deny(missing_docs, trivial_numeric_casts, unused_extern_crates, unstable_features)]
+#![warn(unused_import_braces)]
+
+use cranelift_codegen::isa;
+use target_lexicon::Triple;
+
+/// Return an `isa` builder configured for the current host
+/// machine, or `Err(())` if the host machine is not supported
+/// in the current configuration.
+pub fn builder() -> Result<isa::Builder, &'static str> {
+    builder_with_options(true)
+}
+
+/// Return an `isa` builder configured for the current host
+/// machine, or `Err(())` if the host machine is not supported
+/// in the current configuration.
+///
+/// Selects the given backend variant specifically; this is
+/// useful when more than oen backend exists for a given target
+/// (e.g., on x86-64).
+pub fn builder_with_options(infer_native_flags: bool) -> Result<isa::Builder, &'static str> {
+    let mut isa_builder = isa::lookup(Triple::host()).map_err(|err| match err {
+        isa::LookupError::SupportDisabled => "support for architecture disabled at compile time",
+        isa::LookupError::Unsupported => "unsupported architecture",
+    })?;
+
+    #[cfg(target_arch = "x86_64")]
+    {
+        use cranelift_codegen::settings::Configurable;
+
+        if !std::is_x86_feature_detected!("sse2") {
+            return Err("x86 support requires SSE2");
+        }
+
+        if !infer_native_flags {
+            return Ok(isa_builder);
+        }
+
+        // These are temporarily enabled by default (see #3810 for
+        // more) so that a default-constructed `Flags` can work with
+        // default Wasmtime features. Otherwise, the user must
+        // explicitly use native flags or turn these on when on x86-64
+        // platforms to avoid a configuration panic. In order for the
+        // "enable if detected" logic below to work, we must turn them
+        // *off* (differing from the default) and then re-enable below
+        // if present.
+        isa_builder.set("has_sse3", "false").unwrap();
+        isa_builder.set("has_ssse3", "false").unwrap();
+        isa_builder.set("has_sse41", "false").unwrap();
+        isa_builder.set("has_sse42", "false").unwrap();
+
+        if std::is_x86_feature_detected!("sse3") {
+            isa_builder.enable("has_sse3").unwrap();
+        }
+        if std::is_x86_feature_detected!("ssse3") {
+            isa_builder.enable("has_ssse3").unwrap();
+        }
+        if std::is_x86_feature_detected!("sse4.1") {
+            isa_builder.enable("has_sse41").unwrap();
+        }
+        if std::is_x86_feature_detected!("sse4.2") {
+            isa_builder.enable("has_sse42").unwrap();
+        }
+        if std::is_x86_feature_detected!("popcnt") {
+            isa_builder.enable("has_popcnt").unwrap();
+        }
+        if std::is_x86_feature_detected!("avx") {
+            isa_builder.enable("has_avx").unwrap();
+        }
+        if std::is_x86_feature_detected!("avx2") {
+            isa_builder.enable("has_avx2").unwrap();
+        }
+        if std::is_x86_feature_detected!("fma") {
+            isa_builder.enable("has_fma").unwrap();
+        }
+        if std::is_x86_feature_detected!("bmi1") {
+            isa_builder.enable("has_bmi1").unwrap();
+        }
+        if std::is_x86_feature_detected!("bmi2") {
+            isa_builder.enable("has_bmi2").unwrap();
+        }
+        if std::is_x86_feature_detected!("avx512bitalg") {
+            isa_builder.enable("has_avx512bitalg").unwrap();
+        }
+        if std::is_x86_feature_detected!("avx512dq") {
+            isa_builder.enable("has_avx512dq").unwrap();
+        }
+        if std::is_x86_feature_detected!("avx512f") {
+            isa_builder.enable("has_avx512f").unwrap();
+        }
+        if std::is_x86_feature_detected!("avx512vl") {
+            isa_builder.enable("has_avx512vl").unwrap();
+        }
+        if std::is_x86_feature_detected!("avx512vbmi") {
+            isa_builder.enable("has_avx512vbmi").unwrap();
+        }
+        if std::is_x86_feature_detected!("lzcnt") {
+            isa_builder.enable("has_lzcnt").unwrap();
+        }
+    }
+
+    #[cfg(target_arch = "aarch64")]
+    {
+        use cranelift_codegen::settings::Configurable;
+
+        if !infer_native_flags {
+            return Ok(isa_builder);
+        }
+
+        if std::arch::is_aarch64_feature_detected!("lse") {
+            isa_builder.enable("has_lse").unwrap();
+        }
+
+        if std::arch::is_aarch64_feature_detected!("paca") {
+            isa_builder.enable("has_pauth").unwrap();
+        }
+
+        if cfg!(target_os = "macos") {
+            // Pointer authentication is always available on Apple Silicon.
+            isa_builder.enable("sign_return_address").unwrap();
+            // macOS enforces the use of the B key for return addresses.
+            isa_builder.enable("sign_return_address_with_bkey").unwrap();
+        }
+    }
+
+    // There is no is_s390x_feature_detected macro yet, so for now
+    // we use getauxval from the libc crate directly.
+    #[cfg(all(target_arch = "s390x", target_os = "linux"))]
+    {
+        use cranelift_codegen::settings::Configurable;
+
+        if !infer_native_flags {
+            return Ok(isa_builder);
+        }
+
+        let v = unsafe { libc::getauxval(libc::AT_HWCAP) };
+        const HWCAP_S390X_VXRS_EXT2: libc::c_ulong = 32768;
+        if (v & HWCAP_S390X_VXRS_EXT2) != 0 {
+            isa_builder.enable("has_vxrs_ext2").unwrap();
+            // There is no separate HWCAP bit for mie2, so assume
+            // that any machine with vxrs_ext2 also has mie2.
+            isa_builder.enable("has_mie2").unwrap();
+        }
+    }
+
+    // `is_riscv_feature_detected` is nightly only for now, use
+    // getauxval from the libc crate directly as a temporary measure.
+    #[cfg(all(target_arch = "riscv64", target_os = "linux"))]
+    {
+        use cranelift_codegen::settings::Configurable;
+
+        if !infer_native_flags {
+            return Ok(isa_builder);
+        }
+
+        let v = unsafe { libc::getauxval(libc::AT_HWCAP) };
+
+        const HWCAP_RISCV_EXT_A: libc::c_ulong = 1 << (b'a' - b'a');
+        const HWCAP_RISCV_EXT_C: libc::c_ulong = 1 << (b'c' - b'a');
+        const HWCAP_RISCV_EXT_D: libc::c_ulong = 1 << (b'd' - b'a');
+        const HWCAP_RISCV_EXT_F: libc::c_ulong = 1 << (b'f' - b'a');
+        const HWCAP_RISCV_EXT_M: libc::c_ulong = 1 << (b'm' - b'a');
+        const HWCAP_RISCV_EXT_V: libc::c_ulong = 1 << (b'v' - b'a');
+
+        if (v & HWCAP_RISCV_EXT_A) != 0 {
+            isa_builder.enable("has_a").unwrap();
+        }
+
+        if (v & HWCAP_RISCV_EXT_C) != 0 {
+            isa_builder.enable("has_c").unwrap();
+        }
+
+        if (v & HWCAP_RISCV_EXT_D) != 0 {
+            isa_builder.enable("has_d").unwrap();
+        }
+
+        if (v & HWCAP_RISCV_EXT_F) != 0 {
+            isa_builder.enable("has_f").unwrap();
+
+            // TODO: There doesn't seem to be a bit associated with this extension
+            // rust enables it with the `f` extension:
+            // https://github.com/rust-lang/stdarch/blob/790411f93c4b5eada3c23abb4c9a063fb0b24d99/crates/std_detect/src/detect/os/linux/riscv.rs#L43
+            isa_builder.enable("has_zicsr").unwrap();
+        }
+
+        if (v & HWCAP_RISCV_EXT_M) != 0 {
+            isa_builder.enable("has_m").unwrap();
+        }
+
+        if (v & HWCAP_RISCV_EXT_V) != 0 {
+            isa_builder.enable("has_v").unwrap();
+        }
+
+        // TODO: ZiFencei does not have a bit associated with it
+        // TODO: Zbkb does not have a bit associated with it
+    }
+
+    // squelch warnings about unused mut/variables on some platforms.
+    drop(&mut isa_builder);
+    drop(infer_native_flags);
+
+    Ok(isa_builder)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::builder;
+    use cranelift_codegen::isa::CallConv;
+    use cranelift_codegen::settings;
+
+    #[test]
+    fn test() {
+        if let Ok(isa_builder) = builder() {
+            let flag_builder = settings::builder();
+            let isa = isa_builder.finish(settings::Flags::new(flag_builder)).unwrap();
+
+            if cfg!(all(target_os = "macos", target_arch = "aarch64")) {
+                assert_eq!(isa.default_call_conv(), CallConv::AppleAarch64);
+            } else if cfg!(any(unix, target_os = "nebulet")) {
+                assert_eq!(isa.default_call_conv(), CallConv::SystemV);
+            } else if cfg!(windows) {
+                assert_eq!(isa.default_call_conv(), CallConv::WindowsFastcall);
+            }
+
+            if cfg!(target_pointer_width = "64") {
+                assert_eq!(isa.pointer_bits(), 64);
+            } else if cfg!(target_pointer_width = "32") {
+                assert_eq!(isa.pointer_bits(), 32);
+            } else if cfg!(target_pointer_width = "16") {
+                assert_eq!(isa.pointer_bits(), 16);
+            }
+        }
+    }
+}
+
+/// Version number of this crate.
+pub const VERSION: &str = env!("CARGO_PKG_VERSION");
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
index 28fbcb15b..3a7421d8b 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
@@ -20,6 +20,14 @@ use indexmap::IndexSet;
 pub(crate) use emit::{DebugReloc, DebugRelocName};
 pub(crate) use unwind::UnwindContext;
 
+pub(crate) fn producer() -> String {
+    format!(
+        "cg_clif (rustc {}, cranelift {})",
+        rustc_interface::util::rustc_version_str().unwrap_or("unknown version"),
+        cranelift_codegen::VERSION,
+    )
+}
+
 pub(crate) struct DebugContext {
     endian: RunTimeEndian,
 
@@ -57,11 +65,7 @@ impl DebugContext {
 
         let mut dwarf = DwarfUnit::new(encoding);
 
-        let producer = format!(
-            "cg_clif (rustc {}, cranelift {})",
-            rustc_interface::util::rustc_version_str().unwrap_or("unknown version"),
-            cranelift_codegen::VERSION,
-        );
+        let producer = producer();
         let comp_dir = tcx
             .sess
             .opts
diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs
index f873561c1..7c6fd9f6f 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs
@@ -108,6 +108,8 @@ impl OngoingCodegen {
 
         self.concurrency_limiter.finished();
 
+        sess.abort_if_errors();
+
         (
             CodegenResults {
                 modules,
@@ -169,10 +171,22 @@ fn emit_cgu(
 fn emit_module(
     output_filenames: &OutputFilenames,
     prof: &SelfProfilerRef,
-    object: cranelift_object::object::write::Object<'_>,
+    mut object: cranelift_object::object::write::Object<'_>,
     kind: ModuleKind,
     name: String,
 ) -> Result<CompiledModule, String> {
+    if object.format() == cranelift_object::object::BinaryFormat::Elf {
+        let comment_section = object.add_section(
+            Vec::new(),
+            b".comment".to_vec(),
+            cranelift_object::object::SectionKind::OtherString,
+        );
+        let mut producer = vec![0];
+        producer.extend(crate::debuginfo::producer().as_bytes());
+        producer.push(0);
+        object.set_section_data(comment_section, producer, 1);
+    }
+
     let tmp_file = output_filenames.temp_path(OutputType::Object, Some(&name));
     let mut file = match File::create(&tmp_file) {
         Ok(file) => file,
@@ -234,17 +248,13 @@ fn reuse_workproduct_for_cgu(
             dwarf_object: None,
             bytecode: None,
         },
-        module_global_asm: if has_global_asm {
-            Some(CompiledModule {
-                name: cgu.name().to_string(),
-                kind: ModuleKind::Regular,
-                object: Some(obj_out_global_asm),
-                dwarf_object: None,
-                bytecode: None,
-            })
-        } else {
-            None
-        },
+        module_global_asm: has_global_asm.then(|| CompiledModule {
+            name: cgu.name().to_string(),
+            kind: ModuleKind::Regular,
+            object: Some(obj_out_global_asm),
+            dwarf_object: None,
+            bytecode: None,
+        }),
         existing_work_product: Some((cgu.work_product_id(), work_product)),
     })
 }
@@ -258,25 +268,25 @@ fn module_codegen(
         ConcurrencyLimiterToken,
     ),
 ) -> OngoingModuleCodegen {
-    let (cgu_name, mut cx, mut module, codegened_functions) = tcx.sess.time("codegen cgu", || {
-        let cgu = tcx.codegen_unit(cgu_name);
-        let mono_items = cgu.items_in_deterministic_order(tcx);
-
-        let mut module = make_module(tcx.sess, &backend_config, cgu_name.as_str().to_string());
-
-        let mut cx = crate::CodegenCx::new(
-            tcx,
-            backend_config.clone(),
-            module.isa(),
-            tcx.sess.opts.debuginfo != DebugInfo::None,
-            cgu_name,
-        );
-        super::predefine_mono_items(tcx, &mut module, &mono_items);
-        let mut codegened_functions = vec![];
-        for (mono_item, _) in mono_items {
-            match mono_item {
-                MonoItem::Fn(inst) => {
-                    tcx.sess.time("codegen fn", || {
+    let (cgu_name, mut cx, mut module, codegened_functions) =
+        tcx.prof.verbose_generic_activity_with_arg("codegen cgu", cgu_name.as_str()).run(|| {
+            let cgu = tcx.codegen_unit(cgu_name);
+            let mono_items = cgu.items_in_deterministic_order(tcx);
+
+            let mut module = make_module(tcx.sess, &backend_config, cgu_name.as_str().to_string());
+
+            let mut cx = crate::CodegenCx::new(
+                tcx,
+                backend_config.clone(),
+                module.isa(),
+                tcx.sess.opts.debuginfo != DebugInfo::None,
+                cgu_name,
+            );
+            super::predefine_mono_items(tcx, &mut module, &mono_items);
+            let mut codegened_functions = vec![];
+            for (mono_item, _) in mono_items {
+                match mono_item {
+                    MonoItem::Fn(inst) => {
                         let codegened_function = crate::base::codegen_fn(
                             tcx,
                             &mut cx,
@@ -285,53 +295,68 @@ fn module_codegen(
                             inst,
                         );
                         codegened_functions.push(codegened_function);
-                    });
-                }
-                MonoItem::Static(def_id) => {
-                    crate::constant::codegen_static(tcx, &mut module, def_id)
-                }
-                MonoItem::GlobalAsm(item_id) => {
-                    crate::global_asm::codegen_global_asm_item(tcx, &mut cx.global_asm, item_id);
+                    }
+                    MonoItem::Static(def_id) => {
+                        crate::constant::codegen_static(tcx, &mut module, def_id)
+                    }
+                    MonoItem::GlobalAsm(item_id) => {
+                        crate::global_asm::codegen_global_asm_item(
+                            tcx,
+                            &mut cx.global_asm,
+                            item_id,
+                        );
+                    }
                 }
             }
-        }
-        crate::main_shim::maybe_create_entry_wrapper(
-            tcx,
-            &mut module,
-            &mut cx.unwind_context,
-            false,
-            cgu.is_primary(),
-        );
+            crate::main_shim::maybe_create_entry_wrapper(
+                tcx,
+                &mut module,
+                &mut cx.unwind_context,
+                false,
+                cgu.is_primary(),
+            );
 
-        let cgu_name = cgu.name().as_str().to_owned();
+            let cgu_name = cgu.name().as_str().to_owned();
 
-        (cgu_name, cx, module, codegened_functions)
-    });
+            (cgu_name, cx, module, codegened_functions)
+        });
 
     OngoingModuleCodegen::Async(std::thread::spawn(move || {
-        cx.profiler.clone().verbose_generic_activity("compile functions").run(|| {
-            let mut cached_context = Context::new();
-            for codegened_func in codegened_functions {
-                crate::base::compile_fn(&mut cx, &mut cached_context, &mut module, codegened_func);
-            }
-        });
+        cx.profiler.clone().verbose_generic_activity_with_arg("compile functions", &*cgu_name).run(
+            || {
+                let mut cached_context = Context::new();
+                for codegened_func in codegened_functions {
+                    crate::base::compile_fn(
+                        &mut cx,
+                        &mut cached_context,
+                        &mut module,
+                        codegened_func,
+                    );
+                }
+            },
+        );
 
-        let global_asm_object_file =
-            cx.profiler.verbose_generic_activity("compile assembly").run(|| {
+        let global_asm_object_file = cx
+            .profiler
+            .verbose_generic_activity_with_arg("compile assembly", &*cgu_name)
+            .run(|| {
                 crate::global_asm::compile_global_asm(&global_asm_config, &cgu_name, &cx.global_asm)
             })?;
 
-        let codegen_result = cx.profiler.verbose_generic_activity("write object file").run(|| {
-            emit_cgu(
-                &global_asm_config.output_filenames,
-                &cx.profiler,
-                cgu_name,
-                module,
-                cx.debug_context,
-                cx.unwind_context,
-                global_asm_object_file,
-            )
-        });
+        let codegen_result = cx
+            .profiler
+            .verbose_generic_activity_with_arg("write object file", &*cgu_name)
+            .run(|| {
+                emit_cgu(
+                    &global_asm_config.output_filenames,
+                    &cx.profiler,
+                    cgu_name,
+                    module,
+                    cx.debug_context,
+                    cx.unwind_context,
+                    global_asm_object_file,
+                )
+            });
         std::mem::drop(token);
         codegen_result
     }))
@@ -361,7 +386,7 @@ pub(crate) fn run_aot(
 
     let mut concurrency_limiter = ConcurrencyLimiter::new(tcx.sess, cgus.len());
 
-    let modules = super::time(tcx, backend_config.display_cg_time, "codegen mono items", || {
+    let modules = tcx.sess.time("codegen mono items", || {
         cgus.iter()
             .map(|cgu| {
                 let cgu_reuse = if backend_config.disable_incr_cache {
@@ -399,8 +424,6 @@ pub(crate) fn run_aot(
             .collect::<Vec<_>>()
     });
 
-    tcx.sess.abort_if_errors();
-
     let mut allocator_module = make_module(tcx.sess, &backend_config, "allocator_shim".to_string());
     let mut allocator_unwind_context = UnwindContext::new(allocator_module.isa(), true);
     let created_alloc_shim =
@@ -425,7 +448,6 @@ pub(crate) fn run_aot(
     };
 
     let metadata_module = if need_metadata_module {
-        let _timer = tcx.prof.generic_activity("codegen crate metadata");
         let (metadata_cgu_name, tmp_file) = tcx.sess.time("write compressed metadata", || {
             use rustc_middle::mir::mono::CodegenUnitNameBuilder;
 
diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
index be1b8c9ea..8b5a2da2c 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
@@ -121,22 +121,20 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
         .into_iter()
         .collect::<Vec<(_, (_, _))>>();
 
-    super::time(tcx, backend_config.display_cg_time, "codegen mono items", || {
+    tcx.sess.time("codegen mono items", || {
         super::predefine_mono_items(tcx, &mut jit_module, &mono_items);
         for (mono_item, _) in mono_items {
             match mono_item {
                 MonoItem::Fn(inst) => match backend_config.codegen_mode {
                     CodegenMode::Aot => unreachable!(),
                     CodegenMode::Jit => {
-                        tcx.sess.time("codegen fn", || {
-                            crate::base::codegen_and_compile_fn(
-                                tcx,
-                                &mut cx,
-                                &mut cached_context,
-                                &mut jit_module,
-                                inst,
-                            )
-                        });
+                        codegen_and_compile_fn(
+                            tcx,
+                            &mut cx,
+                            &mut cached_context,
+                            &mut jit_module,
+                            inst,
+                        );
                     }
                     CodegenMode::JitLazy => {
                         codegen_shim(tcx, &mut cx, &mut cached_context, &mut jit_module, inst)
@@ -219,6 +217,24 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
     }
 }
 
+pub(crate) fn codegen_and_compile_fn<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    cx: &mut crate::CodegenCx,
+    cached_context: &mut Context,
+    module: &mut dyn Module,
+    instance: Instance<'tcx>,
+) {
+    tcx.prof.generic_activity("codegen and compile fn").run(|| {
+        let _inst_guard =
+            crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name));
+
+        let cached_func = std::mem::replace(&mut cached_context.func, Function::new());
+        let codegened_func = crate::base::codegen_fn(tcx, cx, cached_func, module, instance);
+
+        crate::base::compile_fn(cx, cached_context, module, codegened_func);
+    });
+}
+
 extern "C" fn clif_jit_fn(
     instance_ptr: *const Instance<'static>,
     trampoline_ptr: *const u8,
@@ -271,15 +287,7 @@ fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) ->
                 false,
                 Symbol::intern("dummy_cgu_name"),
             );
-            tcx.sess.time("codegen fn", || {
-                crate::base::codegen_and_compile_fn(
-                    tcx,
-                    &mut cx,
-                    &mut Context::new(),
-                    jit_module,
-                    instance,
-                )
-            });
+            codegen_and_compile_fn(tcx, &mut cx, &mut Context::new(), jit_module, instance);
 
             assert!(cx.global_asm.is_empty());
             jit_module.finalize_definitions().unwrap();
diff --git a/compiler/rustc_codegen_cranelift/src/driver/mod.rs b/compiler/rustc_codegen_cranelift/src/driver/mod.rs
index 6e925cea2..d09d3a529 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/mod.rs
@@ -17,7 +17,7 @@ fn predefine_mono_items<'tcx>(
     module: &mut dyn Module,
     mono_items: &[(MonoItem<'tcx>, (RLinkage, Visibility))],
 ) {
-    tcx.sess.time("predefine functions", || {
+    tcx.prof.generic_activity("predefine functions").run(|| {
         let is_compiler_builtins = tcx.is_compiler_builtins(LOCAL_CRATE);
         for &(mono_item, (linkage, visibility)) in mono_items {
             match mono_item {
@@ -39,16 +39,3 @@ fn predefine_mono_items<'tcx>(
         }
     });
 }
-
-fn time<R>(tcx: TyCtxt<'_>, display: bool, name: &'static str, f: impl FnOnce() -> R) -> R {
-    if display {
-        println!("[{:<30}: {}] start", tcx.crate_name(LOCAL_CRATE), name);
-        let before = std::time::Instant::now();
-        let res = tcx.sess.time(name, f);
-        let after = std::time::Instant::now();
-        println!("[{:<30}: {}] end time: {:?}", tcx.crate_name(LOCAL_CRATE), name, after - before);
-        res
-    } else {
-        tcx.sess.time(name, f)
-    }
-}
diff --git a/compiler/rustc_codegen_cranelift/src/global_asm.rs b/compiler/rustc_codegen_cranelift/src/global_asm.rs
index dcbcaba30..46c78ce6a 100644
--- a/compiler/rustc_codegen_cranelift/src/global_asm.rs
+++ b/compiler/rustc_codegen_cranelift/src/global_asm.rs
@@ -7,7 +7,7 @@ use std::process::{Command, Stdio};
 use std::sync::Arc;
 
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
-use rustc_hir::ItemId;
+use rustc_hir::{InlineAsmOperand, ItemId};
 use rustc_session::config::{OutputFilenames, OutputType};
 
 use crate::prelude::*;
@@ -23,7 +23,46 @@ pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String,
         for piece in asm.template {
             match *piece {
                 InlineAsmTemplatePiece::String(ref s) => global_asm.push_str(s),
-                InlineAsmTemplatePiece::Placeholder { .. } => todo!(),
+                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: op_sp } => {
+                    match asm.operands[operand_idx].0 {
+                        InlineAsmOperand::Const { ref anon_const } => {
+                            let const_value =
+                                tcx.const_eval_poly(anon_const.def_id.to_def_id()).unwrap_or_else(
+                                    |_| span_bug!(op_sp, "asm const cannot be resolved"),
+                                );
+                            let ty = tcx.typeck_body(anon_const.body).node_type(anon_const.hir_id);
+                            let string = rustc_codegen_ssa::common::asm_const_to_str(
+                                tcx,
+                                op_sp,
+                                const_value,
+                                RevealAllLayoutCx(tcx).layout_of(ty),
+                            );
+                            global_asm.push_str(&string);
+                        }
+                        InlineAsmOperand::SymFn { anon_const } => {
+                            let ty = tcx.typeck_body(anon_const.body).node_type(anon_const.hir_id);
+                            let instance = match ty.kind() {
+                                &ty::FnDef(def_id, substs) => Instance::new(def_id, substs),
+                                _ => span_bug!(op_sp, "asm sym is not a function"),
+                            };
+                            let symbol = tcx.symbol_name(instance);
+                            // FIXME handle the case where the function was made private to the
+                            // current codegen unit
+                            global_asm.push_str(symbol.name);
+                        }
+                        InlineAsmOperand::SymStatic { path: _, def_id } => {
+                            let instance = Instance::mono(tcx, def_id).polymorphize(tcx);
+                            let symbol = tcx.symbol_name(instance);
+                            global_asm.push_str(symbol.name);
+                        }
+                        InlineAsmOperand::In { .. }
+                        | InlineAsmOperand::Out { .. }
+                        | InlineAsmOperand::InOut { .. }
+                        | InlineAsmOperand::SplitInOut { .. } => {
+                            span_bug!(op_sp, "invalid operand type for global_asm!")
+                        }
+                    }
+                }
             }
         }
         global_asm.push_str("\n.att_syntax\n\n");
diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
index 3fcc84d39..6206fbf7d 100644
--- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs
+++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
@@ -9,9 +9,33 @@ use rustc_middle::mir::InlineAsmOperand;
 use rustc_span::sym;
 use rustc_target::asm::*;
 
+enum CInlineAsmOperand<'tcx> {
+    In {
+        reg: InlineAsmRegOrRegClass,
+        value: CValue<'tcx>,
+    },
+    Out {
+        reg: InlineAsmRegOrRegClass,
+        late: bool,
+        place: Option<CPlace<'tcx>>,
+    },
+    InOut {
+        reg: InlineAsmRegOrRegClass,
+        _late: bool,
+        in_value: CValue<'tcx>,
+        out_place: Option<CPlace<'tcx>>,
+    },
+    Const {
+        value: String,
+    },
+    Symbol {
+        symbol: String,
+    },
+}
+
 pub(crate) fn codegen_inline_asm<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
-    _span: Span,
+    span: Span,
     template: &[InlineAsmTemplatePiece],
     operands: &[InlineAsmOperand<'tcx>],
     options: InlineAsmOptions,
@@ -198,6 +222,81 @@ pub(crate) fn codegen_inline_asm<'tcx>(
         }
     }
 
+    let operands = operands
+        .into_iter()
+        .map(|operand| match *operand {
+            InlineAsmOperand::In { reg, ref value } => {
+                CInlineAsmOperand::In { reg, value: crate::base::codegen_operand(fx, value) }
+            }
+            InlineAsmOperand::Out { reg, late, ref place } => CInlineAsmOperand::Out {
+                reg,
+                late,
+                place: place.map(|place| crate::base::codegen_place(fx, place)),
+            },
+            InlineAsmOperand::InOut { reg, late, ref in_value, ref out_place } => {
+                CInlineAsmOperand::InOut {
+                    reg,
+                    _late: late,
+                    in_value: crate::base::codegen_operand(fx, in_value),
+                    out_place: out_place.map(|place| crate::base::codegen_place(fx, place)),
+                }
+            }
+            InlineAsmOperand::Const { ref value } => {
+                let (const_value, ty) = crate::constant::eval_mir_constant(fx, &*value)
+                    .unwrap_or_else(|| span_bug!(span, "asm const cannot be resolved"));
+                let value = rustc_codegen_ssa::common::asm_const_to_str(
+                    fx.tcx,
+                    span,
+                    const_value,
+                    fx.layout_of(ty),
+                );
+                CInlineAsmOperand::Const { value }
+            }
+            InlineAsmOperand::SymFn { ref value } => {
+                let literal = fx.monomorphize(value.literal);
+                if let ty::FnDef(def_id, substs) = *literal.ty().kind() {
+                    let instance = ty::Instance::resolve_for_fn_ptr(
+                        fx.tcx,
+                        ty::ParamEnv::reveal_all(),
+                        def_id,
+                        substs,
+                    )
+                    .unwrap();
+                    let symbol = fx.tcx.symbol_name(instance);
+
+                    // Pass a wrapper rather than the function itself as the function itself may not
+                    // be exported from the main codegen unit and may thus be unreachable from the
+                    // object file created by an external assembler.
+                    let inline_asm_index = fx.cx.inline_asm_index.get();
+                    fx.cx.inline_asm_index.set(inline_asm_index + 1);
+                    let wrapper_name = format!(
+                        "__inline_asm_{}_wrapper_n{}",
+                        fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
+                        inline_asm_index
+                    );
+                    let sig =
+                        get_function_sig(fx.tcx, fx.target_config.default_call_conv, instance);
+                    create_wrapper_function(
+                        fx.module,
+                        &mut fx.cx.unwind_context,
+                        sig,
+                        &wrapper_name,
+                        symbol.name,
+                    );
+
+                    CInlineAsmOperand::Symbol { symbol: wrapper_name }
+                } else {
+                    span_bug!(span, "invalid type for asm sym (fn)");
+                }
+            }
+            InlineAsmOperand::SymStatic { def_id } => {
+                assert!(fx.tcx.is_static(def_id));
+                let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
+                CInlineAsmOperand::Symbol { symbol: fx.tcx.symbol_name(instance).name.to_owned() }
+            }
+        })
+        .collect::<Vec<_>>();
+
     let mut inputs = Vec::new();
     let mut outputs = Vec::new();
 
@@ -206,7 +305,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
         arch: fx.tcx.sess.asm_arch.unwrap(),
         enclosing_def_id: fx.instance.def_id(),
         template,
-        operands,
+        operands: &operands,
         options,
         registers: Vec::new(),
         stack_slots_clobber: Vec::new(),
@@ -229,36 +328,22 @@ pub(crate) fn codegen_inline_asm<'tcx>(
     fx.cx.global_asm.push_str(&generated_asm);
 
     for (i, operand) in operands.iter().enumerate() {
-        match *operand {
-            InlineAsmOperand::In { reg: _, ref value } => {
-                inputs.push((
-                    asm_gen.stack_slots_input[i].unwrap(),
-                    crate::base::codegen_operand(fx, value).load_scalar(fx),
-                ));
-            }
-            InlineAsmOperand::Out { reg: _, late: _, place } => {
+        match operand {
+            CInlineAsmOperand::In { reg: _, value } => {
+                inputs.push((asm_gen.stack_slots_input[i].unwrap(), value.load_scalar(fx)));
+            }
+            CInlineAsmOperand::Out { reg: _, late: _, place } => {
                 if let Some(place) = place {
-                    outputs.push((
-                        asm_gen.stack_slots_output[i].unwrap(),
-                        crate::base::codegen_place(fx, place),
-                    ));
+                    outputs.push((asm_gen.stack_slots_output[i].unwrap(), place.clone()));
                 }
             }
-            InlineAsmOperand::InOut { reg: _, late: _, ref in_value, out_place } => {
-                inputs.push((
-                    asm_gen.stack_slots_input[i].unwrap(),
-                    crate::base::codegen_operand(fx, in_value).load_scalar(fx),
-                ));
+            CInlineAsmOperand::InOut { reg: _, _late: _, in_value, out_place } => {
+                inputs.push((asm_gen.stack_slots_input[i].unwrap(), in_value.load_scalar(fx)));
                 if let Some(out_place) = out_place {
-                    outputs.push((
-                        asm_gen.stack_slots_output[i].unwrap(),
-                        crate::base::codegen_place(fx, out_place),
-                    ));
+                    outputs.push((asm_gen.stack_slots_output[i].unwrap(), out_place.clone()));
                 }
             }
-            InlineAsmOperand::Const { value: _ } => todo!(),
-            InlineAsmOperand::SymFn { value: _ } => todo!(),
-            InlineAsmOperand::SymStatic { def_id: _ } => todo!(),
+            CInlineAsmOperand::Const { value: _ } | CInlineAsmOperand::Symbol { symbol: _ } => {}
         }
     }
 
@@ -280,7 +365,7 @@ struct InlineAssemblyGenerator<'a, 'tcx> {
     arch: InlineAsmArch,
     enclosing_def_id: DefId,
     template: &'a [InlineAsmTemplatePiece],
-    operands: &'a [InlineAsmOperand<'tcx>],
+    operands: &'a [CInlineAsmOperand<'tcx>],
     options: InlineAsmOptions,
     registers: Vec<Option<InlineAsmReg>>,
     stack_slots_clobber: Vec<Option<Size>>,
@@ -304,18 +389,20 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         // Add explicit registers to the allocated set.
         for (i, operand) in self.operands.iter().enumerate() {
             match *operand {
-                InlineAsmOperand::In { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
+                CInlineAsmOperand::In { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
                     regs[i] = Some(reg);
                     allocated.entry(reg).or_default().0 = true;
                 }
-                InlineAsmOperand::Out {
-                    reg: InlineAsmRegOrRegClass::Reg(reg), late: true, ..
+                CInlineAsmOperand::Out {
+                    reg: InlineAsmRegOrRegClass::Reg(reg),
+                    late: true,
+                    ..
                 } => {
                     regs[i] = Some(reg);
                     allocated.entry(reg).or_default().1 = true;
                 }
-                InlineAsmOperand::Out { reg: InlineAsmRegOrRegClass::Reg(reg), .. }
-                | InlineAsmOperand::InOut { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
+                CInlineAsmOperand::Out { reg: InlineAsmRegOrRegClass::Reg(reg), .. }
+                | CInlineAsmOperand::InOut { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
                     regs[i] = Some(reg);
                     allocated.insert(reg, (true, true));
                 }
@@ -326,12 +413,12 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         // Allocate out/inout/inlateout registers first because they are more constrained.
         for (i, operand) in self.operands.iter().enumerate() {
             match *operand {
-                InlineAsmOperand::Out {
+                CInlineAsmOperand::Out {
                     reg: InlineAsmRegOrRegClass::RegClass(class),
                     late: false,
                     ..
                 }
-                | InlineAsmOperand::InOut {
+                | CInlineAsmOperand::InOut {
                     reg: InlineAsmRegOrRegClass::RegClass(class), ..
                 } => {
                     let mut alloc_reg = None;
@@ -360,7 +447,7 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         // Allocate in/lateout.
         for (i, operand) in self.operands.iter().enumerate() {
             match *operand {
-                InlineAsmOperand::In { reg: InlineAsmRegOrRegClass::RegClass(class), .. } => {
+                CInlineAsmOperand::In { reg: InlineAsmRegOrRegClass::RegClass(class), .. } => {
                     let mut alloc_reg = None;
                     for &reg in &map[&class] {
                         let mut used = false;
@@ -380,7 +467,7 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
                     regs[i] = Some(reg);
                     allocated.entry(reg).or_default().0 = true;
                 }
-                InlineAsmOperand::Out {
+                CInlineAsmOperand::Out {
                     reg: InlineAsmRegOrRegClass::RegClass(class),
                     late: true,
                     ..
@@ -455,7 +542,7 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         // Allocate stack slots for inout
         for (i, operand) in self.operands.iter().enumerate() {
             match *operand {
-                InlineAsmOperand::InOut { reg, out_place: Some(_), .. } => {
+                CInlineAsmOperand::InOut { reg, out_place: Some(_), .. } => {
                     let slot = new_slot(reg.reg_class());
                     slots_input[i] = Some(slot);
                     slots_output[i] = Some(slot);
@@ -470,8 +557,8 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         // Allocate stack slots for input
         for (i, operand) in self.operands.iter().enumerate() {
             match *operand {
-                InlineAsmOperand::In { reg, .. }
-                | InlineAsmOperand::InOut { reg, out_place: None, .. } => {
+                CInlineAsmOperand::In { reg, .. }
+                | CInlineAsmOperand::InOut { reg, out_place: None, .. } => {
                     slots_input[i] = Some(new_slot(reg.reg_class()));
                 }
                 _ => (),
@@ -487,7 +574,7 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         // Allocate stack slots for output
         for (i, operand) in self.operands.iter().enumerate() {
             match *operand {
-                InlineAsmOperand::Out { reg, place: Some(_), .. } => {
+                CInlineAsmOperand::Out { reg, place: Some(_), .. } => {
                     slots_output[i] = Some(new_slot(reg.reg_class()));
                 }
                 _ => (),
@@ -549,13 +636,23 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
                     generated_asm.push_str(s);
                 }
                 InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => {
-                    if self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
-                        generated_asm.push('%');
+                    match self.operands[*operand_idx] {
+                        CInlineAsmOperand::In { .. }
+                        | CInlineAsmOperand::Out { .. }
+                        | CInlineAsmOperand::InOut { .. } => {
+                            if self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
+                                generated_asm.push('%');
+                            }
+                            self.registers[*operand_idx]
+                                .unwrap()
+                                .emit(&mut generated_asm, self.arch, *modifier)
+                                .unwrap();
+                        }
+                        CInlineAsmOperand::Const { ref value } => {
+                            generated_asm.push_str(value);
+                        }
+                        CInlineAsmOperand::Symbol { ref symbol } => generated_asm.push_str(symbol),
                     }
-                    self.registers[*operand_idx]
-                        .unwrap()
-                        .emit(&mut generated_asm, self.arch, *modifier)
-                        .unwrap();
                 }
             }
         }
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
index 7bc161fbe..e5c4b244a 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
@@ -33,8 +33,8 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
 
                 // cast float to int
                 let a_lane = match lane_ty {
-                    types::F32 => fx.bcx.ins().bitcast(types::I32, a_lane),
-                    types::F64 => fx.bcx.ins().bitcast(types::I64, a_lane),
+                    types::F32 => codegen_bitcast(fx, types::I32, a_lane),
+                    types::F64 => codegen_bitcast(fx, types::I64, a_lane),
                     _ => a_lane,
                 };
 
@@ -191,7 +191,7 @@ fn llvm_add_sub<'tcx>(
     // carry0 | carry1 -> carry or borrow respectively
     let cb_out = fx.bcx.ins().bor(cb0, cb1);
 
-    let layout = fx.layout_of(fx.tcx.mk_tup([fx.tcx.types.u8, fx.tcx.types.u64].iter()));
+    let layout = fx.layout_of(fx.tcx.mk_tup(&[fx.tcx.types.u8, fx.tcx.types.u64]));
     let val = CValue::by_val_pair(cb_out, c, layout);
     ret.write_cvalue(fx, val);
 }
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
index e4ac89a7b..e74aabf2f 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
@@ -21,6 +21,8 @@ mod simd;
 pub(crate) use cpuid::codegen_cpuid_call;
 pub(crate) use llvm::codegen_llvm_intrinsic_call;
 
+use rustc_middle::ty;
+use rustc_middle::ty::layout::{HasParamEnv, ValidityRequirement};
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::subst::SubstsRef;
 use rustc_span::symbol::{kw, sym, Symbol};
@@ -200,7 +202,7 @@ fn bool_to_zero_or_max_uint<'tcx>(
     let mut res = fx.bcx.ins().bmask(int_ty, val);
 
     if ty.is_float() {
-        res = fx.bcx.ins().bitcast(ty, res);
+        res = codegen_bitcast(fx, ty, res);
     }
 
     res
@@ -217,22 +219,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
     let intrinsic = fx.tcx.item_name(instance.def_id());
     let substs = instance.substs;
 
-    let target = if let Some(target) = target {
-        target
-    } else {
-        // Insert non returning intrinsics here
-        match intrinsic {
-            sym::abort => {
-                fx.bcx.ins().trap(TrapCode::User(0));
-            }
-            sym::transmute => {
-                crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", source_info);
-            }
-            _ => unimplemented!("unsupported intrinsic {}", intrinsic),
-        }
-        return;
-    };
-
     if intrinsic.as_str().starts_with("simd_") {
         self::simd::codegen_simd_intrinsic_call(
             fx,
@@ -240,12 +226,11 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             substs,
             args,
             destination,
+            target.expect("target for simd intrinsic"),
             source_info.span,
         );
-        let ret_block = fx.get_block(target);
-        fx.bcx.ins().jump(ret_block, &[]);
     } else if codegen_float_intrinsic_call(fx, intrinsic, args, destination) {
-        let ret_block = fx.get_block(target);
+        let ret_block = fx.get_block(target.expect("target for float intrinsic"));
         fx.bcx.ins().jump(ret_block, &[]);
     } else {
         codegen_regular_intrinsic_call(
@@ -255,7 +240,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             substs,
             args,
             destination,
-            Some(target),
+            target,
             source_info,
         );
     }
@@ -382,6 +367,10 @@ fn codegen_regular_intrinsic_call<'tcx>(
     let usize_layout = fx.layout_of(fx.tcx.types.usize);
 
     match intrinsic {
+        sym::abort => {
+            fx.bcx.ins().trap(TrapCode::User(0));
+            return;
+        }
         sym::likely | sym::unlikely => {
             intrinsic_args!(fx, args => (a); intrinsic);
 
@@ -505,20 +494,6 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let res = crate::num::codegen_int_binop(fx, bin_op, x, y);
             ret.write_cvalue(fx, res);
         }
-        sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
-            intrinsic_args!(fx, args => (x, y); intrinsic);
-
-            assert_eq!(x.layout().ty, y.layout().ty);
-            let bin_op = match intrinsic {
-                sym::add_with_overflow => BinOp::Add,
-                sym::sub_with_overflow => BinOp::Sub,
-                sym::mul_with_overflow => BinOp::Mul,
-                _ => unreachable!(),
-            };
-
-            let res = crate::num::codegen_checked_int_binop(fx, bin_op, x, y);
-            ret.write_cvalue(fx, res);
-        }
         sym::saturating_add | sym::saturating_sub => {
             intrinsic_args!(fx, args => (lhs, rhs); intrinsic);
 
@@ -579,6 +554,11 @@ fn codegen_regular_intrinsic_call<'tcx>(
         sym::transmute => {
             intrinsic_args!(fx, args => (from); intrinsic);
 
+            if ret.layout().abi.is_uninhabited() {
+                crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", source_info);
+                return;
+            }
+
             ret.write_cvalue_transmute(fx, from);
         }
         sym::write_bytes | sym::volatile_set_memory => {
@@ -647,46 +627,40 @@ fn codegen_regular_intrinsic_call<'tcx>(
         sym::assert_inhabited | sym::assert_zero_valid | sym::assert_mem_uninitialized_valid => {
             intrinsic_args!(fx, args => (); intrinsic);
 
-            let layout = fx.layout_of(substs.type_at(0));
-            if layout.abi.is_uninhabited() {
-                with_no_trimmed_paths!({
-                    crate::base::codegen_panic(
-                        fx,
-                        &format!("attempted to instantiate uninhabited type `{}`", layout.ty),
-                        source_info,
-                    )
-                });
-                return;
-            }
-
-            if intrinsic == sym::assert_zero_valid && !fx.tcx.permits_zero_init(layout) {
-                with_no_trimmed_paths!({
-                    crate::base::codegen_panic(
-                        fx,
-                        &format!(
-                            "attempted to zero-initialize type `{}`, which is invalid",
-                            layout.ty
-                        ),
-                        source_info,
-                    );
-                });
-                return;
-            }
+            let ty = substs.type_at(0);
 
-            if intrinsic == sym::assert_mem_uninitialized_valid
-                && !fx.tcx.permits_uninit_init(layout)
-            {
-                with_no_trimmed_paths!({
-                    crate::base::codegen_panic(
-                        fx,
-                        &format!(
-                            "attempted to leave type `{}` uninitialized, which is invalid",
-                            layout.ty
-                        ),
-                        source_info,
-                    )
-                });
-                return;
+            let requirement = ValidityRequirement::from_intrinsic(intrinsic);
+
+            if let Some(requirement) = requirement {
+                let do_panic = !fx
+                    .tcx
+                    .check_validity_requirement((requirement, fx.param_env().and(ty)))
+                    .expect("expect to have layout during codegen");
+
+                if do_panic {
+                    let layout = fx.layout_of(ty);
+
+                    with_no_trimmed_paths!({
+                        crate::base::codegen_panic_nounwind(
+                            fx,
+                            &if layout.abi.is_uninhabited() {
+                                format!("attempted to instantiate uninhabited type `{}`", layout.ty)
+                            } else if requirement == ValidityRequirement::Zero {
+                                format!(
+                                    "attempted to zero-initialize type `{}`, which is invalid",
+                                    layout.ty
+                                )
+                            } else {
+                                format!(
+                                    "attempted to leave type `{}` uninitialized, which is invalid",
+                                    layout.ty
+                                )
+                            },
+                            source_info,
+                        )
+                    });
+                    return;
+                }
             }
         }
 
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
index 14f5e9187..a1d63acfb 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
@@ -24,6 +24,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
     _substs: SubstsRef<'tcx>,
     args: &[mir::Operand<'tcx>],
     ret: CPlace<'tcx>,
+    target: BasicBlock,
     span: Span,
 ) {
     match intrinsic {
@@ -140,7 +141,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 let idx_ty = fx.monomorphize(idx.ty(fx.mir, fx.tcx));
                 match idx_ty.kind() {
                     ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => len
-                        .try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all())
+                        .try_eval_target_usize(fx.tcx, ty::ParamEnv::reveal_all())
                         .unwrap_or_else(|| {
                             span_bug!(span, "could not evaluate shuffle index array length")
                         })
@@ -277,16 +278,15 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             } else {
                 fx.tcx.sess.span_warn(span, "Index argument for `simd_extract` is not a constant");
                 let trap_block = fx.bcx.create_block();
-                let dummy_block = fx.bcx.create_block();
                 let true_ = fx.bcx.ins().iconst(types::I8, 1);
                 fx.bcx.ins().brnz(true_, trap_block, &[]);
-                fx.bcx.ins().jump(dummy_block, &[]);
+                let ret_block = fx.get_block(target);
+                fx.bcx.ins().jump(ret_block, &[]);
                 fx.bcx.switch_to_block(trap_block);
                 crate::trap::trap_unimplemented(
                     fx,
                     "Index argument for `simd_extract` is not a constant",
                 );
-                fx.bcx.switch_to_block(dummy_block);
                 return;
             };
 
@@ -735,7 +735,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => {}
                 ty::Array(elem, len)
                     if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
-                        && len.try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all())
+                        && len.try_eval_target_usize(fx.tcx, ty::ParamEnv::reveal_all())
                             == Some(expected_bytes) => {}
                 _ => {
                     fx.tcx.sess.span_fatal(
@@ -770,11 +770,119 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             });
         }
 
-        // simd_arith_offset
-        // simd_scatter
-        // simd_gather
+        sym::simd_expose_addr | sym::simd_from_exposed_addr | sym::simd_cast_ptr => {
+            intrinsic_args!(fx, args => (arg); intrinsic);
+            ret.write_cvalue_transmute(fx, arg);
+        }
+
+        sym::simd_arith_offset => {
+            intrinsic_args!(fx, args => (ptr, offset); intrinsic);
+
+            let (lane_count, ptr_lane_ty) = ptr.layout().ty.simd_size_and_type(fx.tcx);
+            let pointee_ty = ptr_lane_ty.builtin_deref(true).unwrap().ty;
+            let pointee_size = fx.layout_of(pointee_ty).size.bytes();
+            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+            let ret_lane_layout = fx.layout_of(ret_lane_ty);
+            assert_eq!(lane_count, ret_lane_count);
+
+            for lane_idx in 0..lane_count {
+                let ptr_lane = ptr.value_lane(fx, lane_idx).load_scalar(fx);
+                let offset_lane = offset.value_lane(fx, lane_idx).load_scalar(fx);
+
+                let ptr_diff = if pointee_size != 1 {
+                    fx.bcx.ins().imul_imm(offset_lane, pointee_size as i64)
+                } else {
+                    offset_lane
+                };
+                let res_lane = fx.bcx.ins().iadd(ptr_lane, ptr_diff);
+                let res_lane = CValue::by_val(res_lane, ret_lane_layout);
+
+                ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
+            }
+        }
+
+        sym::simd_gather => {
+            intrinsic_args!(fx, args => (val, ptr, mask); intrinsic);
+
+            let (val_lane_count, val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
+            let (ptr_lane_count, _ptr_lane_ty) = ptr.layout().ty.simd_size_and_type(fx.tcx);
+            let (mask_lane_count, _mask_lane_ty) = mask.layout().ty.simd_size_and_type(fx.tcx);
+            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+            assert_eq!(val_lane_count, ptr_lane_count);
+            assert_eq!(val_lane_count, mask_lane_count);
+            assert_eq!(val_lane_count, ret_lane_count);
+
+            let lane_clif_ty = fx.clif_type(val_lane_ty).unwrap();
+            let ret_lane_layout = fx.layout_of(ret_lane_ty);
+
+            for lane_idx in 0..ptr_lane_count {
+                let val_lane = val.value_lane(fx, lane_idx).load_scalar(fx);
+                let ptr_lane = ptr.value_lane(fx, lane_idx).load_scalar(fx);
+                let mask_lane = mask.value_lane(fx, lane_idx).load_scalar(fx);
+
+                let if_enabled = fx.bcx.create_block();
+                let if_disabled = fx.bcx.create_block();
+                let next = fx.bcx.create_block();
+                let res_lane = fx.bcx.append_block_param(next, lane_clif_ty);
+
+                fx.bcx.ins().brnz(mask_lane, if_enabled, &[]);
+                fx.bcx.ins().jump(if_disabled, &[]);
+                fx.bcx.seal_block(if_enabled);
+                fx.bcx.seal_block(if_disabled);
+
+                fx.bcx.switch_to_block(if_enabled);
+                let res = fx.bcx.ins().load(lane_clif_ty, MemFlags::trusted(), ptr_lane, 0);
+                fx.bcx.ins().jump(next, &[res]);
+
+                fx.bcx.switch_to_block(if_disabled);
+                fx.bcx.ins().jump(next, &[val_lane]);
+
+                fx.bcx.seal_block(next);
+                fx.bcx.switch_to_block(next);
+
+                fx.bcx.ins().nop();
+
+                ret.place_lane(fx, lane_idx)
+                    .write_cvalue(fx, CValue::by_val(res_lane, ret_lane_layout));
+            }
+        }
+
+        sym::simd_scatter => {
+            intrinsic_args!(fx, args => (val, ptr, mask); intrinsic);
+
+            let (val_lane_count, _val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
+            let (ptr_lane_count, _ptr_lane_ty) = ptr.layout().ty.simd_size_and_type(fx.tcx);
+            let (mask_lane_count, _mask_lane_ty) = mask.layout().ty.simd_size_and_type(fx.tcx);
+            assert_eq!(val_lane_count, ptr_lane_count);
+            assert_eq!(val_lane_count, mask_lane_count);
+
+            for lane_idx in 0..ptr_lane_count {
+                let val_lane = val.value_lane(fx, lane_idx).load_scalar(fx);
+                let ptr_lane = ptr.value_lane(fx, lane_idx).load_scalar(fx);
+                let mask_lane = mask.value_lane(fx, lane_idx).load_scalar(fx);
+
+                let if_enabled = fx.bcx.create_block();
+                let next = fx.bcx.create_block();
+
+                fx.bcx.ins().brnz(mask_lane, if_enabled, &[]);
+                fx.bcx.ins().jump(next, &[]);
+                fx.bcx.seal_block(if_enabled);
+
+                fx.bcx.switch_to_block(if_enabled);
+                fx.bcx.ins().store(MemFlags::trusted(), val_lane, ptr_lane, 0);
+                fx.bcx.ins().jump(next, &[]);
+
+                fx.bcx.seal_block(next);
+                fx.bcx.switch_to_block(next);
+            }
+        }
+
         _ => {
-            fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
+            fx.tcx.sess.span_err(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
+            // Prevent verifier error
+            fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
         }
     }
+    let ret_block = fx.get_block(target);
+    fx.bcx.ins().jump(ret_block, &[]);
 }
diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs
index 629d79d50..80ce3dc93 100644
--- a/compiler/rustc_codegen_cranelift/src/lib.rs
+++ b/compiler/rustc_codegen_cranelift/src/lib.rs
@@ -57,6 +57,8 @@ mod compiler_builtins;
 mod concurrency_limiter;
 mod config;
 mod constant;
+// FIXME revert back to the external crate with Cranelift 0.93
+mod cranelift_native;
 mod debuginfo;
 mod discriminant;
 mod driver;
@@ -84,7 +86,7 @@ mod prelude {
     pub(crate) use rustc_middle::ty::layout::{self, LayoutOf, TyAndLayout};
     pub(crate) use rustc_middle::ty::{
         self, FloatTy, Instance, InstanceDef, IntTy, ParamEnv, Ty, TyCtxt, TypeAndMut,
-        TypeFoldable, TypeVisitable, UintTy,
+        TypeFoldable, TypeVisitableExt, UintTy,
     };
     pub(crate) use rustc_target::abi::{Abi, Scalar, Size, VariantIdx};
 
@@ -170,6 +172,11 @@ pub struct CraneliftCodegenBackend {
 }
 
 impl CodegenBackend for CraneliftCodegenBackend {
+    fn locale_resource(&self) -> &'static str {
+        // FIXME(rust-lang/rust#100717) - cranelift codegen backend is not yet translated
+        ""
+    }
+
     fn init(&self, sess: &Session) {
         use rustc_session::config::Lto;
         match sess.lto() {
@@ -278,12 +285,14 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::Tar
         }
     }
 
-    if target_triple.architecture == target_lexicon::Architecture::X86_64 {
+    if let target_lexicon::Architecture::Aarch64(_) | target_lexicon::Architecture::X86_64 =
+        target_triple.architecture
+    {
         // Windows depends on stack probes to grow the committed part of the stack
         flags_builder.enable("enable_probestack").unwrap();
         flags_builder.set("probestack_strategy", "inline").unwrap();
     } else {
-        // __cranelift_probestack is not provided and inline stack probes are only supported on x86_64
+        // __cranelift_probestack is not provided and inline stack probes are only supported on AArch64 and x86_64
         flags_builder.set("enable_probestack", "false").unwrap();
     }
 
diff --git a/compiler/rustc_codegen_cranelift/src/main_shim.rs b/compiler/rustc_codegen_cranelift/src/main_shim.rs
index c10054e7f..be908df83 100644
--- a/compiler/rustc_codegen_cranelift/src/main_shim.rs
+++ b/compiler/rustc_codegen_cranelift/src/main_shim.rs
@@ -46,7 +46,7 @@ pub(crate) fn maybe_create_entry_wrapper(
         is_main_fn: bool,
         sigpipe: u8,
     ) {
-        let main_ret_ty = tcx.fn_sig(rust_main_def_id).output();
+        let main_ret_ty = tcx.fn_sig(rust_main_def_id).no_bound_vars().unwrap().output();
         // Given that `main()` has no arguments,
         // then its return type cannot have
         // late-bound regions, since late-bound
@@ -64,13 +64,20 @@ pub(crate) fn maybe_create_entry_wrapper(
             ],
             returns: vec![AbiParam::new(m.target_config().pointer_type() /*isize*/)],
             call_conv: crate::conv_to_call_conv(
+                tcx.sess,
                 tcx.sess.target.options.entry_abi,
                 m.target_config().default_call_conv,
             ),
         };
 
         let entry_name = tcx.sess.target.options.entry_name.as_ref();
-        let cmain_func_id = m.declare_function(entry_name, Linkage::Export, &cmain_sig).unwrap();
+        let cmain_func_id = match m.declare_function(entry_name, Linkage::Export, &cmain_sig) {
+            Ok(func_id) => func_id,
+            Err(err) => {
+                tcx.sess
+                    .fatal(&format!("entry symbol `{entry_name}` declared multiple times: {err}"));
+            }
+        };
 
         let instance = Instance::mono(tcx, rust_main_def_id).polymorphize(tcx);
 
@@ -112,7 +119,7 @@ pub(crate) fn maybe_create_entry_wrapper(
                     tcx,
                     ParamEnv::reveal_all(),
                     report.def_id,
-                    tcx.mk_substs([GenericArg::from(main_ret_ty)].iter()),
+                    tcx.mk_substs(&[GenericArg::from(main_ret_ty)]),
                 )
                 .unwrap()
                 .unwrap()
@@ -139,7 +146,7 @@ pub(crate) fn maybe_create_entry_wrapper(
                     tcx,
                     ParamEnv::reveal_all(),
                     start_def_id,
-                    tcx.intern_substs(&[main_ret_ty.into()]),
+                    tcx.mk_substs(&[main_ret_ty.into()]),
                 )
                 .unwrap()
                 .unwrap()
@@ -162,7 +169,11 @@ pub(crate) fn maybe_create_entry_wrapper(
             bcx.seal_all_blocks();
             bcx.finalize();
         }
-        m.define_function(cmain_func_id, &mut ctx).unwrap();
+
+        if let Err(err) = m.define_function(cmain_func_id, &mut ctx) {
+            tcx.sess.fatal(&format!("entry symbol `{entry_name}` defined multiple times: {err}"));
+        }
+
         unwind_context.add_function(cmain_func_id, &ctx, m.isa());
     }
 }
diff --git a/compiler/rustc_codegen_cranelift/src/num.rs b/compiler/rustc_codegen_cranelift/src/num.rs
index afacbec64..c058ece96 100644
--- a/compiler/rustc_codegen_cranelift/src/num.rs
+++ b/compiler/rustc_codegen_cranelift/src/num.rs
@@ -289,7 +289,7 @@ pub(crate) fn codegen_checked_int_binop<'tcx>(
         _ => bug!("binop {:?} on checked int/uint lhs: {:?} rhs: {:?}", bin_op, in_lhs, in_rhs),
     };
 
-    let out_layout = fx.layout_of(fx.tcx.mk_tup([in_lhs.layout().ty, fx.tcx.types.bool].iter()));
+    let out_layout = fx.layout_of(fx.tcx.mk_tup(&[in_lhs.layout().ty, fx.tcx.types.bool]));
     CValue::by_val_pair(res, has_overflow, out_layout)
 }
 
diff --git a/compiler/rustc_codegen_cranelift/src/optimize/peephole.rs b/compiler/rustc_codegen_cranelift/src/optimize/peephole.rs
index 7f45bbd8f..26327dca2 100644
--- a/compiler/rustc_codegen_cranelift/src/optimize/peephole.rs
+++ b/compiler/rustc_codegen_cranelift/src/optimize/peephole.rs
@@ -7,7 +7,7 @@ use cranelift_frontend::FunctionBuilder;
 /// otherwise return the given value and false.
 pub(crate) fn maybe_unwrap_bool_not(bcx: &mut FunctionBuilder<'_>, arg: Value) -> (Value, bool) {
     if let ValueDef::Result(arg_inst, 0) = bcx.func.dfg.value_def(arg) {
-        match bcx.func.dfg[arg_inst] {
+        match bcx.func.dfg.insts[arg_inst] {
             // This is the lowering of `Rvalue::Not`
             InstructionData::IntCompareImm {
                 opcode: Opcode::IcmpImm,
@@ -34,7 +34,7 @@ pub(crate) fn maybe_known_branch_taken(
         return None;
     };
 
-    match bcx.func.dfg[arg_inst] {
+    match bcx.func.dfg.insts[arg_inst] {
         InstructionData::UnaryImm { opcode: Opcode::Iconst, imm } => {
             if test_zero {
                 Some(imm.bits() == 0)
diff --git a/compiler/rustc_codegen_cranelift/src/unsize.rs b/compiler/rustc_codegen_cranelift/src/unsize.rs
index 9c88f7dbc..a0745582d 100644
--- a/compiler/rustc_codegen_cranelift/src/unsize.rs
+++ b/compiler/rustc_codegen_cranelift/src/unsize.rs
@@ -24,7 +24,7 @@ pub(crate) fn unsized_info<'tcx>(
         (&ty::Array(_, len), &ty::Slice(_)) => fx
             .bcx
             .ins()
-            .iconst(fx.pointer_type, len.eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64),
+            .iconst(fx.pointer_type, len.eval_target_usize(fx.tcx, ParamEnv::reveal_all()) as i64),
         (
             &ty::Dynamic(ref data_a, _, src_dyn_kind),
             &ty::Dynamic(ref data_b, _, target_dyn_kind),
diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
index fe8af21ac..cc1edaa97 100644
--- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs
+++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
@@ -514,8 +514,8 @@ impl<'tcx> CPlace<'tcx> {
                 (types::I32, types::F32)
                 | (types::F32, types::I32)
                 | (types::I64, types::F64)
-                | (types::F64, types::I64) => fx.bcx.ins().bitcast(dst_ty, data),
-                _ if src_ty.is_vector() && dst_ty.is_vector() => fx.bcx.ins().bitcast(dst_ty, data),
+                | (types::F64, types::I64) => codegen_bitcast(fx, dst_ty, data),
+                _ if src_ty.is_vector() && dst_ty.is_vector() => codegen_bitcast(fx, dst_ty, data),
                 _ if src_ty.is_vector() || dst_ty.is_vector() => {
                     // FIXME do something more efficient for transmutes between vectors and integers.
                     let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
@@ -564,8 +564,8 @@ impl<'tcx> CPlace<'tcx> {
             CPlaceInner::Var(_local, var) => {
                 if let ty::Array(element, len) = dst_layout.ty.kind() {
                     // Can only happen for vector types
-                    let len =
-                        u32::try_from(len.eval_usize(fx.tcx, ParamEnv::reveal_all())).unwrap();
+                    let len = u32::try_from(len.eval_target_usize(fx.tcx, ParamEnv::reveal_all()))
+                        .unwrap();
                     let vector_ty = fx.clif_type(*element).unwrap().by(len).unwrap();
 
                     let data = match from.0 {
@@ -588,10 +588,13 @@ impl<'tcx> CPlace<'tcx> {
                 return;
             }
             CPlaceInner::VarPair(_local, var1, var2) => {
-                let (ptr, meta) = from.force_stack(fx);
-                assert!(meta.is_none());
-                let (data1, data2) =
-                    CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar_pair(fx);
+                let (data1, data2) = if from.layout().ty == dst_layout.ty {
+                    CValue(from.0, dst_layout).load_scalar_pair(fx)
+                } else {
+                    let (ptr, meta) = from.force_stack(fx);
+                    assert!(meta.is_none());
+                    CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar_pair(fx)
+                };
                 let (dst_ty1, dst_ty2) = fx.clif_pair_type(self.layout().ty).unwrap();
                 transmute_value(fx, var1, data1, dst_ty1);
                 transmute_value(fx, var2, data2, dst_ty2);
diff --git a/compiler/rustc_codegen_cranelift/y.rs b/compiler/rustc_codegen_cranelift/y.rs
index 02e1e21ad..fd825d02e 100755
--- a/compiler/rustc_codegen_cranelift/y.rs
+++ b/compiler/rustc_codegen_cranelift/y.rs
@@ -3,7 +3,7 @@
 # This block is ignored by rustc
 set -e
 echo "[BUILD] y.rs" 1>&2
-rustc $0 -o ${0/.rs/.bin} -Cdebuginfo=1 --edition 2021
+rustc $0 -o ${0/.rs/.bin} -Cdebuginfo=1 --edition 2021 -Cpanic=abort
 exec ${0/.rs/.bin} $@
 */
 
diff --git a/compiler/rustc_error_messages/locales/en-US/codegen_gcc.ftl b/compiler/rustc_codegen_gcc/locales/en-US.ftl
index 08ce51725..6101b28ab 100644
--- a/compiler/rustc_error_messages/locales/en-US/codegen_gcc.ftl
+++ b/compiler/rustc_codegen_gcc/locales/en-US.ftl
@@ -23,7 +23,7 @@ codegen_gcc_invalid_monomorphization_unsupported_element =
     invalid monomorphization of `{$name}` intrinsic: unsupported {$name} from `{$in_ty}` with element `{$elem_ty}` to `{$ret_ty}`
 
 codegen_gcc_invalid_monomorphization_invalid_bitmask =
-    invalid monomorphization of `{$name}` intrinsic: invalid bitmask `{ty}`, expected `u{$expected_int_bits}` or `[u8; {$expected_bytes}]`
+    invalid monomorphization of `{$name}` intrinsic: invalid bitmask `{$ty}`, expected `u{$expected_int_bits}` or `[u8; {$expected_bytes}]`
 
 codegen_gcc_invalid_monomorphization_simd_shuffle =
     invalid monomorphization of `{$name}` intrinsic: simd_shuffle index must be an array of `u32`, got `{$ty}`
diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs
index a92242b26..e88c12716 100644
--- a/compiler/rustc_codegen_gcc/src/builder.rs
+++ b/compiler/rustc_codegen_gcc/src/builder.rs
@@ -709,7 +709,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
                         bx.range_metadata(load, vr);
                     }
                 }
-                abi::Pointer if vr.start < vr.end && !vr.contains(0) => {
+                abi::Pointer(_) if vr.start < vr.end && !vr.contains(0) => {
                     bx.nonnull_metadata(load);
                 }
                 _ => {}
diff --git a/compiler/rustc_codegen_gcc/src/callee.rs b/compiler/rustc_codegen_gcc/src/callee.rs
index c1041125e..9e3a22ee0 100644
--- a/compiler/rustc_codegen_gcc/src/callee.rs
+++ b/compiler/rustc_codegen_gcc/src/callee.rs
@@ -1,6 +1,6 @@
 use gccjit::{FunctionType, RValue};
 use rustc_codegen_ssa::traits::BaseTypeMethods;
-use rustc_middle::ty::{self, Instance, TypeVisitable};
+use rustc_middle::ty::{self, Instance, TypeVisitableExt};
 use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt};
 
 use crate::abi::FnAbiGccExt;
diff --git a/compiler/rustc_codegen_gcc/src/common.rs b/compiler/rustc_codegen_gcc/src/common.rs
index 0afc56b44..c939da9ce 100644
--- a/compiler/rustc_codegen_gcc/src/common.rs
+++ b/compiler/rustc_codegen_gcc/src/common.rs
@@ -211,7 +211,7 @@ impl<'gcc, 'tcx> ConstMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
                 let base_addr = self.const_bitcast(base_addr, self.usize_type);
                 let offset = self.context.new_rvalue_from_long(self.usize_type, offset.bytes() as i64);
                 let ptr = self.const_bitcast(base_addr + offset, ptr_type);
-                if layout.primitive() != Pointer {
+                if !matches!(layout.primitive(), Pointer(_)) {
                     self.const_bitcast(ptr.dereference(None).to_rvalue(), ty)
                 }
                 else {
diff --git a/compiler/rustc_codegen_gcc/src/consts.rs b/compiler/rustc_codegen_gcc/src/consts.rs
index ea8ab7611..dc41cb761 100644
--- a/compiler/rustc_codegen_gcc/src/consts.rs
+++ b/compiler/rustc_codegen_gcc/src/consts.rs
@@ -322,13 +322,16 @@ pub fn const_alloc_to_gcc<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, alloc: ConstAl
             )
             .expect("const_alloc_to_llvm: could not read relocation pointer")
             as u64;
+
+        let address_space = cx.tcx.global_alloc(alloc_id).address_space(cx);
+
         llvals.push(cx.scalar_to_backend(
             InterpScalar::from_pointer(
                 interpret::Pointer::new(alloc_id, Size::from_bytes(ptr_offset)),
                 &cx.tcx,
             ),
-            abi::Scalar::Initialized { value: Primitive::Pointer, valid_range: WrappingRange::full(dl.pointer_size) },
-            cx.type_i8p(),
+            abi::Scalar::Initialized { value: Primitive::Pointer(address_space), valid_range: WrappingRange::full(dl.pointer_size) },
+            cx.type_i8p_ext(address_space),
         ));
         next_offset = offset + pointer_size;
     }
diff --git a/compiler/rustc_codegen_gcc/src/context.rs b/compiler/rustc_codegen_gcc/src/context.rs
index 4424b31c0..457006319 100644
--- a/compiler/rustc_codegen_gcc/src/context.rs
+++ b/compiler/rustc_codegen_gcc/src/context.rs
@@ -383,7 +383,7 @@ impl<'gcc, 'tcx> MiscMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
                     tcx,
                     ty::ParamEnv::reveal_all(),
                     def_id,
-                    tcx.intern_substs(&[]),
+                    ty::List::empty(),
                 )
                 .unwrap().unwrap(),
             ),
diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs b/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs
index 12e416f62..cb8168b40 100644
--- a/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs
+++ b/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs
@@ -1,6 +1,6 @@
 use std::cmp::Ordering;
 
-use gccjit::{BinaryOp, RValue, Type, ToRValue};
+use gccjit::{BinaryOp, RValue, ToRValue, Type};
 use rustc_codegen_ssa::base::compare_simd_types;
 use rustc_codegen_ssa::common::TypeKind;
 use rustc_codegen_ssa::mir::operand::OperandRef;
@@ -10,52 +10,57 @@ use rustc_hir as hir;
 use rustc_middle::span_bug;
 use rustc_middle::ty::layout::HasTyCtxt;
 use rustc_middle::ty::{self, Ty};
-use rustc_span::{Span, Symbol, sym};
+use rustc_span::{sym, Span, Symbol};
 use rustc_target::abi::Align;
 
 use crate::builder::Builder;
 use crate::errors::{
-    InvalidMonomorphizationInvalidFloatVector,
-    InvalidMonomorphizationNotFloat,
-    InvalidMonomorphizationUnrecognized,
-    InvalidMonomorphizationExpectedSignedUnsigned,
-    InvalidMonomorphizationUnsupportedElement,
-    InvalidMonomorphizationInvalidBitmask,
-    InvalidMonomorphizationSimdShuffle,
-    InvalidMonomorphizationExpectedSimd,
-    InvalidMonomorphizationMaskType,
-    InvalidMonomorphizationReturnLength,
-    InvalidMonomorphizationReturnLengthInputType,
-    InvalidMonomorphizationReturnElement,
-    InvalidMonomorphizationReturnType,
-    InvalidMonomorphizationInsertedType,
-    InvalidMonomorphizationReturnIntegerType,
-    InvalidMonomorphizationMismatchedLengths,
-    InvalidMonomorphizationUnsupportedCast,
-    InvalidMonomorphizationUnsupportedOperation
+    InvalidMonomorphizationExpectedSignedUnsigned, InvalidMonomorphizationExpectedSimd,
+    InvalidMonomorphizationInsertedType, InvalidMonomorphizationInvalidBitmask,
+    InvalidMonomorphizationInvalidFloatVector, InvalidMonomorphizationMaskType,
+    InvalidMonomorphizationMismatchedLengths, InvalidMonomorphizationNotFloat,
+    InvalidMonomorphizationReturnElement, InvalidMonomorphizationReturnIntegerType,
+    InvalidMonomorphizationReturnLength, InvalidMonomorphizationReturnLengthInputType,
+    InvalidMonomorphizationReturnType, InvalidMonomorphizationSimdShuffle,
+    InvalidMonomorphizationUnrecognized, InvalidMonomorphizationUnsupportedCast,
+    InvalidMonomorphizationUnsupportedElement, InvalidMonomorphizationUnsupportedOperation,
 };
 use crate::intrinsic;
 
-pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>, name: Symbol, callee_ty: Ty<'tcx>, args: &[OperandRef<'tcx, RValue<'gcc>>], ret_ty: Ty<'tcx>, llret_ty: Type<'gcc>, span: Span) -> Result<RValue<'gcc>, ()> {
+pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
+    bx: &mut Builder<'a, 'gcc, 'tcx>,
+    name: Symbol,
+    callee_ty: Ty<'tcx>,
+    args: &[OperandRef<'tcx, RValue<'gcc>>],
+    ret_ty: Ty<'tcx>,
+    llret_ty: Type<'gcc>,
+    span: Span,
+) -> Result<RValue<'gcc>, ()> {
     // macros for error handling:
     macro_rules! return_error {
-        ($err:expr) => {
-            {
-                bx.sess().emit_err($err);
-                return Err(());
-            }
-        }
+        ($err:expr) => {{
+            bx.sess().emit_err($err);
+            return Err(());
+        }};
     }
     macro_rules! require {
         ($cond:expr, $err:expr) => {
             if !$cond {
                 return_error!($err);
             }
-        }
+        };
     }
     macro_rules! require_simd {
         ($ty: expr, $position: expr) => {
-            require!($ty.is_simd(), InvalidMonomorphizationExpectedSimd { span, name, position: $position, found_ty: $ty })
+            require!(
+                $ty.is_simd(),
+                InvalidMonomorphizationExpectedSimd {
+                    span,
+                    name,
+                    position: $position,
+                    found_ty: $ty
+                }
+            )
         };
     }
 
@@ -77,7 +82,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
             ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
             ty::Array(elem, len)
                 if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
-                    && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all())
+                    && len.try_eval_target_usize(bx.tcx, ty::ParamEnv::reveal_all())
                         == Some(expected_bytes) =>
             {
                 let place = PlaceRef::alloca(bx, args[0].layout);
@@ -86,9 +91,13 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
                 let ptr = bx.pointercast(place.llval, bx.cx.type_ptr_to(int_ty));
                 bx.load(int_ty, ptr, Align::ONE)
             }
-            _ => return_error!(
-                InvalidMonomorphizationInvalidBitmask { span, name, ty: mask_ty, expected_int_bits, expected_bytes }
-            ),
+            _ => return_error!(InvalidMonomorphizationInvalidBitmask {
+                span,
+                name,
+                ty: mask_ty,
+                expected_int_bits,
+                expected_bytes
+            }),
         };
 
         let arg1 = args[1].immediate();
@@ -129,11 +138,18 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
         let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx());
         require!(
             in_len == out_len,
-            InvalidMonomorphizationReturnLengthInputType { span, name, in_len, in_ty, ret_ty, out_len }
+            InvalidMonomorphizationReturnLengthInputType {
+                span,
+                name,
+                in_len,
+                in_ty,
+                ret_ty,
+                out_len
+            }
         );
         require!(
             bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer,
-            InvalidMonomorphizationReturnIntegerType {span, name, ret_ty, out_ty}
+            InvalidMonomorphizationReturnIntegerType { span, name, ret_ty, out_ty }
         );
 
         return Ok(compare_simd_types(
@@ -147,26 +163,26 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
     }
 
     if let Some(stripped) = name.as_str().strip_prefix("simd_shuffle") {
-        let n: u64 =
-            if stripped.is_empty() {
-                // Make sure this is actually an array, since typeck only checks the length-suffixed
-                // version of this intrinsic.
-                match args[2].layout.ty.kind() {
-                    ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => {
-                        len.try_eval_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(|| {
-                            span_bug!(span, "could not evaluate shuffle index array length")
-                        })
-                    }
-                    _ => return_error!(
-                        InvalidMonomorphizationSimdShuffle { span, name, ty: args[2].layout.ty }
-                    ),
+        let n: u64 = if stripped.is_empty() {
+            // Make sure this is actually an array, since typeck only checks the length-suffixed
+            // version of this intrinsic.
+            match args[2].layout.ty.kind() {
+                ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => {
+                    len.try_eval_target_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(
+                        || span_bug!(span, "could not evaluate shuffle index array length"),
+                    )
                 }
+                _ => return_error!(InvalidMonomorphizationSimdShuffle {
+                    span,
+                    name,
+                    ty: args[2].layout.ty
+                }),
             }
-            else {
-                stripped.parse().unwrap_or_else(|_| {
-                    span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?")
-                })
-            };
+        } else {
+            stripped.parse().unwrap_or_else(|_| {
+                span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?")
+            })
+        };
 
         require_simd!(ret_ty, "return");
 
@@ -182,14 +198,10 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
 
         let vector = args[2].immediate();
 
-        return Ok(bx.shuffle_vector(
-            args[0].immediate(),
-            args[1].immediate(),
-            vector,
-        ));
+        return Ok(bx.shuffle_vector(args[0].immediate(), args[1].immediate(), vector));
     }
 
-    #[cfg(feature="master")]
+    #[cfg(feature = "master")]
     if name == sym::simd_insert {
         require!(
             in_elem == arg_tys[2],
@@ -205,44 +217,44 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
         // not be an l-value. So, call a builtin to set the element.
         // TODO(antoyo): perhaps we could create a new vector or maybe there's a GIMPLE instruction for that?
         // TODO(antoyo): don't use target specific builtins here.
-        let func_name =
-            match in_len {
-                2 => {
-                    if element_type == bx.i64_type {
-                        "__builtin_ia32_vec_set_v2di"
-                    }
-                    else {
-                        unimplemented!();
-                    }
-                },
-                4 => {
-                    if element_type == bx.i32_type {
-                        "__builtin_ia32_vec_set_v4si"
-                    }
-                    else {
-                        unimplemented!();
-                    }
-                },
-                8 => {
-                    if element_type == bx.i16_type {
-                        "__builtin_ia32_vec_set_v8hi"
-                    }
-                    else {
-                        unimplemented!();
-                    }
-                },
-                _ => unimplemented!("Len: {}", in_len),
-            };
+        let func_name = match in_len {
+            2 => {
+                if element_type == bx.i64_type {
+                    "__builtin_ia32_vec_set_v2di"
+                } else {
+                    unimplemented!();
+                }
+            }
+            4 => {
+                if element_type == bx.i32_type {
+                    "__builtin_ia32_vec_set_v4si"
+                } else {
+                    unimplemented!();
+                }
+            }
+            8 => {
+                if element_type == bx.i16_type {
+                    "__builtin_ia32_vec_set_v8hi"
+                } else {
+                    unimplemented!();
+                }
+            }
+            _ => unimplemented!("Len: {}", in_len),
+        };
         let builtin = bx.context.get_target_builtin_function(func_name);
         let param1_type = builtin.get_param(0).to_rvalue().get_type();
         // TODO(antoyo): perhaps use __builtin_convertvector for vector casting.
         let vector = bx.cx.bitcast_if_needed(vector, param1_type);
-        let result = bx.context.new_call(None, builtin, &[vector, value, bx.context.new_cast(None, index, bx.int_type)]);
+        let result = bx.context.new_call(
+            None,
+            builtin,
+            &[vector, value, bx.context.new_cast(None, index, bx.int_type)],
+        );
         // TODO(antoyo): perhaps use __builtin_convertvector for vector casting.
         return Ok(bx.context.new_bitcast(None, result, vector.get_type()));
     }
 
-    #[cfg(feature="master")]
+    #[cfg(feature = "master")]
     if name == sym::simd_extract {
         require!(
             ret_ty == in_elem,
@@ -273,7 +285,14 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
         let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
         require!(
             in_len == out_len,
-            InvalidMonomorphizationReturnLengthInputType { span, name, in_len, in_ty, ret_ty, out_len }
+            InvalidMonomorphizationReturnLengthInputType {
+                span,
+                name,
+                in_len,
+                in_ty,
+                ret_ty,
+                out_len
+            }
         );
         // casting cares about nominal type, not just structural type
         if in_elem == out_elem {
@@ -322,19 +341,27 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
 
             let cast_vec_element = |index| {
                 let index = bx.context.new_rvalue_from_int(bx.int_type, index);
-                bx.context.new_cast(None, bx.context.new_array_access(None, array, index).to_rvalue(), out_type)
+                bx.context.new_cast(
+                    None,
+                    bx.context.new_array_access(None, array, index).to_rvalue(),
+                    out_type,
+                )
             };
 
-            bx.context.new_rvalue_from_vector(None, vector_type, &[
-                cast_vec_element(0),
-                cast_vec_element(1),
-                cast_vec_element(2),
-                cast_vec_element(3),
-                cast_vec_element(4),
-                cast_vec_element(5),
-                cast_vec_element(6),
-                cast_vec_element(7),
-            ])
+            bx.context.new_rvalue_from_vector(
+                None,
+                vector_type,
+                &[
+                    cast_vec_element(0),
+                    cast_vec_element(1),
+                    cast_vec_element(2),
+                    cast_vec_element(3),
+                    cast_vec_element(4),
+                    cast_vec_element(5),
+                    cast_vec_element(6),
+                    cast_vec_element(7),
+                ],
+            )
         };
 
         match (in_style, out_style) {
@@ -385,9 +412,14 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
             }
             _ => { /* Unsupported. Fallthrough. */ }
         }
-        return_error!(
-            InvalidMonomorphizationUnsupportedCast { span, name, in_ty, in_elem, ret_ty, out_elem }
-        );
+        return_error!(InvalidMonomorphizationUnsupportedCast {
+            span,
+            name,
+            in_ty,
+            in_elem,
+            ret_ty,
+            out_elem
+        });
     }
 
     macro_rules! arith_binary {
@@ -414,54 +446,60 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
         args: &[OperandRef<'tcx, RValue<'gcc>>],
     ) -> Result<RValue<'gcc>, ()> {
         macro_rules! return_error {
-            ($err:expr) => {
-                {
-                    bx.sess().emit_err($err);
-                    return Err(());
-                }
-            }
+            ($err:expr) => {{
+                bx.sess().emit_err($err);
+                return Err(());
+            }};
         }
-        let (elem_ty_str, elem_ty) =
-            if let ty::Float(f) = in_elem.kind() {
-                let elem_ty = bx.cx.type_float_from_ty(*f);
-                match f.bit_width() {
-                    32 => ("f32", elem_ty),
-                    64 => ("f64", elem_ty),
-                    _ => {
-                        return_error!(InvalidMonomorphizationInvalidFloatVector { span, name, elem_ty: f.name_str(), vec_ty: in_ty });
-                    }
+        let (elem_ty_str, elem_ty) = if let ty::Float(f) = in_elem.kind() {
+            let elem_ty = bx.cx.type_float_from_ty(*f);
+            match f.bit_width() {
+                32 => ("f32", elem_ty),
+                64 => ("f64", elem_ty),
+                _ => {
+                    return_error!(InvalidMonomorphizationInvalidFloatVector {
+                        span,
+                        name,
+                        elem_ty: f.name_str(),
+                        vec_ty: in_ty
+                    });
                 }
             }
-            else {
-                return_error!(InvalidMonomorphizationNotFloat { span, name, ty: in_ty });
-            };
+        } else {
+            return_error!(InvalidMonomorphizationNotFloat { span, name, ty: in_ty });
+        };
 
         let vec_ty = bx.cx.type_vector(elem_ty, in_len);
 
-        let (intr_name, fn_ty) =
-            match name {
-                sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)),
-                sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)), // TODO(antoyo): pand with 170141183420855150465331762880109871103
-                sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)),
-                sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)),
-                sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)),
-                sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)),
-                sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)),
-                sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)),
-                sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)),
-                sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
-                sym::simd_fpowi => ("powi", bx.type_func(&[vec_ty, bx.type_i32()], vec_ty)),
-                sym::simd_fpow => ("pow", bx.type_func(&[vec_ty, vec_ty], vec_ty)),
-                sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)),
-                sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)),
-                sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)),
-                sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)),
-                _ => return_error!(InvalidMonomorphizationUnrecognized { span, name })
-            };
+        let (intr_name, fn_ty) = match name {
+            sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)), // TODO(antoyo): pand with 170141183420855150465331762880109871103
+            sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
+            sym::simd_fpowi => ("powi", bx.type_func(&[vec_ty, bx.type_i32()], vec_ty)),
+            sym::simd_fpow => ("pow", bx.type_func(&[vec_ty, vec_ty], vec_ty)),
+            sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)),
+            sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)),
+            _ => return_error!(InvalidMonomorphizationUnrecognized { span, name }),
+        };
         let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str);
         let function = intrinsic::llvm::intrinsic(llvm_name, &bx.cx);
         let function: RValue<'gcc> = unsafe { std::mem::transmute(function) };
-        let c = bx.call(fn_ty, None, function, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None);
+        let c = bx.call(
+            fn_ty,
+            None,
+            function,
+            &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
+            None,
+        );
         Ok(c)
     }
 
@@ -518,7 +556,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
         simd_neg: Int => neg, Float => fneg;
     }
 
-    #[cfg(feature="master")]
+    #[cfg(feature = "master")]
     if name == sym::simd_saturating_add || name == sym::simd_saturating_sub {
         let lhs = args[0].immediate();
         let rhs = args[1].immediate();
@@ -536,18 +574,23 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
                 });
             }
         };
-        let builtin_name =
-            match (signed, is_add, in_len, elem_width) {
-                (true, true, 32, 8) => "__builtin_ia32_paddsb256", // TODO(antoyo): cast arguments to unsigned.
-                (false, true, 32, 8) => "__builtin_ia32_paddusb256",
-                (true, true, 16, 16) => "__builtin_ia32_paddsw256",
-                (false, true, 16, 16) => "__builtin_ia32_paddusw256",
-                (true, false, 16, 16) => "__builtin_ia32_psubsw256",
-                (false, false, 16, 16) => "__builtin_ia32_psubusw256",
-                (true, false, 32, 8) => "__builtin_ia32_psubsb256",
-                (false, false, 32, 8) => "__builtin_ia32_psubusb256",
-                _ => unimplemented!("signed: {}, is_add: {}, in_len: {}, elem_width: {}", signed, is_add, in_len, elem_width),
-            };
+        let builtin_name = match (signed, is_add, in_len, elem_width) {
+            (true, true, 32, 8) => "__builtin_ia32_paddsb256", // TODO(antoyo): cast arguments to unsigned.
+            (false, true, 32, 8) => "__builtin_ia32_paddusb256",
+            (true, true, 16, 16) => "__builtin_ia32_paddsw256",
+            (false, true, 16, 16) => "__builtin_ia32_paddusw256",
+            (true, false, 16, 16) => "__builtin_ia32_psubsw256",
+            (false, false, 16, 16) => "__builtin_ia32_psubusw256",
+            (true, false, 32, 8) => "__builtin_ia32_psubsb256",
+            (false, false, 32, 8) => "__builtin_ia32_psubusb256",
+            _ => unimplemented!(
+                "signed: {}, is_add: {}, in_len: {}, elem_width: {}",
+                signed,
+                is_add,
+                in_len,
+                elem_width
+            ),
+        };
         let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64);
 
         let func = bx.context.get_target_builtin_function(builtin_name);
@@ -575,8 +618,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
                             // if overflow occurs, the result is the
                             // mathematical result modulo 2^n:
                             Ok(bx.$op(args[1].immediate(), r))
-                        }
-                        else {
+                        } else {
                             Ok(bx.vector_reduce_op(args[0].immediate(), $vec_op))
                         }
                     }
@@ -585,12 +627,17 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
                             // ordered arithmetic reductions take an accumulator
                             let acc = args[1].immediate();
                             Ok(bx.$float_reduce(acc, args[0].immediate()))
-                        }
-                        else {
+                        } else {
                             Ok(bx.vector_reduce_op(args[0].immediate(), $vec_op))
                         }
                     }
-                    _ => return_error!(InvalidMonomorphizationUnsupportedElement { span, name, in_ty, elem_ty: in_elem, ret_ty }),
+                    _ => return_error!(InvalidMonomorphizationUnsupportedElement {
+                        span,
+                        name,
+                        in_ty,
+                        elem_ty: in_elem,
+                        ret_ty
+                    }),
                 };
             }
         };
@@ -603,13 +650,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
         add,
         0.0 // TODO: Use this argument.
     );
-    arith_red!(
-        simd_reduce_mul_unordered: BinaryOp::Mult,
-        vector_reduce_fmul_fast,
-        false,
-        mul,
-        1.0
-    );
+    arith_red!(simd_reduce_mul_unordered: BinaryOp::Mult, vector_reduce_fmul_fast, false, mul, 1.0);
 
     macro_rules! minmax_red {
         ($name:ident: $reduction:ident) => {
@@ -619,8 +660,16 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
                     InvalidMonomorphizationReturnType { span, name, in_elem, in_ty, ret_ty }
                 );
                 return match in_elem.kind() {
-                    ty::Int(_) | ty::Uint(_) | ty::Float(_) => Ok(bx.$reduction(args[0].immediate())),
-                    _ => return_error!(InvalidMonomorphizationUnsupportedElement { span, name, in_ty, elem_ty: in_elem, ret_ty }),
+                    ty::Int(_) | ty::Uint(_) | ty::Float(_) => {
+                        Ok(bx.$reduction(args[0].immediate()))
+                    }
+                    _ => return_error!(InvalidMonomorphizationUnsupportedElement {
+                        span,
+                        name,
+                        in_ty,
+                        elem_ty: in_elem,
+                        ret_ty
+                    }),
                 };
             }
         };
@@ -641,7 +690,13 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
                 } else {
                     match in_elem.kind() {
                         ty::Int(_) | ty::Uint(_) => {}
-                        _ => return_error!(InvalidMonomorphizationUnsupportedElement { span, name, in_ty, elem_ty: in_elem, ret_ty }),
+                        _ => return_error!(InvalidMonomorphizationUnsupportedElement {
+                            span,
+                            name,
+                            in_ty,
+                            elem_ty: in_elem,
+                            ret_ty
+                        }),
                     }
 
                     // boolean reductions operate on vectors of i1s:
@@ -654,9 +709,13 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>,
                         let r = bx.vector_reduce_op(input, $op);
                         Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) })
                     }
-                    _ => return_error!(
-                        InvalidMonomorphizationUnsupportedElement { span, name, in_ty, elem_ty: in_elem, ret_ty }
-                    ),
+                    _ => return_error!(InvalidMonomorphizationUnsupportedElement {
+                        span,
+                        name,
+                        in_ty,
+                        elem_ty: in_elem,
+                        ret_ty
+                    }),
                 };
             }
         };
diff --git a/compiler/rustc_codegen_gcc/src/lib.rs b/compiler/rustc_codegen_gcc/src/lib.rs
index bf1da3831..44538b415 100644
--- a/compiler/rustc_codegen_gcc/src/lib.rs
+++ b/compiler/rustc_codegen_gcc/src/lib.rs
@@ -73,7 +73,8 @@ use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModul
 use rustc_codegen_ssa::target_features::supported_target_features;
 use rustc_codegen_ssa::traits::{CodegenBackend, ExtraBackendMethods, ModuleBufferMethods, ThinBufferMethods, WriteBackendMethods};
 use rustc_data_structures::fx::FxHashMap;
-use rustc_errors::{ErrorGuaranteed, Handler};
+use rustc_errors::{DiagnosticMessage, ErrorGuaranteed, Handler, SubdiagnosticMessage};
+use rustc_macros::fluent_messages;
 use rustc_metadata::EncodedMetadata;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::ty::TyCtxt;
@@ -84,6 +85,8 @@ use rustc_span::Symbol;
 use rustc_span::fatal_error::FatalError;
 use tempfile::TempDir;
 
+fluent_messages! { "../locales/en-US.ftl" }
+
 pub struct PrintOnPanic<F: Fn() -> String>(pub F);
 
 impl<F: Fn() -> String> Drop for PrintOnPanic<F> {
@@ -100,6 +103,10 @@ pub struct GccCodegenBackend {
 }
 
 impl CodegenBackend for GccCodegenBackend {
+    fn locale_resource(&self) -> &'static str {
+        crate::DEFAULT_LOCALE_RESOURCE
+    }
+
     fn init(&self, sess: &Session) {
         if sess.lto() != Lto::No {
             sess.emit_warning(LTONotSupported {});
@@ -200,6 +207,7 @@ unsafe impl Sync for GccContext {}
 impl WriteBackendMethods for GccCodegenBackend {
     type Module = GccContext;
     type TargetMachine = ();
+    type TargetMachineError = ();
     type ModuleBuffer = ModuleBuffer;
     type ThinData = ();
     type ThinBuffer = ThinBuffer;
diff --git a/compiler/rustc_codegen_gcc/src/mono_item.rs b/compiler/rustc_codegen_gcc/src/mono_item.rs
index 9468a1ef4..a7c868354 100644
--- a/compiler/rustc_codegen_gcc/src/mono_item.rs
+++ b/compiler/rustc_codegen_gcc/src/mono_item.rs
@@ -1,7 +1,7 @@
 use rustc_codegen_ssa::traits::PreDefineMethods;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::mono::{Linkage, Visibility};
-use rustc_middle::ty::{self, Instance, TypeVisitable};
+use rustc_middle::ty::{self, Instance, TypeVisitableExt};
 use rustc_middle::ty::layout::{FnAbiOf, LayoutOf};
 use rustc_span::def_id::DefId;
 
diff --git a/compiler/rustc_codegen_gcc/src/type_of.rs b/compiler/rustc_codegen_gcc/src/type_of.rs
index 524d10fb5..ea2ce7650 100644
--- a/compiler/rustc_codegen_gcc/src/type_of.rs
+++ b/compiler/rustc_codegen_gcc/src/type_of.rs
@@ -3,7 +3,7 @@ use std::fmt::Write;
 use gccjit::{Struct, Type};
 use crate::rustc_codegen_ssa::traits::{BaseTypeMethods, DerivedTypeMethods, LayoutTypeMethods};
 use rustc_middle::bug;
-use rustc_middle::ty::{self, Ty, TypeVisitable};
+use rustc_middle::ty::{self, Ty, TypeVisitableExt};
 use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout};
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_target::abi::{self, Abi, F32, F64, FieldsShape, Int, Integer, Pointer, PointeeInfo, Size, TyAbiInterface, Variants};
@@ -253,7 +253,7 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
             Int(i, false) => cx.type_from_unsigned_integer(i),
             F32 => cx.type_f32(),
             F64 => cx.type_f64(),
-            Pointer => {
+            Pointer(address_space) => {
                 // If we know the alignment, pick something better than i8.
                 let pointee =
                     if let Some(pointee) = self.pointee_info_at(cx, offset) {
@@ -262,7 +262,7 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
                     else {
                         cx.type_i8()
                     };
-                cx.type_ptr_to(pointee)
+                cx.type_ptr_to_ext(pointee, address_space)
             }
         }
     }
diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml
index 9c1bcd431..a7ba2f8b6 100644
--- a/compiler/rustc_codegen_llvm/Cargo.toml
+++ b/compiler/rustc_codegen_llvm/Cargo.toml
@@ -30,10 +30,11 @@ rustc_macros = { path = "../rustc_macros" }
 rustc_metadata = { path = "../rustc_metadata" }
 rustc_query_system = { path = "../rustc_query_system" }
 rustc_session = { path = "../rustc_session" }
-rustc_serialize = { path = "../rustc_serialize" }
 rustc_symbol_mangling = { path = "../rustc_symbol_mangling" }
 rustc_target = { path = "../rustc_target" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 rustc_ast = { path = "../rustc_ast" }
 rustc_span = { path = "../rustc_span" }
 tempfile = "3.2.0"
+serde = { version = "1", features = [ "derive" ]}
+serde_json = "1"
diff --git a/compiler/rustc_codegen_llvm/locales/en-US.ftl b/compiler/rustc_codegen_llvm/locales/en-US.ftl
new file mode 100644
index 000000000..e5df41737
--- /dev/null
+++ b/compiler/rustc_codegen_llvm/locales/en-US.ftl
@@ -0,0 +1,89 @@
+codegen_llvm_unknown_ctarget_feature =
+    unknown feature specified for `-Ctarget-feature`: `{$feature}`
+    .note = it is still passed through to the codegen backend
+    .possible_feature = you might have meant: `{$rust_feature}`
+    .consider_filing_feature_request = consider filing a feature request
+
+codegen_llvm_unknown_ctarget_feature_prefix =
+    unknown feature specified for `-Ctarget-feature`: `{$feature}`
+    .note = features must begin with a `+` to enable or `-` to disable it
+
+codegen_llvm_error_creating_import_library =
+    Error creating import library for {$lib_name}: {$error}
+
+codegen_llvm_symbol_already_defined =
+    symbol `{$symbol_name}` is already defined
+
+codegen_llvm_invalid_minimum_alignment =
+    invalid minimum global alignment: {$err}
+
+codegen_llvm_sanitizer_memtag_requires_mte =
+    `-Zsanitizer=memtag` requires `-Ctarget-feature=+mte`
+
+codegen_llvm_error_writing_def_file =
+    Error writing .DEF file: {$error}
+
+codegen_llvm_error_calling_dlltool =
+    Error calling dlltool: {$error}
+
+codegen_llvm_dlltool_fail_import_library =
+    Dlltool could not create import library: {$stdout}\n{$stderr}
+
+codegen_llvm_target_feature_disable_or_enable =
+    the target features {$features} must all be either enabled or disabled together
+
+codegen_llvm_missing_features =
+    add the missing features in a `target_feature` attribute
+
+codegen_llvm_dynamic_linking_with_lto =
+    cannot prefer dynamic linking when performing LTO
+    .note = only 'staticlib', 'bin', and 'cdylib' outputs are supported with LTO
+
+codegen_llvm_parse_target_machine_config =
+    failed to parse target machine config to target machine: {$error}
+
+codegen_llvm_lto_disallowed = lto can only be run for executables, cdylibs and static library outputs
+
+codegen_llvm_lto_dylib = lto cannot be used for `dylib` crate type without `-Zdylib-lto`
+
+codegen_llvm_lto_bitcode_from_rlib = failed to get bitcode from object file for LTO ({$llvm_err})
+
+codegen_llvm_write_output = could not write output to {$path}
+codegen_llvm_write_output_with_llvm_err = could not write output to {$path}: {$llvm_err}
+
+codegen_llvm_target_machine = could not create LLVM TargetMachine for triple: {$triple}
+codegen_llvm_target_machine_with_llvm_err = could not create LLVM TargetMachine for triple: {$triple}: {$llvm_err}
+
+codegen_llvm_run_passes = failed to run LLVM passes
+codegen_llvm_run_passes_with_llvm_err = failed to run LLVM passes: {$llvm_err}
+
+codegen_llvm_serialize_module = failed to serialize module {$name}
+codegen_llvm_serialize_module_with_llvm_err = failed to serialize module {$name}: {$llvm_err}
+
+codegen_llvm_write_ir = failed to write LLVM IR to {$path}
+codegen_llvm_write_ir_with_llvm_err = failed to write LLVM IR to {$path}: {$llvm_err}
+
+codegen_llvm_prepare_thin_lto_context = failed to prepare thin LTO context
+codegen_llvm_prepare_thin_lto_context_with_llvm_err = failed to prepare thin LTO context: {$llvm_err}
+
+codegen_llvm_load_bitcode = failed to load bitcode of module "{$name}"
+codegen_llvm_load_bitcode_with_llvm_err = failed to load bitcode of module "{$name}": {$llvm_err}
+
+codegen_llvm_write_thinlto_key = error while writing ThinLTO key data: {$err}
+codegen_llvm_write_thinlto_key_with_llvm_err = error while writing ThinLTO key data: {$err}: {$llvm_err}
+
+codegen_llvm_multiple_source_dicompileunit = multiple source DICompileUnits found
+codegen_llvm_multiple_source_dicompileunit_with_llvm_err = multiple source DICompileUnits found: {$llvm_err}
+
+codegen_llvm_prepare_thin_lto_module = failed to prepare thin LTO module
+codegen_llvm_prepare_thin_lto_module_with_llvm_err = failed to prepare thin LTO module: {$llvm_err}
+
+codegen_llvm_parse_bitcode = failed to parse bitcode for LTO module
+codegen_llvm_parse_bitcode_with_llvm_err = failed to parse bitcode for LTO module: {$llvm_err}
+
+codegen_llvm_from_llvm_optimization_diag = {$filename}:{$line}:{$column} {$pass_name}: {$message}
+codegen_llvm_from_llvm_diag = {$message}
+
+codegen_llvm_write_bytecode = failed to write bytecode to {$path}: {$err}
+
+codegen_llvm_copy_bitcode = failed to copy bitcode to object file: {$err}
diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs
index 52c8b5179..d9f8170a3 100644
--- a/compiler/rustc_codegen_llvm/src/asm.rs
+++ b/compiler/rustc_codegen_llvm/src/asm.rs
@@ -849,6 +849,7 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'
 /// Helper function to get the LLVM type for a Scalar. Pointers are returned as
 /// the equivalent integer type.
 fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Type {
+    let dl = &cx.tcx.data_layout;
     match scalar.primitive() {
         Primitive::Int(Integer::I8, _) => cx.type_i8(),
         Primitive::Int(Integer::I16, _) => cx.type_i16(),
@@ -856,7 +857,8 @@ fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Ty
         Primitive::Int(Integer::I64, _) => cx.type_i64(),
         Primitive::F32 => cx.type_f32(),
         Primitive::F64 => cx.type_f64(),
-        Primitive::Pointer => cx.type_isize(),
+        // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
+        Primitive::Pointer(_) => cx.type_from_integer(dl.ptr_sized_integer()),
         _ => unreachable!(),
     }
 }
@@ -868,6 +870,7 @@ fn llvm_fixup_input<'ll, 'tcx>(
     reg: InlineAsmRegClass,
     layout: &TyAndLayout<'tcx>,
 ) -> &'ll Value {
+    let dl = &bx.tcx.data_layout;
     match (reg, layout.abi) {
         (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
             if let Primitive::Int(Integer::I8, _) = s.primitive() {
@@ -881,8 +884,10 @@ fn llvm_fixup_input<'ll, 'tcx>(
             let elem_ty = llvm_asm_scalar_type(bx.cx, s);
             let count = 16 / layout.size.bytes();
             let vec_ty = bx.cx.type_vector(elem_ty, count);
-            if let Primitive::Pointer = s.primitive() {
-                value = bx.ptrtoint(value, bx.cx.type_isize());
+            // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
+            if let Primitive::Pointer(_) = s.primitive() {
+                let t = bx.type_from_integer(dl.ptr_sized_integer());
+                value = bx.ptrtoint(value, t);
             }
             bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
         }
@@ -958,7 +963,7 @@ fn llvm_fixup_output<'ll, 'tcx>(
         }
         (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => {
             value = bx.extract_element(value, bx.const_i32(0));
-            if let Primitive::Pointer = s.primitive() {
+            if let Primitive::Pointer(_) = s.primitive() {
                 value = bx.inttoptr(value, layout.llvm_type(bx.cx));
             }
             value
diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs
index 95baa95b0..651d644eb 100644
--- a/compiler/rustc_codegen_llvm/src/attributes.rs
+++ b/compiler/rustc_codegen_llvm/src/attributes.rs
@@ -62,7 +62,7 @@ pub fn sanitize_attrs<'ll>(
 ) -> SmallVec<[&'ll Attribute; 4]> {
     let mut attrs = SmallVec::new();
     let enabled = cx.tcx.sess.opts.unstable_opts.sanitizer - no_sanitize;
-    if enabled.contains(SanitizerSet::ADDRESS) {
+    if enabled.contains(SanitizerSet::ADDRESS) || enabled.contains(SanitizerSet::KERNELADDRESS) {
         attrs.push(llvm::AttributeKind::SanitizeAddress.create_attr(cx.llcx));
     }
     if enabled.contains(SanitizerSet::MEMORY) {
@@ -118,7 +118,8 @@ pub fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attr
 
 /// Tell LLVM what instrument function to insert.
 #[inline]
-fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> {
+fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 4]> {
+    let mut attrs = SmallVec::new();
     if cx.sess().opts.unstable_opts.instrument_mcount {
         // Similar to `clang -pg` behavior. Handled by the
         // `post-inline-ee-instrument` LLVM pass.
@@ -127,14 +128,41 @@ fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribu
         // See test/CodeGen/mcount.c in clang.
         let mcount_name = cx.sess().target.mcount.as_ref();
 
-        Some(llvm::CreateAttrStringValue(
+        attrs.push(llvm::CreateAttrStringValue(
             cx.llcx,
             "instrument-function-entry-inlined",
             &mcount_name,
-        ))
-    } else {
-        None
+        ));
+    }
+    if let Some(options) = &cx.sess().opts.unstable_opts.instrument_xray {
+        // XRay instrumentation is similar to __cyg_profile_func_{enter,exit}.
+        // Function prologue and epilogue are instrumented with NOP sleds,
+        // a runtime library later replaces them with detours into tracing code.
+        if options.always {
+            attrs.push(llvm::CreateAttrStringValue(cx.llcx, "function-instrument", "xray-always"));
+        }
+        if options.never {
+            attrs.push(llvm::CreateAttrStringValue(cx.llcx, "function-instrument", "xray-never"));
+        }
+        if options.ignore_loops {
+            attrs.push(llvm::CreateAttrString(cx.llcx, "xray-ignore-loops"));
+        }
+        // LLVM will not choose the default for us, but rather requires specific
+        // threshold in absence of "xray-always". Use the same default as Clang.
+        let threshold = options.instruction_threshold.unwrap_or(200);
+        attrs.push(llvm::CreateAttrStringValue(
+            cx.llcx,
+            "xray-instruction-threshold",
+            &threshold.to_string(),
+        ));
+        if options.skip_entry {
+            attrs.push(llvm::CreateAttrString(cx.llcx, "xray-skip-entry"));
+        }
+        if options.skip_exit {
+            attrs.push(llvm::CreateAttrString(cx.llcx, "xray-skip-exit"));
+        }
     }
+    attrs
 }
 
 fn nojumptables_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> {
@@ -441,7 +469,7 @@ pub fn from_fn_attrs<'ll, 'tcx>(
         // the WebAssembly specification, which has this feature. This won't be
         // needed when LLVM enables this `multivalue` feature by default.
         if !cx.tcx.is_closure(instance.def_id()) {
-            let abi = cx.tcx.fn_sig(instance.def_id()).abi();
+            let abi = cx.tcx.fn_sig(instance.def_id()).skip_binder().abi();
             if abi == Abi::Wasm {
                 function_features.push("+multivalue".to_string());
             }
diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs
index 58ca87524..dd3268d77 100644
--- a/compiler/rustc_codegen_llvm/src/back/archive.rs
+++ b/compiler/rustc_codegen_llvm/src/back/archive.rs
@@ -183,6 +183,12 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder {
             // able to control the *exact* spelling of each of the symbols that are being imported:
             // hence we don't want `dlltool` adding leading underscores automatically.
             let dlltool = find_binutils_dlltool(sess);
+            let temp_prefix = {
+                let mut path = PathBuf::from(&output_path);
+                path.pop();
+                path.push(lib_name);
+                path
+            };
             let result = std::process::Command::new(dlltool)
                 .args([
                     "-d",
@@ -192,6 +198,8 @@ impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder {
                     "-l",
                     output_path.to_str().unwrap(),
                     "--no-leading-underscore",
+                    "--temp-prefix",
+                    temp_prefix.to_str().unwrap(),
                 ])
                 .output();
 
diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs
index 6c0faf37a..d2e01708a 100644
--- a/compiler/rustc_codegen_llvm/src/back/lto.rs
+++ b/compiler/rustc_codegen_llvm/src/back/lto.rs
@@ -1,5 +1,7 @@
 use crate::back::write::{self, save_temp_bitcode, DiagnosticHandlers};
-use crate::errors::DynamicLinkingWithLTO;
+use crate::errors::{
+    DynamicLinkingWithLTO, LlvmError, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib,
+};
 use crate::llvm::{self, build_string};
 use crate::{LlvmCodegenBackend, ModuleLlvm};
 use object::read::archive::ArchiveFile;
@@ -77,15 +79,12 @@ fn prepare_lto(
         // Make sure we actually can run LTO
         for crate_type in cgcx.crate_types.iter() {
             if !crate_type_allows_lto(*crate_type) {
-                let e = diag_handler.fatal(
-                    "lto can only be run for executables, cdylibs and \
-                                            static library outputs",
-                );
-                return Err(e);
+                diag_handler.emit_err(LtoDisallowed);
+                return Err(FatalError);
             } else if *crate_type == CrateType::Dylib {
                 if !cgcx.opts.unstable_opts.dylib_lto {
-                    return Err(diag_handler
-                        .fatal("lto cannot be used for `dylib` crate type without `-Zdylib-lto`"));
+                    diag_handler.emit_err(LtoDylib);
+                    return Err(FatalError);
                 }
             }
         }
@@ -127,7 +126,10 @@ fn prepare_lto(
                         let module = SerializedModule::FromRlib(data.to_vec());
                         upstream_modules.push((module, CString::new(name).unwrap()));
                     }
-                    Err(msg) => return Err(diag_handler.fatal(&msg)),
+                    Err(e) => {
+                        diag_handler.emit_err(e);
+                        return Err(FatalError);
+                    }
                 }
             }
         }
@@ -140,7 +142,7 @@ fn prepare_lto(
     Ok((symbols_below_threshold, upstream_modules))
 }
 
-fn get_bitcode_slice_from_object_data(obj: &[u8]) -> Result<&[u8], String> {
+fn get_bitcode_slice_from_object_data(obj: &[u8]) -> Result<&[u8], LtoBitcodeFromRlib> {
     let mut len = 0;
     let data =
         unsafe { llvm::LLVMRustGetBitcodeSliceFromObjectData(obj.as_ptr(), obj.len(), &mut len) };
@@ -155,8 +157,9 @@ fn get_bitcode_slice_from_object_data(obj: &[u8]) -> Result<&[u8], String> {
         Ok(bc)
     } else {
         assert!(len == 0);
-        let msg = llvm::last_error().unwrap_or_else(|| "unknown LLVM error".to_string());
-        Err(format!("failed to get bitcode from object file for LTO ({})", msg))
+        Err(LtoBitcodeFromRlib {
+            llvm_err: llvm::last_error().unwrap_or_else(|| "unknown LLVM error".to_string()),
+        })
     }
 }
 
@@ -328,10 +331,9 @@ fn fat_lto(
                 });
             info!("linking {:?}", name);
             let data = bc_decoded.data();
-            linker.add(data).map_err(|()| {
-                let msg = format!("failed to load bitcode of module {:?}", name);
-                write::llvm_err(diag_handler, &msg)
-            })?;
+            linker
+                .add(data)
+                .map_err(|()| write::llvm_err(diag_handler, LlvmError::LoadBitcode { name }))?;
             serialized_bitcode.push(bc_decoded);
         }
         drop(linker);
@@ -489,7 +491,7 @@ fn thin_lto(
             symbols_below_threshold.as_ptr(),
             symbols_below_threshold.len() as u32,
         )
-        .ok_or_else(|| write::llvm_err(diag_handler, "failed to prepare thin LTO context"))?;
+        .ok_or_else(|| write::llvm_err(diag_handler, LlvmError::PrepareThinLtoContext))?;
 
         let data = ThinData(data);
 
@@ -562,8 +564,7 @@ fn thin_lto(
         // session, overwriting the previous serialized data (if any).
         if let Some(path) = key_map_path {
             if let Err(err) = curr_key_map.save_to_file(&path) {
-                let msg = format!("Error while writing ThinLTO key data: {}", err);
-                return Err(write::llvm_err(diag_handler, &msg));
+                return Err(write::llvm_err(diag_handler, LlvmError::WriteThinLtoKey { err }));
             }
         }
 
@@ -689,8 +690,7 @@ pub unsafe fn optimize_thin_module(
 
     let module_name = &thin_module.shared.module_names[thin_module.idx];
     let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, module_name.to_str().unwrap());
-    let tm =
-        (cgcx.tm_factory)(tm_factory_config).map_err(|e| write::llvm_err(&diag_handler, &e))?;
+    let tm = (cgcx.tm_factory)(tm_factory_config).map_err(|e| write::llvm_err(&diag_handler, e))?;
 
     // Right now the implementation we've got only works over serialized
     // modules, so we create a fresh new LLVM context and parse the module
@@ -717,8 +717,7 @@ pub unsafe fn optimize_thin_module(
         let mut cu2 = ptr::null_mut();
         llvm::LLVMRustThinLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2);
         if !cu2.is_null() {
-            let msg = "multiple source DICompileUnits found";
-            return Err(write::llvm_err(&diag_handler, msg));
+            return Err(write::llvm_err(&diag_handler, LlvmError::MultipleSourceDiCompileUnit));
         }
 
         // Up next comes the per-module local analyses that we do for Thin LTO.
@@ -733,8 +732,7 @@ pub unsafe fn optimize_thin_module(
             let _timer =
                 cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_rename", thin_module.name());
             if !llvm::LLVMRustPrepareThinLTORename(thin_module.shared.data.0, llmod, target) {
-                let msg = "failed to prepare thin LTO module";
-                return Err(write::llvm_err(&diag_handler, msg));
+                return Err(write::llvm_err(&diag_handler, LlvmError::PrepareThinLtoModule));
             }
             save_temp_bitcode(cgcx, &module, "thin-lto-after-rename");
         }
@@ -744,8 +742,7 @@ pub unsafe fn optimize_thin_module(
                 .prof
                 .generic_activity_with_arg("LLVM_thin_lto_resolve_weak", thin_module.name());
             if !llvm::LLVMRustPrepareThinLTOResolveWeak(thin_module.shared.data.0, llmod) {
-                let msg = "failed to prepare thin LTO module";
-                return Err(write::llvm_err(&diag_handler, msg));
+                return Err(write::llvm_err(&diag_handler, LlvmError::PrepareThinLtoModule));
             }
             save_temp_bitcode(cgcx, &module, "thin-lto-after-resolve");
         }
@@ -755,8 +752,7 @@ pub unsafe fn optimize_thin_module(
                 .prof
                 .generic_activity_with_arg("LLVM_thin_lto_internalize", thin_module.name());
             if !llvm::LLVMRustPrepareThinLTOInternalize(thin_module.shared.data.0, llmod) {
-                let msg = "failed to prepare thin LTO module";
-                return Err(write::llvm_err(&diag_handler, msg));
+                return Err(write::llvm_err(&diag_handler, LlvmError::PrepareThinLtoModule));
             }
             save_temp_bitcode(cgcx, &module, "thin-lto-after-internalize");
         }
@@ -765,8 +761,7 @@ pub unsafe fn optimize_thin_module(
             let _timer =
                 cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_import", thin_module.name());
             if !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod, target) {
-                let msg = "failed to prepare thin LTO module";
-                return Err(write::llvm_err(&diag_handler, msg));
+                return Err(write::llvm_err(&diag_handler, LlvmError::PrepareThinLtoModule));
             }
             save_temp_bitcode(cgcx, &module, "thin-lto-after-import");
         }
@@ -886,11 +881,7 @@ pub fn parse_module<'a>(
     diag_handler: &Handler,
 ) -> Result<&'a llvm::Module, FatalError> {
     unsafe {
-        llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr()).ok_or_else(
-            || {
-                let msg = "failed to parse bitcode for LTO module";
-                write::llvm_err(diag_handler, msg)
-            },
-        )
+        llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr())
+            .ok_or_else(|| write::llvm_err(diag_handler, LlvmError::ParseBitcode))
     }
 }
diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs
index b2af9f31e..a4ae1b01e 100644
--- a/compiler/rustc_codegen_llvm/src/back/write.rs
+++ b/compiler/rustc_codegen_llvm/src/back/write.rs
@@ -5,6 +5,9 @@ use crate::back::profiling::{
 use crate::base;
 use crate::common;
 use crate::consts;
+use crate::errors::{
+    CopyBitcode, FromLlvmDiag, FromLlvmOptimizationDiag, LlvmError, WithLlvmError, WriteBytecode,
+};
 use crate::llvm::{self, DiagnosticInfo, PassManager};
 use crate::llvm_util;
 use crate::type_::Type;
@@ -37,10 +40,10 @@ use std::slice;
 use std::str;
 use std::sync::Arc;
 
-pub fn llvm_err(handler: &rustc_errors::Handler, msg: &str) -> FatalError {
+pub fn llvm_err<'a>(handler: &rustc_errors::Handler, err: LlvmError<'a>) -> FatalError {
     match llvm::last_error() {
-        Some(err) => handler.fatal(&format!("{}: {}", msg, err)),
-        None => handler.fatal(msg),
+        Some(llvm_err) => handler.emit_almost_fatal(WithLlvmError(err, llvm_err)),
+        None => handler.emit_almost_fatal(err),
     }
 }
 
@@ -85,10 +88,9 @@ pub fn write_output_file<'ll>(
             }
         }
 
-        result.into_result().map_err(|()| {
-            let msg = format!("could not write output to {}", output.display());
-            llvm_err(handler, &msg)
-        })
+        result
+            .into_result()
+            .map_err(|()| llvm_err(handler, LlvmError::WriteOutput { path: output }))
     }
 }
 
@@ -98,7 +100,7 @@ pub fn create_informational_target_machine(sess: &Session) -> &'static mut llvm:
     // system/tcx is set up.
     let features = llvm_util::global_llvm_features(sess, false);
     target_machine_factory(sess, config::OptLevel::No, &features)(config)
-        .unwrap_or_else(|err| llvm_err(sess.diagnostic(), &err).raise())
+        .unwrap_or_else(|err| llvm_err(sess.diagnostic(), err).raise())
 }
 
 pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut llvm::TargetMachine {
@@ -117,7 +119,7 @@ pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut ll
         tcx.backend_optimization_level(()),
         tcx.global_backend_features(()),
     )(config)
-    .unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), &err).raise())
+    .unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), err).raise())
 }
 
 pub fn to_llvm_opt_settings(
@@ -240,9 +242,7 @@ pub fn target_machine_factory(
             )
         };
 
-        tm.ok_or_else(|| {
-            format!("Could not create LLVM TargetMachine for triple: {}", triple.to_str().unwrap())
-        })
+        tm.ok_or_else(|| LlvmError::CreateTargetMachine { triple: triple.clone() })
     })
 }
 
@@ -355,25 +355,28 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void
             };
 
             if enabled {
-                diag_handler.note_without_error(&format!(
-                    "{}:{}:{}: {}: {}",
-                    opt.filename, opt.line, opt.column, opt.pass_name, opt.message,
-                ));
+                diag_handler.emit_note(FromLlvmOptimizationDiag {
+                    filename: &opt.filename,
+                    line: opt.line,
+                    column: opt.column,
+                    pass_name: &opt.pass_name,
+                    message: &opt.message,
+                });
             }
         }
         llvm::diagnostic::PGO(diagnostic_ref) | llvm::diagnostic::Linker(diagnostic_ref) => {
-            let msg = llvm::build_string(|s| {
+            let message = llvm::build_string(|s| {
                 llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s)
             })
             .expect("non-UTF8 diagnostic");
-            diag_handler.warn(&msg);
+            diag_handler.emit_warning(FromLlvmDiag { message });
         }
         llvm::diagnostic::Unsupported(diagnostic_ref) => {
-            let msg = llvm::build_string(|s| {
+            let message = llvm::build_string(|s| {
                 llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s)
             })
             .expect("non-UTF8 diagnostic");
-            diag_handler.err(&msg);
+            diag_handler.emit_err(FromLlvmDiag { message });
         }
         llvm::diagnostic::UnknownDiagnostic(..) => {}
     }
@@ -409,11 +412,7 @@ fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option<CString> {
 }
 
 fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> {
-    if config.instrument_coverage {
-        Some(CString::new("default_%m_%p.profraw").unwrap())
-    } else {
-        None
-    }
+    config.instrument_coverage.then(|| CString::new("default_%m_%p.profraw").unwrap())
 }
 
 pub(crate) unsafe fn llvm_optimize(
@@ -443,16 +442,19 @@ pub(crate) unsafe fn llvm_optimize(
             sanitize_thread: config.sanitizer.contains(SanitizerSet::THREAD),
             sanitize_hwaddress: config.sanitizer.contains(SanitizerSet::HWADDRESS),
             sanitize_hwaddress_recover: config.sanitizer_recover.contains(SanitizerSet::HWADDRESS),
+            sanitize_kernel_address: config.sanitizer.contains(SanitizerSet::KERNELADDRESS),
+            sanitize_kernel_address_recover: config
+                .sanitizer_recover
+                .contains(SanitizerSet::KERNELADDRESS),
         })
     } else {
         None
     };
 
-    let mut llvm_profiler = if cgcx.prof.llvm_recording_enabled() {
-        Some(LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap()))
-    } else {
-        None
-    };
+    let mut llvm_profiler = cgcx
+        .prof
+        .llvm_recording_enabled()
+        .then(|| LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap()));
 
     let llvm_selfprofiler =
         llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut());
@@ -494,7 +496,7 @@ pub(crate) unsafe fn llvm_optimize(
         llvm_plugins.as_ptr().cast(),
         llvm_plugins.len(),
     );
-    result.into_result().map_err(|()| llvm_err(diag_handler, "failed to run LLVM passes"))
+    result.into_result().map_err(|()| llvm_err(diag_handler, LlvmError::RunLlvmPasses))
 }
 
 // Unsafe due to LLVM calls.
@@ -547,8 +549,7 @@ pub(crate) fn link(
         let _timer = cgcx.prof.generic_activity_with_arg("LLVM_link_module", &*module.name);
         let buffer = ModuleBuffer::new(module.module_llvm.llmod());
         linker.add(buffer.data()).map_err(|()| {
-            let msg = format!("failed to serialize module {:?}", module.name);
-            llvm_err(diag_handler, &msg)
+            llvm_err(diag_handler, LlvmError::SerializeModule { name: &module.name })
         })?;
     }
     drop(linker);
@@ -626,9 +627,8 @@ pub(crate) unsafe fn codegen(
                 let _timer = cgcx
                     .prof
                     .generic_activity_with_arg("LLVM_module_codegen_emit_bitcode", &*module.name);
-                if let Err(e) = fs::write(&bc_out, data) {
-                    let msg = format!("failed to write bytecode to {}: {}", bc_out.display(), e);
-                    diag_handler.err(&msg);
+                if let Err(err) = fs::write(&bc_out, data) {
+                    diag_handler.emit_err(WriteBytecode { path: &bc_out, err });
                 }
             }
 
@@ -678,10 +678,9 @@ pub(crate) unsafe fn codegen(
                 record_artifact_size(&cgcx.prof, "llvm_ir", &out);
             }
 
-            result.into_result().map_err(|()| {
-                let msg = format!("failed to write LLVM IR to {}", out.display());
-                llvm_err(diag_handler, &msg)
-            })?;
+            result
+                .into_result()
+                .map_err(|()| llvm_err(diag_handler, LlvmError::WriteIr { path: &out }))?;
         }
 
         if config.emit_asm {
@@ -749,8 +748,8 @@ pub(crate) unsafe fn codegen(
 
             EmitObj::Bitcode => {
                 debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out);
-                if let Err(e) = link_or_copy(&bc_out, &obj_out) {
-                    diag_handler.err(&format!("failed to copy bitcode to object file: {}", e));
+                if let Err(err) = link_or_copy(&bc_out, &obj_out) {
+                    diag_handler.emit_err(CopyBitcode { err });
                 }
 
                 if !config.emit_bc {
@@ -762,6 +761,7 @@ pub(crate) unsafe fn codegen(
             EmitObj::None => {}
         }
 
+        record_llvm_cgu_instructions_stats(&cgcx.prof, llmod);
         drop(handlers);
     }
 
@@ -975,3 +975,23 @@ fn record_artifact_size(
         self_profiler_ref.artifact_size(artifact_kind, artifact_name.to_string_lossy(), file_size);
     }
 }
+
+fn record_llvm_cgu_instructions_stats(prof: &SelfProfilerRef, llmod: &llvm::Module) {
+    if !prof.enabled() {
+        return;
+    }
+
+    let raw_stats =
+        llvm::build_string(|s| unsafe { llvm::LLVMRustModuleInstructionStats(&llmod, s) })
+            .expect("cannot get module instruction stats");
+
+    #[derive(serde::Deserialize)]
+    struct InstructionsStats {
+        module: String,
+        total: u64,
+    }
+
+    let InstructionsStats { module, total } =
+        serde_json::from_str(&raw_stats).expect("cannot parse llvm cgu instructions stats");
+    prof.artifact_size("cgu_instructions", module, total);
+}
diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs
index 5e98deae4..0f33b9854 100644
--- a/compiler/rustc_codegen_llvm/src/builder.rs
+++ b/compiler/rustc_codegen_llvm/src/builder.rs
@@ -511,7 +511,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
                         bx.range_metadata(load, scalar.valid_range(bx));
                     }
                 }
-                abi::Pointer => {
+                abi::Pointer(_) => {
                     if !scalar.valid_range(bx).contains(0) {
                         bx.nonnull_metadata(load);
                     }
diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs
index f1d01a460..6ee2a05ff 100644
--- a/compiler/rustc_codegen_llvm/src/callee.rs
+++ b/compiler/rustc_codegen_llvm/src/callee.rs
@@ -13,7 +13,7 @@ use crate::value::Value;
 use rustc_codegen_ssa::traits::*;
 
 use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt};
-use rustc_middle::ty::{self, Instance, TypeVisitable};
+use rustc_middle::ty::{self, Instance, TypeVisitableExt};
 
 /// Codegens a reference to a fn/method item, monomorphizing and
 /// inlining as it goes.
diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs
index acee9134f..b0a9a30ab 100644
--- a/compiler/rustc_codegen_llvm/src/common.rs
+++ b/compiler/rustc_codegen_llvm/src/common.rs
@@ -10,6 +10,7 @@ use crate::value::Value;
 use rustc_ast::Mutability;
 use rustc_codegen_ssa::mir::place::PlaceRef;
 use rustc_codegen_ssa::traits::*;
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_hir::def_id::DefId;
 use rustc_middle::bug;
 use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar};
@@ -236,7 +237,7 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
             Scalar::Int(int) => {
                 let data = int.assert_bits(layout.size(self));
                 let llval = self.const_uint_big(self.type_ix(bitsize), data);
-                if layout.primitive() == Pointer {
+                if matches!(layout.primitive(), Pointer(_)) {
                     unsafe { llvm::LLVMConstIntToPtr(llval, llty) }
                 } else {
                     self.const_bitcast(llval, llty)
@@ -252,8 +253,13 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                             Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None),
                             _ => self.static_addr_of(init, alloc.align, None),
                         };
-                        if !self.sess().fewer_names() {
-                            llvm::set_value_name(value, format!("{:?}", alloc_id).as_bytes());
+                        if !self.sess().fewer_names() && llvm::get_value_name(value).is_empty() {
+                            let hash = self.tcx.with_stable_hashing_context(|mut hcx| {
+                                let mut hasher = StableHasher::new();
+                                alloc.hash_stable(&mut hcx, &mut hasher);
+                                hasher.finish::<u128>()
+                            });
+                            llvm::set_value_name(value, format!("alloc_{hash:032x}").as_bytes());
                         }
                         (value, AddressSpace::DATA)
                     }
@@ -284,7 +290,7 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                         1,
                     )
                 };
-                if layout.primitive() != Pointer {
+                if !matches!(layout.primitive(), Pointer(_)) {
                     unsafe { llvm::LLVMConstPtrToInt(llval, llty) }
                 } else {
                     self.const_bitcast(llval, llty)
diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs
index 16467b614..9116e71be 100644
--- a/compiler/rustc_codegen_llvm/src/consts.rs
+++ b/compiler/rustc_codegen_llvm/src/consts.rs
@@ -3,7 +3,6 @@ use crate::common::{self, CodegenCx};
 use crate::debuginfo;
 use crate::errors::{InvalidMinimumAlignment, SymbolAlreadyDefined};
 use crate::llvm::{self, True};
-use crate::llvm_util;
 use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
@@ -13,7 +12,7 @@ use rustc_codegen_ssa::traits::*;
 use rustc_hir::def_id::DefId;
 use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
 use rustc_middle::mir::interpret::{
-    read_target_uint, Allocation, ConstAllocation, ErrorHandled, GlobalAlloc, InitChunk, Pointer,
+    read_target_uint, Allocation, ConstAllocation, ErrorHandled, InitChunk, Pointer,
     Scalar as InterpScalar,
 };
 use rustc_middle::mir::mono::MonoItem;
@@ -21,9 +20,7 @@ use rustc_middle::ty::layout::LayoutOf;
 use rustc_middle::ty::{self, Instance, Ty};
 use rustc_middle::{bug, span_bug};
 use rustc_session::config::Lto;
-use rustc_target::abi::{
-    AddressSpace, Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange,
-};
+use rustc_target::abi::{Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange};
 use std::ops::Range;
 
 pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<'_>) -> &'ll Value {
@@ -58,13 +55,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<
         // to avoid the cost of generating large complex const expressions.
         // For example, `[(u32, u8); 1024 * 1024]` contains uninit padding in each element,
         // and would result in `{ [5 x i8] zeroinitializer, [3 x i8] undef, ...repeat 1M times... }`.
-        let max = if llvm_util::get_version() < (14, 0, 0) {
-            // Generating partially-uninit consts inhibits optimizations in LLVM < 14.
-            // See https://github.com/rust-lang/rust/issues/84565.
-            1
-        } else {
-            cx.sess().opts.unstable_opts.uninit_const_chunk_threshold
-        };
+        let max = cx.sess().opts.unstable_opts.uninit_const_chunk_threshold;
         let allow_uninit_chunks = chunks.clone().take(max.saturating_add(1)).count() <= max;
 
         if allow_uninit_chunks {
@@ -98,12 +89,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<
         .expect("const_alloc_to_llvm: could not read relocation pointer")
             as u64;
 
-        let address_space = match cx.tcx.global_alloc(alloc_id) {
-            GlobalAlloc::Function(..) => cx.data_layout().instruction_address_space,
-            GlobalAlloc::Static(..) | GlobalAlloc::Memory(..) | GlobalAlloc::VTable(..) => {
-                AddressSpace::DATA
-            }
-        };
+        let address_space = cx.tcx.global_alloc(alloc_id).address_space(cx);
 
         llvals.push(cx.scalar_to_backend(
             InterpScalar::from_pointer(
@@ -111,7 +97,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<
                 &cx.tcx,
             ),
             Scalar::Initialized {
-                value: Primitive::Pointer,
+                value: Primitive::Pointer(address_space),
                 valid_range: WrappingRange::full(dl.pointer_size),
             },
             cx.type_i8p_ext(address_space),
@@ -535,7 +521,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
 
                 // The semantics of #[used] in Rust only require the symbol to make it into the
                 // object file. It is explicitly allowed for the linker to strip the symbol if it
-                // is dead, which means we are allowed use `llvm.compiler.used` instead of
+                // is dead, which means we are allowed to use `llvm.compiler.used` instead of
                 // `llvm.used` here.
                 //
                 // Additionally, https://reviews.llvm.org/D97448 in LLVM 13 started emitting unique
@@ -546,7 +532,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
                 // That said, we only ever emit these when compiling for ELF targets, unless
                 // `#[used(compiler)]` is explicitly requested. This is to avoid similar breakage
                 // on other targets, in particular MachO targets have *their* static constructor
-                // lists broken if `llvm.compiler.used` is emitted rather than llvm.used. However,
+                // lists broken if `llvm.compiler.used` is emitted rather than `llvm.used`. However,
                 // that check happens when assigning the `CodegenFnAttrFlags` in `rustc_hir_analysis`,
                 // so we don't need to take care of it here.
                 self.add_compiler_used_global(g);
diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs
index d9ccba07a..3d29968d5 100644
--- a/compiler/rustc_codegen_llvm/src/context.rs
+++ b/compiler/rustc_codegen_llvm/src/context.rs
@@ -143,24 +143,15 @@ pub unsafe fn create_module<'ll>(
 
     let mut target_data_layout = sess.target.data_layout.to_string();
     let llvm_version = llvm_util::get_version();
-    if llvm_version < (14, 0, 0) {
-        if sess.target.llvm_target == "i686-pc-windows-msvc"
-            || sess.target.llvm_target == "i586-pc-windows-msvc"
-        {
-            target_data_layout =
-                "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:32-n8:16:32-a:0:32-S32"
-                    .to_string();
-        }
-        if sess.target.arch == "wasm32" {
-            target_data_layout = target_data_layout.replace("-p10:8:8-p20:8:8", "");
-        }
-    }
     if llvm_version < (16, 0, 0) {
         if sess.target.arch == "s390x" {
+            // LLVM 16 data layout changed to always set 64-bit vector alignment,
+            // which is conditional in earlier LLVM versions.
+            // https://reviews.llvm.org/D131158 for the discussion.
             target_data_layout = target_data_layout.replace("-v128:64", "");
-        }
-
-        if sess.target.arch == "riscv64" {
+        } else if sess.target.arch == "riscv64" {
+            // LLVM 16 introduced this change so as to produce more efficient code.
+            // See https://reviews.llvm.org/D116735 for the discussion.
             target_data_layout = target_data_layout.replace("-n32:64-", "-n64-");
         }
     }
@@ -191,7 +182,7 @@ pub unsafe fn create_module<'ll>(
         //
         // FIXME(#34960)
         let cfg_llvm_root = option_env!("CFG_LLVM_ROOT").unwrap_or("");
-        let custom_llvm_used = cfg_llvm_root.trim() != "";
+        let custom_llvm_used = !cfg_llvm_root.trim().is_empty();
 
         if !custom_llvm_used && target_data_layout != llvm_data_layout {
             bug!(
@@ -416,12 +407,8 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
 
         let (llcx, llmod) = (&*llvm_module.llcx, llvm_module.llmod());
 
-        let coverage_cx = if tcx.sess.instrument_coverage() {
-            let covctx = coverageinfo::CrateCoverageContext::new();
-            Some(covctx)
-        } else {
-            None
-        };
+        let coverage_cx =
+            tcx.sess.instrument_coverage().then(coverageinfo::CrateCoverageContext::new);
 
         let dbg_cx = if tcx.sess.opts.debuginfo != DebugInfo::None {
             let dctx = debuginfo::CodegenUnitDebugContext::new(llmod);
@@ -533,14 +520,9 @@ impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         let tcx = self.tcx;
         let llfn = match tcx.lang_items().eh_personality() {
             Some(def_id) if !wants_msvc_seh(self.sess()) => self.get_fn_addr(
-                ty::Instance::resolve(
-                    tcx,
-                    ty::ParamEnv::reveal_all(),
-                    def_id,
-                    tcx.intern_substs(&[]),
-                )
-                .unwrap()
-                .unwrap(),
+                ty::Instance::resolve(tcx, ty::ParamEnv::reveal_all(), def_id, ty::List::empty())
+                    .unwrap()
+                    .unwrap(),
             ),
             _ => {
                 let name = if wants_msvc_seh(self.sess()) {
diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
index 22c61248b..240a9d2f3 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
@@ -1,6 +1,5 @@
 use crate::common::CodegenCx;
 use crate::coverageinfo;
-use crate::errors::InstrumentCoverageRequiresLLVM12;
 use crate::llvm;
 
 use llvm::coverageinfo::CounterMappingRegion;
@@ -19,8 +18,8 @@ use std::ffi::CString;
 
 /// Generates and exports the Coverage Map.
 ///
-/// Rust Coverage Map generation supports LLVM Coverage Mapping Format versions
-/// 5 (LLVM 12, only) and 6 (zero-based encoded as 4 and 5, respectively), as defined at
+/// Rust Coverage Map generation supports LLVM Coverage Mapping Format version
+/// 6 (zero-based encoded as 5), as defined at
 /// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format).
 /// These versions are supported by the LLVM coverage tools (`llvm-profdata` and `llvm-cov`)
 /// bundled with Rust's fork of LLVM.
@@ -33,13 +32,10 @@ use std::ffi::CString;
 pub fn finalize(cx: &CodegenCx<'_, '_>) {
     let tcx = cx.tcx;
 
-    // Ensure the installed version of LLVM supports at least Coverage Map
-    // Version 5 (encoded as a zero-based value: 4), which was introduced with
-    // LLVM 12.
+    // Ensure the installed version of LLVM supports Coverage Map Version 6
+    // (encoded as a zero-based value: 5), which was introduced with LLVM 13.
     let version = coverageinfo::mapping_version();
-    if version < 4 {
-        tcx.sess.emit_fatal(InstrumentCoverageRequiresLLVM12);
-    }
+    assert_eq!(version, 5, "The `CoverageMappingVersion` exposed by `llvm-wrapper` is out of sync");
 
     debug!("Generating coverage map for CodegenUnit: `{}`", cx.codegen_unit.name());
 
@@ -61,7 +57,7 @@ pub fn finalize(cx: &CodegenCx<'_, '_>) {
         return;
     }
 
-    let mut mapgen = CoverageMapGenerator::new(tcx, version);
+    let mut mapgen = CoverageMapGenerator::new(tcx);
 
     // Encode coverage mappings and generate function records
     let mut function_data = Vec::new();
@@ -124,25 +120,18 @@ struct CoverageMapGenerator {
 }
 
 impl CoverageMapGenerator {
-    fn new(tcx: TyCtxt<'_>, version: u32) -> Self {
+    fn new(tcx: TyCtxt<'_>) -> Self {
         let mut filenames = FxIndexSet::default();
-        if version >= 5 {
-            // LLVM Coverage Mapping Format version 6 (zero-based encoded as 5)
-            // requires setting the first filename to the compilation directory.
-            // Since rustc generates coverage maps with relative paths, the
-            // compilation directory can be combined with the relative paths
-            // to get absolute paths, if needed.
-            let working_dir = tcx
-                .sess
-                .opts
-                .working_dir
-                .remapped_path_if_available()
-                .to_string_lossy()
-                .to_string();
-            let c_filename =
-                CString::new(working_dir).expect("null error converting filename to C string");
-            filenames.insert(c_filename);
-        }
+        // LLVM Coverage Mapping Format version 6 (zero-based encoded as 5)
+        // requires setting the first filename to the compilation directory.
+        // Since rustc generates coverage maps with relative paths, the
+        // compilation directory can be combined with the relative paths
+        // to get absolute paths, if needed.
+        let working_dir =
+            tcx.sess.opts.working_dir.remapped_path_if_available().to_string_lossy().to_string();
+        let c_filename =
+            CString::new(working_dir).expect("null error converting filename to C string");
+        filenames.insert(c_filename);
         Self { filenames }
     }
 
@@ -306,9 +295,8 @@ fn add_unused_functions(cx: &CodegenCx<'_, '_>) {
                 DefKind::Fn | DefKind::AssocFn | DefKind::Closure | DefKind::Generator
             ) {
                 return None;
-            } else if ignore_unused_generics
-                && tcx.generics_of(def_id).requires_monomorphization(tcx)
-            {
+            }
+            if ignore_unused_generics && tcx.generics_of(def_id).requires_monomorphization(tcx) {
                 return None;
             }
             Some(local_def_id.to_def_id())
diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
index ace15cfb0..3dc0ac033 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
@@ -27,8 +27,6 @@ use rustc_middle::ty::Instance;
 use std::cell::RefCell;
 use std::ffi::CString;
 
-use std::iter;
-
 pub mod mapgen;
 
 const UNUSED_FUNCTION_COUNTER_ID: CounterValueReference = CounterValueReference::START;
@@ -201,7 +199,7 @@ fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance<
         tcx.symbol_name(instance).name,
         cx.fn_abi_of_fn_ptr(
             ty::Binder::dummy(tcx.mk_fn_sig(
-                iter::once(tcx.mk_unit()),
+                [tcx.mk_unit()],
                 tcx.mk_unit(),
                 false,
                 hir::Unsafety::Unsafe,
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
index b6eb5ee18..c1b3f34e5 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
@@ -132,7 +132,7 @@ fn build_fixed_size_array_di_node<'ll, 'tcx>(
 
     let (size, align) = cx.size_and_align_of(array_type);
 
-    let upper_bound = len.eval_usize(cx.tcx, ty::ParamEnv::reveal_all()) as c_longlong;
+    let upper_bound = len.eval_target_usize(cx.tcx, ty::ParamEnv::reveal_all()) as c_longlong;
 
     let subrange =
         unsafe { Some(llvm::LLVMRustDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound)) };
@@ -1499,6 +1499,11 @@ pub fn create_vtable_di_node<'ll, 'tcx>(
         return;
     }
 
+    // When full debuginfo is enabled, we want to try and prevent vtables from being
+    // merged. Otherwise debuggers will have a hard time mapping from dyn pointer
+    // to concrete type.
+    llvm::SetUnnamedAddress(vtable, llvm::UnnamedAddr::No);
+
     let vtable_name =
         compute_debuginfo_vtable_name(cx.tcx, ty, poly_trait_ref, VTableNameKind::GlobalVariable);
     let vtable_type_di_node = build_vtable_type_di_node(cx, ty, poly_trait_ref);
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs
index 564ab351b..54e850f25 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs
@@ -122,7 +122,8 @@ fn tag_base_type<'ll, 'tcx>(
                 Primitive::Int(t, _) => t,
                 Primitive::F32 => Integer::I32,
                 Primitive::F64 => Integer::I64,
-                Primitive::Pointer => {
+                // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
+                Primitive::Pointer(_) => {
                     // If the niche is the NULL value of a reference, then `discr_enum_ty` will be
                     // a RawPtr. CodeView doesn't know what to do with enums whose base type is a
                     // pointer so we fix this up to just be `usize`.
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
index ca7a07d83..5392534cf 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
@@ -27,7 +27,7 @@ use rustc_index::vec::IndexVec;
 use rustc_middle::mir;
 use rustc_middle::ty::layout::LayoutOf;
 use rustc_middle::ty::subst::{GenericArgKind, SubstsRef};
-use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeVisitable};
+use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeVisitableExt};
 use rustc_session::config::{self, DebugInfo};
 use rustc_session::Session;
 use rustc_span::symbol::Symbol;
@@ -508,7 +508,7 @@ impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                     let impl_self_ty = cx.tcx.subst_and_normalize_erasing_regions(
                         instance.substs,
                         ty::ParamEnv::reveal_all(),
-                        cx.tcx.type_of(impl_def_id),
+                        cx.tcx.type_of(impl_def_id).skip_binder(),
                     );
 
                     // Only "class" methods are generally understood by LLVM,
diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs
index b46209972..bae88d942 100644
--- a/compiler/rustc_codegen_llvm/src/errors.rs
+++ b/compiler/rustc_codegen_llvm/src/errors.rs
@@ -1,10 +1,12 @@
 use std::borrow::Cow;
-
-use rustc_errors::fluent;
-use rustc_errors::DiagnosticBuilder;
-use rustc_errors::ErrorGuaranteed;
-use rustc_errors::Handler;
-use rustc_errors::IntoDiagnostic;
+use std::ffi::CString;
+use std::path::Path;
+
+use crate::fluent_generated as fluent;
+use rustc_data_structures::small_c_str::SmallCStr;
+use rustc_errors::{
+    DiagnosticBuilder, EmissionGuarantee, ErrorGuaranteed, Handler, IntoDiagnostic,
+};
 use rustc_macros::{Diagnostic, Subdiagnostic};
 use rustc_span::Span;
 
@@ -26,9 +28,9 @@ pub(crate) struct UnknownCTargetFeature<'a> {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum PossibleFeature<'a> {
-    #[help(possible_feature)]
+    #[help(codegen_llvm_possible_feature)]
     Some { rust_feature: &'a str },
-    #[help(consider_filing_feature_request)]
+    #[help(codegen_llvm_consider_filing_feature_request)]
     None,
 }
 
@@ -40,10 +42,6 @@ pub(crate) struct ErrorCreatingImportLibrary<'a> {
 }
 
 #[derive(Diagnostic)]
-#[diag(codegen_llvm_instrument_coverage_requires_llvm_12)]
-pub(crate) struct InstrumentCoverageRequiresLLVM12;
-
-#[derive(Diagnostic)]
 #[diag(codegen_llvm_symbol_already_defined)]
 pub(crate) struct SymbolAlreadyDefined<'a> {
     #[primary_span]
@@ -85,10 +83,18 @@ pub(crate) struct DlltoolFailImportLibrary<'a> {
 #[note]
 pub(crate) struct DynamicLinkingWithLTO;
 
-#[derive(Diagnostic)]
-#[diag(codegen_llvm_fail_parsing_target_machine_config_to_target_machine)]
-pub(crate) struct FailParsingTargetMachineConfigToTargetMachine {
-    pub error: String,
+pub(crate) struct ParseTargetMachineConfig<'a>(pub LlvmError<'a>);
+
+impl<EM: EmissionGuarantee> IntoDiagnostic<'_, EM> for ParseTargetMachineConfig<'_> {
+    fn into_diagnostic(self, sess: &'_ Handler) -> DiagnosticBuilder<'_, EM> {
+        let diag: DiagnosticBuilder<'_, EM> = self.0.into_diagnostic(sess);
+        let (message, _) = diag.styled_message().first().expect("`LlvmError` with no message");
+        let message = sess.eagerly_translate_to_string(message.clone(), diag.args());
+
+        let mut diag = sess.struct_diagnostic(fluent::codegen_llvm_parse_target_machine_config);
+        diag.set_arg("error", message);
+        diag
+    }
 }
 
 pub(crate) struct TargetFeatureDisableOrEnable<'a> {
@@ -114,3 +120,99 @@ impl IntoDiagnostic<'_, ErrorGuaranteed> for TargetFeatureDisableOrEnable<'_> {
         diag
     }
 }
+
+#[derive(Diagnostic)]
+#[diag(codegen_llvm_lto_disallowed)]
+pub(crate) struct LtoDisallowed;
+
+#[derive(Diagnostic)]
+#[diag(codegen_llvm_lto_dylib)]
+pub(crate) struct LtoDylib;
+
+#[derive(Diagnostic)]
+#[diag(codegen_llvm_lto_bitcode_from_rlib)]
+pub(crate) struct LtoBitcodeFromRlib {
+    pub llvm_err: String,
+}
+
+#[derive(Diagnostic)]
+pub enum LlvmError<'a> {
+    #[diag(codegen_llvm_write_output)]
+    WriteOutput { path: &'a Path },
+    #[diag(codegen_llvm_target_machine)]
+    CreateTargetMachine { triple: SmallCStr },
+    #[diag(codegen_llvm_run_passes)]
+    RunLlvmPasses,
+    #[diag(codegen_llvm_serialize_module)]
+    SerializeModule { name: &'a str },
+    #[diag(codegen_llvm_write_ir)]
+    WriteIr { path: &'a Path },
+    #[diag(codegen_llvm_prepare_thin_lto_context)]
+    PrepareThinLtoContext,
+    #[diag(codegen_llvm_load_bitcode)]
+    LoadBitcode { name: CString },
+    #[diag(codegen_llvm_write_thinlto_key)]
+    WriteThinLtoKey { err: std::io::Error },
+    #[diag(codegen_llvm_multiple_source_dicompileunit)]
+    MultipleSourceDiCompileUnit,
+    #[diag(codegen_llvm_prepare_thin_lto_module)]
+    PrepareThinLtoModule,
+    #[diag(codegen_llvm_parse_bitcode)]
+    ParseBitcode,
+}
+
+pub(crate) struct WithLlvmError<'a>(pub LlvmError<'a>, pub String);
+
+impl<EM: EmissionGuarantee> IntoDiagnostic<'_, EM> for WithLlvmError<'_> {
+    fn into_diagnostic(self, sess: &'_ Handler) -> DiagnosticBuilder<'_, EM> {
+        use LlvmError::*;
+        let msg_with_llvm_err = match &self.0 {
+            WriteOutput { .. } => fluent::codegen_llvm_write_output_with_llvm_err,
+            CreateTargetMachine { .. } => fluent::codegen_llvm_target_machine_with_llvm_err,
+            RunLlvmPasses => fluent::codegen_llvm_run_passes_with_llvm_err,
+            SerializeModule { .. } => fluent::codegen_llvm_serialize_module_with_llvm_err,
+            WriteIr { .. } => fluent::codegen_llvm_write_ir_with_llvm_err,
+            PrepareThinLtoContext => fluent::codegen_llvm_prepare_thin_lto_context_with_llvm_err,
+            LoadBitcode { .. } => fluent::codegen_llvm_load_bitcode_with_llvm_err,
+            WriteThinLtoKey { .. } => fluent::codegen_llvm_write_thinlto_key_with_llvm_err,
+            MultipleSourceDiCompileUnit => {
+                fluent::codegen_llvm_multiple_source_dicompileunit_with_llvm_err
+            }
+            PrepareThinLtoModule => fluent::codegen_llvm_prepare_thin_lto_module_with_llvm_err,
+            ParseBitcode => fluent::codegen_llvm_parse_bitcode_with_llvm_err,
+        };
+        let mut diag = self.0.into_diagnostic(sess);
+        diag.set_primary_message(msg_with_llvm_err);
+        diag.set_arg("llvm_err", self.1);
+        diag
+    }
+}
+
+#[derive(Diagnostic)]
+#[diag(codegen_llvm_from_llvm_optimization_diag)]
+pub(crate) struct FromLlvmOptimizationDiag<'a> {
+    pub filename: &'a str,
+    pub line: std::ffi::c_uint,
+    pub column: std::ffi::c_uint,
+    pub pass_name: &'a str,
+    pub message: &'a str,
+}
+
+#[derive(Diagnostic)]
+#[diag(codegen_llvm_from_llvm_diag)]
+pub(crate) struct FromLlvmDiag {
+    pub message: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(codegen_llvm_write_bytecode)]
+pub(crate) struct WriteBytecode<'a> {
+    pub path: &'a Path,
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(codegen_llvm_copy_bitcode)]
+pub(crate) struct CopyBitcode {
+    pub err: std::io::Error,
+}
diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index a6a75eff9..39afb4af6 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -22,7 +22,6 @@ use rustc_target::abi::{self, Align, HasDataLayout, Primitive};
 use rustc_target::spec::{HasTargetSpec, PanicStrategy};
 
 use std::cmp::Ordering;
-use std::iter;
 
 fn get_simple_intrinsic<'ll>(
     cx: &CodegenCx<'ll, '_>,
@@ -149,7 +148,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                                     emit_va_arg(self, args[0], ret_ty)
                                 }
                             }
-                            Primitive::F64 | Primitive::Pointer => {
+                            Primitive::F64 | Primitive::Pointer(_) => {
                                 emit_va_arg(self, args[0], ret_ty)
                             }
                             // `va_arg` should never be used with the return type f32.
@@ -798,7 +797,7 @@ fn get_rust_try_fn<'ll, 'tcx>(
     let i8p = tcx.mk_mut_ptr(tcx.types.i8);
     // `unsafe fn(*mut i8) -> ()`
     let try_fn_ty = tcx.mk_fn_ptr(ty::Binder::dummy(tcx.mk_fn_sig(
-        iter::once(i8p),
+        [i8p],
         tcx.mk_unit(),
         false,
         hir::Unsafety::Unsafe,
@@ -806,7 +805,7 @@ fn get_rust_try_fn<'ll, 'tcx>(
     )));
     // `unsafe fn(*mut i8, *mut i8) -> ()`
     let catch_fn_ty = tcx.mk_fn_ptr(ty::Binder::dummy(tcx.mk_fn_sig(
-        [i8p, i8p].iter().cloned(),
+        [i8p, i8p],
         tcx.mk_unit(),
         false,
         hir::Unsafety::Unsafe,
@@ -814,7 +813,7 @@ fn get_rust_try_fn<'ll, 'tcx>(
     )));
     // `unsafe fn(unsafe fn(*mut i8) -> (), *mut i8, unsafe fn(*mut i8, *mut i8) -> ()) -> i32`
     let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig(
-        [try_fn_ty, i8p, catch_fn_ty].into_iter(),
+        [try_fn_ty, i8p, catch_fn_ty],
         tcx.types.i32,
         false,
         hir::Unsafety::Unsafe,
@@ -877,7 +876,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
             ty::Array(elem, len)
                 if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
-                    && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all())
+                    && len.try_eval_target_usize(bx.tcx, ty::ParamEnv::reveal_all())
                         == Some(expected_bytes) =>
             {
                 let place = PlaceRef::alloca(bx, args[0].layout);
@@ -957,9 +956,9 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             // version of this intrinsic.
             match args[2].layout.ty.kind() {
                 ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => {
-                    len.try_eval_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(|| {
-                        span_bug!(span, "could not evaluate shuffle index array length")
-                    })
+                    len.try_eval_target_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(
+                        || span_bug!(span, "could not evaluate shuffle index array length"),
+                    )
                 }
                 _ => return_error!(InvalidMonomorphization::SimdShuffle {
                     span,
@@ -1123,7 +1122,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             }
             ty::Array(elem, len)
                 if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
-                    && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all())
+                    && len.try_eval_target_usize(bx.tcx, ty::ParamEnv::reveal_all())
                         == Some(expected_bytes) =>
             {
                 // Zero-extend iN to the array length:
diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs
index 246e82545..c41e74c51 100644
--- a/compiler/rustc_codegen_llvm/src/lib.rs
+++ b/compiler/rustc_codegen_llvm/src/lib.rs
@@ -5,11 +5,12 @@
 //! This API is completely unstable and subject to change.
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
+#![feature(extern_types)]
 #![feature(hash_raw_entry)]
+#![feature(iter_intersperse)]
 #![feature(let_chains)]
-#![feature(extern_types)]
+#![feature(never_type)]
 #![feature(once_cell)]
-#![feature(iter_intersperse)]
 #![recursion_limit = "256"]
 #![allow(rustc::potential_query_instability)]
 #![deny(rustc::untranslatable_diagnostic)]
@@ -22,7 +23,7 @@ extern crate tracing;
 
 use back::write::{create_informational_target_machine, create_target_machine};
 
-use errors::FailParsingTargetMachineConfigToTargetMachine;
+use errors::ParseTargetMachineConfig;
 pub use llvm_util::target_features;
 use rustc_ast::expand::allocator::AllocatorKind;
 use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule};
@@ -33,7 +34,8 @@ use rustc_codegen_ssa::traits::*;
 use rustc_codegen_ssa::ModuleCodegen;
 use rustc_codegen_ssa::{CodegenResults, CompiledModule};
 use rustc_data_structures::fx::FxHashMap;
-use rustc_errors::{ErrorGuaranteed, FatalError, Handler};
+use rustc_errors::{DiagnosticMessage, ErrorGuaranteed, FatalError, Handler, SubdiagnosticMessage};
+use rustc_macros::fluent_messages;
 use rustc_metadata::EncodedMetadata;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::ty::query::Providers;
@@ -82,6 +84,8 @@ mod type_of;
 mod va_arg;
 mod value;
 
+fluent_messages! { "../locales/en-US.ftl" }
+
 #[derive(Clone)]
 pub struct LlvmCodegenBackend(());
 
@@ -169,6 +173,7 @@ impl WriteBackendMethods for LlvmCodegenBackend {
     type Module = ModuleLlvm;
     type ModuleBuffer = back::lto::ModuleBuffer;
     type TargetMachine = &'static mut llvm::TargetMachine;
+    type TargetMachineError = crate::errors::LlvmError<'static>;
     type ThinData = back::lto::ThinData;
     type ThinBuffer = back::lto::ThinBuffer;
     fn print_pass_timings(&self) {
@@ -244,6 +249,10 @@ impl LlvmCodegenBackend {
 }
 
 impl CodegenBackend for LlvmCodegenBackend {
+    fn locale_resource(&self) -> &'static str {
+        crate::DEFAULT_LOCALE_RESOURCE
+    }
+
     fn init(&self, sess: &Session) {
         llvm_util::init(sess); // Make sure llvm is inited
     }
@@ -416,8 +425,7 @@ impl ModuleLlvm {
             let tm = match (cgcx.tm_factory)(tm_factory_config) {
                 Ok(m) => m,
                 Err(e) => {
-                    handler.emit_err(FailParsingTargetMachineConfigToTargetMachine { error: e });
-                    return Err(FatalError);
+                    return Err(handler.emit_almost_fatal(ParseTargetMachineConfig(e)));
                 }
             };
 
diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
index 8b4861962..253c2ca7c 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@@ -482,6 +482,8 @@ pub struct SanitizerOptions {
     pub sanitize_thread: bool,
     pub sanitize_hwaddress: bool,
     pub sanitize_hwaddress_recover: bool,
+    pub sanitize_kernel_address: bool,
+    pub sanitize_kernel_address_recover: bool,
 }
 
 /// LLVMRelocMode
@@ -1812,8 +1814,6 @@ extern "C" {
     /// Creates a legacy pass manager -- only used for final codegen.
     pub fn LLVMCreatePassManager<'a>() -> &'a mut PassManager<'a>;
 
-    pub fn LLVMInitializePasses();
-
     pub fn LLVMTimeTraceProfilerInitialize();
 
     pub fn LLVMTimeTraceProfilerFinishThread();
@@ -2408,6 +2408,8 @@ extern "C" {
     pub fn LLVMRustModuleBufferLen(p: &ModuleBuffer) -> usize;
     pub fn LLVMRustModuleBufferFree(p: &'static mut ModuleBuffer);
     pub fn LLVMRustModuleCost(M: &Module) -> u64;
+    #[allow(improper_ctypes)]
+    pub fn LLVMRustModuleInstructionStats(M: &Module, Str: &RustString);
 
     pub fn LLVMRustThinLTOBufferCreate(M: &Module, is_thin: bool) -> &'static mut ThinLTOBuffer;
     pub fn LLVMRustThinLTOBufferFree(M: &'static mut ThinLTOBuffer);
diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs
index 79b243f73..ba58a2e68 100644
--- a/compiler/rustc_codegen_llvm/src/llvm_util.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs
@@ -120,8 +120,6 @@ unsafe fn configure_llvm(sess: &Session) {
         llvm::LLVMTimeTraceProfilerInitialize();
     }
 
-    llvm::LLVMInitializePasses();
-
     rustc_llvm::initialize_available_targets();
 
     llvm::LLVMRustSetLLVMOptions(llvm_args.len() as c_int, llvm_args.as_ptr());
@@ -152,13 +150,7 @@ pub fn time_trace_profiler_finish(file_name: &Path) {
 pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> SmallVec<[&'a str; 2]> {
     let arch = if sess.target.arch == "x86_64" { "x86" } else { &*sess.target.arch };
     match (arch, s) {
-        ("x86", "sse4.2") => {
-            if get_version() >= (14, 0, 0) {
-                smallvec!["sse4.2", "crc32"]
-            } else {
-                smallvec!["sse4.2"]
-            }
-        }
+        ("x86", "sse4.2") => smallvec!["sse4.2", "crc32"],
         ("x86", "pclmulqdq") => smallvec!["pclmul"],
         ("x86", "rdrand") => smallvec!["rdrnd"],
         ("x86", "bmi1") => smallvec!["bmi"],
@@ -217,7 +209,7 @@ pub fn check_tied_features(
 /// Must express features in the way Rust understands them
 pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> {
     let target_machine = create_informational_target_machine(sess);
-    let mut features: Vec<Symbol> = supported_target_features(sess)
+    supported_target_features(sess)
         .iter()
         .filter_map(|&(feature, gate)| {
             if sess.is_nightly_build() || allow_unstable || gate.is_none() {
@@ -237,16 +229,7 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> {
             true
         })
         .map(|feature| Symbol::intern(feature))
-        .collect();
-
-    // LLVM 14 changed the ABI for i128 arguments to __float/__fix builtins on Win64
-    // (see https://reviews.llvm.org/D110413). This unstable target feature is intended for use
-    // by compiler-builtins, to export the builtins with the expected, LLVM-version-dependent ABI.
-    // The target feature can be dropped once we no longer support older LLVM versions.
-    if sess.is_nightly_build() && get_version() >= (14, 0, 0) {
-        features.push(Symbol::intern("llvm14-builtins-abi"));
-    }
-    features
+        .collect()
 }
 
 pub fn print_version() {
@@ -494,11 +477,6 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str
         .flatten();
     features.extend(feats);
 
-    // FIXME: Move v8a to target definition list when earliest supported LLVM is 14.
-    if get_version() >= (14, 0, 0) && sess.target.arch == "aarch64" {
-        features.push("+v8a".into());
-    }
-
     if diagnostics && let Some(f) = check_tied_features(sess, &featsmap) {
         sess.emit_err(TargetFeatureDisableOrEnable {
             features: f,
diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs
index 76f692b20..d0ae36349 100644
--- a/compiler/rustc_codegen_llvm/src/mono_item.rs
+++ b/compiler/rustc_codegen_llvm/src/mono_item.rs
@@ -9,7 +9,7 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE};
 pub use rustc_middle::mir::mono::MonoItem;
 use rustc_middle::mir::mono::{Linkage, Visibility};
 use rustc_middle::ty::layout::{FnAbiOf, LayoutOf};
-use rustc_middle::ty::{self, Instance, TypeVisitable};
+use rustc_middle::ty::{self, Instance, TypeVisitableExt};
 use rustc_session::config::CrateType;
 use rustc_target::spec::RelocModel;
 
diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs
index 75cd5df97..e264ce78f 100644
--- a/compiler/rustc_codegen_llvm/src/type_of.rs
+++ b/compiler/rustc_codegen_llvm/src/type_of.rs
@@ -1,13 +1,12 @@
 use crate::common::*;
 use crate::context::TypeLowering;
-use crate::llvm_util::get_version;
 use crate::type_::Type;
 use rustc_codegen_ssa::traits::*;
 use rustc_middle::bug;
 use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout};
 use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
-use rustc_middle::ty::{self, Ty, TypeVisitable};
-use rustc_target::abi::{Abi, AddressSpace, Align, FieldsShape};
+use rustc_middle::ty::{self, Ty, TypeVisitableExt};
+use rustc_target::abi::{Abi, Align, FieldsShape};
 use rustc_target::abi::{Int, Pointer, F32, F64};
 use rustc_target::abi::{PointeeInfo, Scalar, Size, TyAbiInterface, Variants};
 use smallvec::{smallvec, SmallVec};
@@ -43,10 +42,8 @@ fn uncached_llvm_type<'a, 'tcx>(
         // in problematically distinct types due to HRTB and subtyping (see #47638).
         // ty::Dynamic(..) |
         ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Generator(..) | ty::Str
-            // For performance reasons we use names only when emitting LLVM IR. Unless we are on
-            // LLVM < 14, where the use of unnamed types resulted in various issues, e.g., #76213,
-            // #79564, and #79246.
-            if get_version() < (14, 0, 0) || !cx.sess().fewer_names() =>
+            // For performance reasons we use names only when emitting LLVM IR.
+            if !cx.sess().fewer_names() =>
         {
             let mut name = with_no_visible_paths!(with_no_trimmed_paths!(layout.ty.to_string()));
             if let (&ty::Adt(def, _), &Variants::Single { index }) =
@@ -157,7 +154,7 @@ fn struct_llfields<'a, 'tcx>(
     } else {
         debug!("struct_llfields: offset: {:?} stride: {:?}", offset, layout.size);
     }
-    let field_remapping = if padding_used { Some(field_remapping) } else { None };
+    let field_remapping = padding_used.then_some(field_remapping);
     (result, packed, field_remapping)
 }
 
@@ -312,14 +309,13 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
             Int(i, _) => cx.type_from_integer(i),
             F32 => cx.type_f32(),
             F64 => cx.type_f64(),
-            Pointer => {
+            Pointer(address_space) => {
                 // If we know the alignment, pick something better than i8.
-                let (pointee, address_space) =
-                    if let Some(pointee) = self.pointee_info_at(cx, offset) {
-                        (cx.type_pointee_for_align(pointee.align), pointee.address_space)
-                    } else {
-                        (cx.type_i8(), AddressSpace::DATA)
-                    };
+                let pointee = if let Some(pointee) = self.pointee_info_at(cx, offset) {
+                    cx.type_pointee_for_align(pointee.align)
+                } else {
+                    cx.type_i8()
+                };
                 cx.type_ptr_to_ext(pointee, address_space)
             }
         }
@@ -333,7 +329,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
     ) -> &'a Type {
         // HACK(eddyb) special-case fat pointers until LLVM removes
         // pointee types, to avoid bitcasting every `OperandRef::deref`.
-        match self.ty.kind() {
+        match *self.ty.kind() {
             ty::Ref(..) | ty::RawPtr(_) => {
                 return self.field(cx, index).llvm_type(cx);
             }
@@ -343,6 +339,11 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
                 let ptr_ty = cx.tcx.mk_mut_ptr(self.ty.boxed_ty());
                 return cx.layout_of(ptr_ty).scalar_pair_element_llvm_type(cx, index, immediate);
             }
+            // `dyn* Trait` has the same ABI as `*mut dyn Trait`
+            ty::Dynamic(bounds, region, ty::DynStar) => {
+                let ptr_ty = cx.tcx.mk_mut_ptr(cx.tcx.mk_dynamic(bounds, region, ty::Dyn));
+                return cx.layout_of(ptr_ty).scalar_pair_element_llvm_type(cx, index, immediate);
+            }
             _ => {}
         }
 
diff --git a/compiler/rustc_codegen_ssa/Cargo.toml b/compiler/rustc_codegen_ssa/Cargo.toml
index 0d2d2ec68..c55991e00 100644
--- a/compiler/rustc_codegen_ssa/Cargo.toml
+++ b/compiler/rustc_codegen_ssa/Cargo.toml
@@ -41,7 +41,6 @@ rustc_metadata = { path = "../rustc_metadata" }
 rustc_query_system = { path = "../rustc_query_system" }
 rustc_target = { path = "../rustc_target" }
 rustc_session = { path = "../rustc_session" }
-rustc_const_eval = { path = "../rustc_const_eval" }
 
 [dependencies.object]
 version = "0.30.1"
diff --git a/compiler/rustc_error_messages/locales/en-US/codegen_ssa.ftl b/compiler/rustc_codegen_ssa/locales/en-US.ftl
index c8c7afb5f..8fe5f8d50 100644
--- a/compiler/rustc_error_messages/locales/en-US/codegen_ssa.ftl
+++ b/compiler/rustc_codegen_ssa/locales/en-US.ftl
@@ -22,7 +22,7 @@ codegen_ssa_ignoring_output = ignoring -o because multiple .{$extension} files w
 
 codegen_ssa_create_temp_dir = couldn't create a temp dir: {$error}
 
-codegen_ssa_incompatible_linking_modifiers = the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs
+codegen_ssa_incompatible_linking_modifiers = link modifiers combination `+bundle,+whole-archive` is unstable when generating rlibs
 
 codegen_ssa_add_native_library = failed to add native library {$library_path}: {$error}
 
@@ -179,9 +179,9 @@ codegen_ssa_extract_bundled_libs_write_file = failed to write file '{$rlib}': {$
 
 codegen_ssa_unsupported_arch = unsupported arch `{$arch}` for os `{$os}`
 
-codegen_ssa_apple_sdk_error_sdk_path = failed to get {$sdk_name} SDK path: {error}
+codegen_ssa_apple_sdk_error_sdk_path = failed to get {$sdk_name} SDK path: {$error}
 
-codegen_ssa_read_file = failed to read file: {message}
+codegen_ssa_read_file = failed to read file: {$message}
 
 codegen_ssa_unsupported_link_self_contained = option `-C link-self-contained` is not supported on this target
 
diff --git a/compiler/rustc_codegen_ssa/src/back/archive.rs b/compiler/rustc_codegen_ssa/src/back/archive.rs
index d3cd085cf..66ec8f5f5 100644
--- a/compiler/rustc_codegen_ssa/src/back/archive.rs
+++ b/compiler/rustc_codegen_ssa/src/back/archive.rs
@@ -203,7 +203,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
             }
         }
 
-        self.src_archives.push((archive_path.to_owned(), archive_map));
+        self.src_archives.push((archive_path, archive_map));
         Ok(())
     }
 
diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs
index 34e042376..8bb143ed3 100644
--- a/compiler/rustc_codegen_ssa/src/back/link.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link.rs
@@ -270,10 +270,9 @@ pub fn each_linked_rlib(
 
 /// Create an 'rlib'.
 ///
-/// An rlib in its current incarnation is essentially a renamed .a file. The rlib primarily contains
-/// the object file of the crate, but it also contains all of the object files from native
-/// libraries. This is done by unzipping native libraries and inserting all of the contents into
-/// this archive.
+/// An rlib in its current incarnation is essentially a renamed .a file (with "dummy" object files).
+/// The rlib primarily contains the object file of the crate, but it also some of the object files
+/// from native libraries.
 fn link_rlib<'a>(
     sess: &'a Session,
     archive_builder_builder: &dyn ArchiveBuilderBuilder,
@@ -347,44 +346,23 @@ fn link_rlib<'a>(
     // loaded from the libraries found here and then encode that into the
     // metadata of the rlib we're generating somehow.
     for lib in codegen_results.crate_info.used_libraries.iter() {
-        match lib.kind {
-            NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) }
-                if flavor == RlibFlavor::Normal && sess.opts.unstable_opts.packed_bundled_libs => {}
-            NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) }
-                if flavor == RlibFlavor::Normal =>
-            {
-                // Don't allow mixing +bundle with +whole_archive since an rlib may contain
-                // multiple native libs, some of which are +whole-archive and some of which are
-                // -whole-archive and it isn't clear how we can currently handle such a
-                // situation correctly.
-                // See https://github.com/rust-lang/rust/issues/88085#issuecomment-901050897
-                sess.emit_err(errors::IncompatibleLinkingModifiers);
-            }
-            NativeLibKind::Static { bundle: None | Some(true), .. } => {}
-            NativeLibKind::Static { bundle: Some(false), .. }
-            | NativeLibKind::Dylib { .. }
-            | NativeLibKind::Framework { .. }
-            | NativeLibKind::RawDylib
-            | NativeLibKind::LinkArg
-            | NativeLibKind::Unspecified => continue,
-        }
-        if let Some(name) = lib.name {
-            let location =
+        let NativeLibKind::Static { bundle: None | Some(true), whole_archive } = lib.kind else {
+            continue;
+        };
+        if whole_archive == Some(true) && !codegen_results.crate_info.feature_packed_bundled_libs {
+            sess.emit_err(errors::IncompatibleLinkingModifiers);
+        }
+        if flavor == RlibFlavor::Normal && let Some(filename) = lib.filename {
+            let path = find_native_static_library(filename.as_str(), true, &lib_search_paths, sess);
+            let src = read(path).map_err(|e| sess.emit_fatal(errors::ReadFileError {message: e }))?;
+            let (data, _) = create_wrapper_file(sess, b".bundled_lib".to_vec(), &src);
+            let wrapper_file = emit_wrapper_file(sess, &data, tmpdir, filename.as_str());
+            packed_bundled_libs.push(wrapper_file);
+        } else if let Some(name) = lib.name {
+            let path =
                 find_native_static_library(name.as_str(), lib.verbatim, &lib_search_paths, sess);
-            if sess.opts.unstable_opts.packed_bundled_libs && flavor == RlibFlavor::Normal {
-                let filename = lib.filename.unwrap();
-                let lib_path =
-                    find_native_static_library(filename.as_str(), true, &lib_search_paths, sess);
-                let src = read(lib_path)
-                    .map_err(|e| sess.emit_fatal(errors::ReadFileError { message: e }))?;
-                let (data, _) = create_wrapper_file(sess, b".bundled_lib".to_vec(), &src);
-                let wrapper_file = emit_wrapper_file(sess, &data, tmpdir, filename.as_str());
-                packed_bundled_libs.push(wrapper_file);
-                continue;
-            }
-            ab.add_archive(&location, Box::new(|_| false)).unwrap_or_else(|error| {
-                sess.emit_fatal(errors::AddNativeLibrary { library_path: location, error });
-            });
+            ab.add_archive(&path, Box::new(|_| false)).unwrap_or_else(|error| {
+                sess.emit_fatal(errors::AddNativeLibrary { library_path: path, error })});
         }
     }
 
@@ -516,36 +494,14 @@ fn link_staticlib<'a>(
         &codegen_results.crate_info,
         Some(CrateType::Staticlib),
         &mut |cnum, path| {
-            let name = codegen_results.crate_info.crate_name[&cnum];
-            let native_libs = &codegen_results.crate_info.native_libraries[&cnum];
-
-            // Here when we include the rlib into our staticlib we need to make a
-            // decision whether to include the extra object files along the way.
-            // These extra object files come from statically included native
-            // libraries, but they may be cfg'd away with #[link(cfg(..))].
-            //
-            // This unstable feature, though, only needs liblibc to work. The only
-            // use case there is where musl is statically included in liblibc.rlib,
-            // so if we don't want the included version we just need to skip it. As
-            // a result the logic here is that if *any* linked library is cfg'd away
-            // we just skip all object files.
-            //
-            // Clearly this is not sufficient for a general purpose feature, and
-            // we'd want to read from the library's metadata to determine which
-            // object files come from where and selectively skip them.
-            let skip_object_files = native_libs.iter().any(|lib| {
-                matches!(lib.kind, NativeLibKind::Static { bundle: None | Some(true), .. })
-                    && !relevant_lib(sess, lib)
-            });
-
             let lto = are_upstream_rust_objects_already_included(sess)
                 && !ignored_for_lto(sess, &codegen_results.crate_info, cnum);
 
-            // Ignoring obj file starting with the crate name
-            // as simple comparison is not enough - there
-            // might be also an extra name suffix
-            let obj_start = name.as_str().to_owned();
+            let native_libs = codegen_results.crate_info.native_libraries[&cnum].iter();
+            let relevant = native_libs.clone().filter(|lib| relevant_lib(sess, &lib));
+            let relevant_libs: FxHashSet<_> = relevant.filter_map(|lib| lib.filename).collect();
 
+            let bundled_libs: FxHashSet<_> = native_libs.filter_map(|lib| lib.filename).collect();
             ab.add_archive(
                 path,
                 Box::new(move |fname: &str| {
@@ -559,20 +515,25 @@ fn link_staticlib<'a>(
                         return true;
                     }
 
-                    // Otherwise if this is *not* a rust object and we're skipping
-                    // objects then skip this file
-                    if skip_object_files
-                        && (!fname.starts_with(&obj_start) || !fname.ends_with(".o"))
-                    {
+                    // Skip objects for bundled libs.
+                    if bundled_libs.contains(&Symbol::intern(fname)) {
                         return true;
                     }
 
-                    // ok, don't skip this
                     false
                 }),
             )
             .unwrap();
 
+            archive_builder_builder
+                .extract_bundled_libs(path, tempdir.as_ref(), &relevant_libs)
+                .unwrap_or_else(|e| sess.emit_fatal(e));
+            for filename in relevant_libs {
+                let joined = tempdir.as_ref().join(filename.as_str());
+                let path = joined.as_path();
+                ab.add_archive(path, Box::new(|_| false)).unwrap();
+            }
+
             all_native_libs
                 .extend(codegen_results.crate_info.native_libraries[&cnum].iter().cloned());
         },
@@ -599,7 +560,8 @@ fn link_dwarf_object<'a>(
     cg_results: &CodegenResults,
     executable_out_filename: &Path,
 ) {
-    let dwp_out_filename = executable_out_filename.with_extension("dwp");
+    let mut dwp_out_filename = executable_out_filename.to_path_buf().into_os_string();
+    dwp_out_filename.push(".dwp");
     debug!(?dwp_out_filename, ?executable_out_filename);
 
     #[derive(Default)]
@@ -1302,12 +1264,6 @@ fn preserve_objects_for_their_debuginfo(sess: &Session) -> (bool, bool) {
         return (false, false);
     }
 
-    // If we're only producing artifacts that are archives, no need to preserve
-    // the objects as they're losslessly contained inside the archives.
-    if sess.crate_types().iter().all(|&x| x.is_archive()) {
-        return (false, false);
-    }
-
     match (sess.split_debuginfo(), sess.opts.unstable_opts.split_dwarf_kind) {
         // If there is no split debuginfo then do not preserve objects.
         (SplitDebuginfo::Off, _) => (false, false),
@@ -2070,7 +2026,7 @@ fn linker_with_args<'a>(
         .native_libraries
         .iter()
         .filter_map(|(cnum, libraries)| {
-            (dependency_linkage[cnum.as_usize() - 1] != Linkage::Static).then(|| libraries)
+            (dependency_linkage[cnum.as_usize() - 1] != Linkage::Static).then_some(libraries)
         })
         .flatten();
     for (raw_dylib_name, raw_dylib_imports) in
@@ -2597,18 +2553,8 @@ fn add_static_crate<'a>(
         cmd.link_rlib(&fix_windows_verbatim_for_gcc(path));
     };
 
-    // See the comment above in `link_staticlib` and `link_rlib` for why if
-    // there's a static library that's not relevant we skip all object
-    // files.
-    let native_libs = &codegen_results.crate_info.native_libraries[&cnum];
-    let skip_native = native_libs.iter().any(|lib| {
-        matches!(lib.kind, NativeLibKind::Static { bundle: None | Some(true), .. })
-            && !relevant_lib(sess, lib)
-    });
-
-    if (!are_upstream_rust_objects_already_included(sess)
-        || ignored_for_lto(sess, &codegen_results.crate_info, cnum))
-        && !skip_native
+    if !are_upstream_rust_objects_already_included(sess)
+        || ignored_for_lto(sess, &codegen_results.crate_info, cnum)
     {
         link_upstream(cratepath);
         return;
@@ -2639,17 +2585,13 @@ fn add_static_crate<'a>(
                 let is_rust_object =
                     canonical.starts_with(&canonical_name) && looks_like_rust_object_file(&f);
 
-                // If we've been requested to skip all native object files
-                // (those not generated by the rust compiler) then we can skip
-                // this file. See above for why we may want to do this.
-                let skip_because_cfg_say_so = skip_native && !is_rust_object;
-
                 // If we're performing LTO and this is a rust-generated object
                 // file, then we don't need the object file as it's part of the
                 // LTO module. Note that `#![no_builtins]` is excluded from LTO,
                 // though, so we let that object file slide.
-                let skip_because_lto =
-                    upstream_rust_objects_already_included && is_rust_object && is_builtins;
+                if upstream_rust_objects_already_included && is_rust_object && is_builtins {
+                    return true;
+                }
 
                 // We skip native libraries because:
                 // 1. This native libraries won't be used from the generated rlib,
@@ -2660,10 +2602,6 @@ fn add_static_crate<'a>(
                     return true;
                 }
 
-                if skip_because_cfg_say_so || skip_because_lto {
-                    return true;
-                }
-
                 false
             }),
         ) {
diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs
index eaf1e9817..52c01b423 100644
--- a/compiler/rustc_codegen_ssa/src/back/linker.rs
+++ b/compiler/rustc_codegen_ssa/src/back/linker.rs
@@ -473,13 +473,13 @@ impl<'a> Linker for GccLinker<'a> {
         self.cmd.arg(path);
     }
     fn full_relro(&mut self) {
-        self.linker_args(&["-zrelro", "-znow"]);
+        self.linker_args(&["-z", "relro", "-z", "now"]);
     }
     fn partial_relro(&mut self) {
-        self.linker_arg("-zrelro");
+        self.linker_args(&["-z", "relro"]);
     }
     fn no_relro(&mut self) {
-        self.linker_arg("-znorelro");
+        self.linker_args(&["-z", "norelro"]);
     }
 
     fn link_rust_dylib(&mut self, lib: &str, _path: &Path) {
@@ -758,7 +758,7 @@ impl<'a> Linker for GccLinker<'a> {
         if self.sess.target.is_like_windows {
             self.linker_arg("--nxcompat");
         } else if self.is_gnu {
-            self.linker_arg("-znoexecstack");
+            self.linker_args(&["-z", "noexecstack"]);
         }
     }
 
@@ -1364,16 +1364,16 @@ impl<'a> Linker for L4Bender<'a> {
     }
 
     fn full_relro(&mut self) {
-        self.cmd.arg("-zrelro");
-        self.cmd.arg("-znow");
+        self.cmd.arg("-z").arg("relro");
+        self.cmd.arg("-z").arg("now");
     }
 
     fn partial_relro(&mut self) {
-        self.cmd.arg("-zrelro");
+        self.cmd.arg("-z").arg("relro");
     }
 
     fn no_relro(&mut self) {
-        self.cmd.arg("-znorelro");
+        self.cmd.arg("-z").arg("norelro");
     }
 
     fn cmd(&mut self) -> &mut Command {
diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs
index 7d3c14fec..019ec0758 100644
--- a/compiler/rustc_codegen_ssa/src/back/metadata.rs
+++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs
@@ -33,6 +33,7 @@ use rustc_target::spec::{RelocModel, Target};
 /// <dt>dylib</dt>
 /// <dd>The metadata can be found in the `.rustc` section of the shared library.</dd>
 /// </dl>
+#[derive(Debug)]
 pub struct DefaultMetadataLoader;
 
 fn load_metadata_with(
diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
index 57a99e74c..067a3e167 100644
--- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
+++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
@@ -2,9 +2,8 @@ use std::collections::hash_map::Entry::*;
 
 use rustc_ast::expand::allocator::ALLOCATOR_METHODS;
 use rustc_data_structures::fx::FxHashMap;
-use rustc_hir as hir;
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LOCAL_CRATE};
-use rustc_hir::Node;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::middle::exported_symbols::{
     metadata_symbol_name, ExportedSymbol, SymbolExportInfo, SymbolExportKind, SymbolExportLevel,
@@ -12,7 +11,7 @@ use rustc_middle::middle::exported_symbols::{
 use rustc_middle::ty::query::{ExternProviders, Providers};
 use rustc_middle::ty::subst::{GenericArgKind, SubstsRef};
 use rustc_middle::ty::Instance;
-use rustc_middle::ty::{self, SymbolName, TyCtxt};
+use rustc_middle::ty::{self, DefIdTree, SymbolName, TyCtxt};
 use rustc_session::config::{CrateType, OomStrategy};
 use rustc_target::spec::SanitizerSet;
 
@@ -74,32 +73,34 @@ fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap<
             //
             // As a result, if this id is an FFI item (foreign item) then we only
             // let it through if it's included statically.
-            match tcx.hir().get_by_def_id(def_id) {
-                Node::ForeignItem(..) => {
-                    tcx.native_library(def_id).map_or(false, |library| library.kind.is_statically_included()).then_some(def_id)
-                }
+            if let Some(parent_id) = tcx.opt_local_parent(def_id)
+                && let DefKind::ForeignMod = tcx.def_kind(parent_id)
+            {
+                let library = tcx.native_library(def_id)?;
+                return library.kind.is_statically_included().then_some(def_id);
+            }
 
-                // Only consider nodes that actually have exported symbols.
-                Node::Item(&hir::Item {
-                    kind: hir::ItemKind::Static(..) | hir::ItemKind::Fn(..),
-                    ..
-                })
-                | Node::ImplItem(&hir::ImplItem { kind: hir::ImplItemKind::Fn(..), .. }) => {
-                    let generics = tcx.generics_of(def_id);
-                    if !generics.requires_monomorphization(tcx)
-                        // Functions marked with #[inline] are codegened with "internal"
-                        // linkage and are not exported unless marked with an extern
-                        // indicator
-                        && (!Instance::mono(tcx, def_id.to_def_id()).def.generates_cgu_internal_copy(tcx)
-                            || tcx.codegen_fn_attrs(def_id.to_def_id()).contains_extern_indicator())
-                    {
-                        Some(def_id)
-                    } else {
-                        None
-                    }
-                }
+            // Only consider nodes that actually have exported symbols.
+            match tcx.def_kind(def_id) {
+                DefKind::Fn | DefKind::Static(_) => {}
+                DefKind::AssocFn if tcx.impl_of_method(def_id.to_def_id()).is_some() => {}
+                _ => return None,
+            };
 
-                _ => None,
+            let generics = tcx.generics_of(def_id);
+            if generics.requires_monomorphization(tcx) {
+                return None;
+            }
+
+            // Functions marked with #[inline] are codegened with "internal"
+            // linkage and are not exported unless marked with an extern
+            // indicator
+            if !Instance::mono(tcx, def_id.to_def_id()).def.generates_cgu_internal_copy(tcx)
+                || tcx.codegen_fn_attrs(def_id.to_def_id()).contains_extern_indicator()
+            {
+                Some(def_id)
+            } else {
+                None
             }
         })
         .map(|def_id| {
@@ -118,7 +119,7 @@ fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap<
                 tcx.symbol_name(Instance::mono(tcx, def_id.to_def_id())),
                 export_level
             );
-            (def_id.to_def_id(), SymbolExportInfo {
+            let info = SymbolExportInfo {
                 level: export_level,
                 kind: if tcx.is_static(def_id.to_def_id()) {
                     if codegen_attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) {
@@ -130,8 +131,10 @@ fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap<
                     SymbolExportKind::Text
                 },
                 used: codegen_attrs.flags.contains(CodegenFnAttrFlags::USED)
-                    || codegen_attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) || used,
-            })
+                    || codegen_attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER)
+                    || used,
+            };
+            (def_id.to_def_id(), info)
         })
         .collect();
 
@@ -370,7 +373,7 @@ fn upstream_monomorphizations_provider(
                 ExportedSymbol::Generic(def_id, substs) => (def_id, substs),
                 ExportedSymbol::DropGlue(ty) => {
                     if let Some(drop_in_place_fn_def_id) = drop_in_place_fn_def_id {
-                        (drop_in_place_fn_def_id, tcx.intern_substs(&[ty.into()]))
+                        (drop_in_place_fn_def_id, tcx.mk_substs(&[ty.into()]))
                     } else {
                         // `drop_in_place` in place does not exist, don't try
                         // to use it.
@@ -457,9 +460,7 @@ fn symbol_export_level(tcx: TyCtxt<'_>, sym_def_id: DefId) -> SymbolExportLevel
         let target = &tcx.sess.target.llvm_target;
         // WebAssembly cannot export data symbols, so reduce their export level
         if target.contains("emscripten") {
-            if let Some(Node::Item(&hir::Item { kind: hir::ItemKind::Static(..), .. })) =
-                tcx.hir().get_if_local(sym_def_id)
-            {
+            if let DefKind::Static(_) = tcx.def_kind(sym_def_id) {
                 return SymbolExportLevel::Rust;
             }
         }
diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs
index 9f1614af7..8508ab875 100644
--- a/compiler/rustc_codegen_ssa/src/back/write.rs
+++ b/compiler/rustc_codegen_ssa/src/back/write.rs
@@ -305,8 +305,12 @@ impl TargetMachineFactoryConfig {
 }
 
 pub type TargetMachineFactoryFn<B> = Arc<
-    dyn Fn(TargetMachineFactoryConfig) -> Result<<B as WriteBackendMethods>::TargetMachine, String>
-        + Send
+    dyn Fn(
+            TargetMachineFactoryConfig,
+        ) -> Result<
+            <B as WriteBackendMethods>::TargetMachine,
+            <B as WriteBackendMethods>::TargetMachineError,
+        > + Send
         + Sync,
 >;
 
diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs
index 32d3cfe6f..73179249b 100644
--- a/compiler/rustc_codegen_ssa/src/base.rs
+++ b/compiler/rustc_codegen_ssa/src/base.rs
@@ -39,7 +39,7 @@ use rustc_session::Session;
 use rustc_span::symbol::sym;
 use rustc_span::Symbol;
 use rustc_span::{DebuggerVisualizerFile, DebuggerVisualizerType};
-use rustc_target::abi::{Align, Size, VariantIdx};
+use rustc_target::abi::{Align, VariantIdx};
 
 use std::collections::BTreeSet;
 use std::time::{Duration, Instant};
@@ -148,7 +148,7 @@ pub fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         cx.tcx().struct_lockstep_tails_erasing_lifetimes(source, target, bx.param_env());
     match (source.kind(), target.kind()) {
         (&ty::Array(_, len), &ty::Slice(_)) => {
-            cx.const_usize(len.eval_usize(cx.tcx(), ty::ParamEnv::reveal_all()))
+            cx.const_usize(len.eval_target_usize(cx.tcx(), ty::ParamEnv::reveal_all()))
         }
         (
             &ty::Dynamic(ref data_a, _, src_dyn_kind),
@@ -273,12 +273,13 @@ pub fn cast_to_dyn_star<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         matches!(dst_ty.kind(), ty::Dynamic(_, _, ty::DynStar)),
         "destination type must be a dyn*"
     );
-    // FIXME(dyn-star): this is probably not the best way to check if this is
-    // a pointer, and really we should ensure that the value is a suitable
-    // pointer earlier in the compilation process.
-    let src = match src_ty_and_layout.pointee_info_at(bx.cx(), Size::ZERO) {
-        Some(_) => bx.ptrtoint(src, bx.cx().type_isize()),
-        None => bx.bitcast(src, bx.type_isize()),
+    // FIXME(dyn-star): We can remove this when all supported LLVMs use opaque ptrs only.
+    let unit_ptr = bx.cx().type_ptr_to(bx.cx().type_struct(&[], false));
+    let src = match bx.cx().type_kind(bx.cx().backend_type(src_ty_and_layout)) {
+        TypeKind::Pointer => bx.pointercast(src, unit_ptr),
+        TypeKind::Integer => bx.inttoptr(src, unit_ptr),
+        // FIXME(dyn-star): We probably have to do a bitcast first, then inttoptr.
+        kind => bug!("unexpected TypeKind for left-hand side of `dyn*` cast: {kind:?}"),
     };
     (src, unsized_info(bx, src_ty_and_layout.ty, dst_ty, old_info))
 }
@@ -436,7 +437,7 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
             cx.type_func(&[], cx.type_int())
         };
 
-        let main_ret_ty = cx.tcx().fn_sig(rust_main_def_id).output();
+        let main_ret_ty = cx.tcx().fn_sig(rust_main_def_id).no_bound_vars().unwrap().output();
         // Given that `main()` has no arguments,
         // then its return type cannot have
         // late-bound regions, since late-bound
@@ -475,7 +476,7 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                     cx.tcx(),
                     ty::ParamEnv::reveal_all(),
                     start_def_id,
-                    cx.tcx().intern_substs(&[main_ret_ty.into()]),
+                    cx.tcx().mk_substs(&[main_ret_ty.into()]),
                 )
                 .unwrap()
                 .unwrap(),
@@ -579,7 +580,7 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
         }
     }
 
-    let metadata_module = if need_metadata_module {
+    let metadata_module = need_metadata_module.then(|| {
         // Emit compressed metadata object.
         let metadata_cgu_name =
             cgu_name_builder.build_cgu_name(LOCAL_CRATE, &["crate"], Some("metadata")).to_string();
@@ -594,17 +595,15 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
             if let Err(error) = std::fs::write(&file_name, data) {
                 tcx.sess.emit_fatal(errors::MetadataObjectFileWrite { error });
             }
-            Some(CompiledModule {
+            CompiledModule {
                 name: metadata_cgu_name,
                 kind: ModuleKind::Metadata,
                 object: Some(file_name),
                 dwarf_object: None,
                 bytecode: None,
-            })
+            }
         })
-    } else {
-        None
-    };
+    });
 
     let ongoing_codegen = start_async_codegen(
         backend.clone(),
@@ -858,6 +857,7 @@ impl CrateInfo {
             dependency_formats: tcx.dependency_formats(()).clone(),
             windows_subsystem,
             natvis_debugger_visualizers: Default::default(),
+            feature_packed_bundled_libs: tcx.features().packed_bundled_libs,
         };
         let crates = tcx.crates(());
 
diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
index 8808ad2dc..7d5c00486 100644
--- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
+++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@@ -85,55 +85,11 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
         } else if attr.has_name(sym::rustc_allocator) {
             codegen_fn_attrs.flags |= CodegenFnAttrFlags::ALLOCATOR;
         } else if attr.has_name(sym::ffi_returns_twice) {
-            if tcx.is_foreign_item(did) {
-                codegen_fn_attrs.flags |= CodegenFnAttrFlags::FFI_RETURNS_TWICE;
-            } else {
-                // `#[ffi_returns_twice]` is only allowed `extern fn`s.
-                struct_span_err!(
-                    tcx.sess,
-                    attr.span,
-                    E0724,
-                    "`#[ffi_returns_twice]` may only be used on foreign functions"
-                )
-                .emit();
-            }
+            codegen_fn_attrs.flags |= CodegenFnAttrFlags::FFI_RETURNS_TWICE;
         } else if attr.has_name(sym::ffi_pure) {
-            if tcx.is_foreign_item(did) {
-                if attrs.iter().any(|a| a.has_name(sym::ffi_const)) {
-                    // `#[ffi_const]` functions cannot be `#[ffi_pure]`
-                    struct_span_err!(
-                        tcx.sess,
-                        attr.span,
-                        E0757,
-                        "`#[ffi_const]` function cannot be `#[ffi_pure]`"
-                    )
-                    .emit();
-                } else {
-                    codegen_fn_attrs.flags |= CodegenFnAttrFlags::FFI_PURE;
-                }
-            } else {
-                // `#[ffi_pure]` is only allowed on foreign functions
-                struct_span_err!(
-                    tcx.sess,
-                    attr.span,
-                    E0755,
-                    "`#[ffi_pure]` may only be used on foreign functions"
-                )
-                .emit();
-            }
+            codegen_fn_attrs.flags |= CodegenFnAttrFlags::FFI_PURE;
         } else if attr.has_name(sym::ffi_const) {
-            if tcx.is_foreign_item(did) {
-                codegen_fn_attrs.flags |= CodegenFnAttrFlags::FFI_CONST;
-            } else {
-                // `#[ffi_const]` is only allowed on foreign functions
-                struct_span_err!(
-                    tcx.sess,
-                    attr.span,
-                    E0756,
-                    "`#[ffi_const]` may only be used on foreign functions"
-                )
-                .emit();
-            }
+            codegen_fn_attrs.flags |= CodegenFnAttrFlags::FFI_CONST;
         } else if attr.has_name(sym::rustc_nounwind) {
             codegen_fn_attrs.flags |= CodegenFnAttrFlags::NEVER_UNWIND;
         } else if attr.has_name(sym::rustc_reallocator) {
@@ -214,7 +170,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
             }
         } else if attr.has_name(sym::cmse_nonsecure_entry) {
             if validate_fn_only_attr(attr.span)
-                && !matches!(tcx.fn_sig(did).abi(), abi::Abi::C { .. })
+                && !matches!(tcx.fn_sig(did).skip_binder().abi(), abi::Abi::C { .. })
             {
                 struct_span_err!(
                     tcx.sess,
@@ -234,7 +190,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
         } else if attr.has_name(sym::track_caller) {
             if !tcx.is_closure(did.to_def_id())
                 && validate_fn_only_attr(attr.span)
-                && tcx.fn_sig(did).abi() != abi::Abi::Rust
+                && tcx.fn_sig(did).skip_binder().abi() != abi::Abi::Rust
             {
                 struct_span_err!(tcx.sess, attr.span, E0737, "`#[track_caller]` requires Rust ABI")
                     .emit();
@@ -266,7 +222,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
             }
         } else if attr.has_name(sym::target_feature) {
             if !tcx.is_closure(did.to_def_id())
-                && tcx.fn_sig(did).unsafety() == hir::Unsafety::Normal
+                && tcx.fn_sig(did).skip_binder().unsafety() == hir::Unsafety::Normal
             {
                 if tcx.sess.target.is_like_wasm || tcx.sess.opts.actually_rustdoc {
                     // The `#[target_feature]` attribute is allowed on
@@ -339,7 +295,8 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
             if let Some(list) = attr.meta_item_list() {
                 for item in list.iter() {
                     if item.has_name(sym::address) {
-                        codegen_fn_attrs.no_sanitize |= SanitizerSet::ADDRESS;
+                        codegen_fn_attrs.no_sanitize |=
+                            SanitizerSet::ADDRESS | SanitizerSet::KERNELADDRESS;
                     } else if item.has_name(sym::cfi) {
                         codegen_fn_attrs.no_sanitize |= SanitizerSet::CFI;
                     } else if item.has_name(sym::kcfi) {
@@ -363,74 +320,62 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: DefId) -> CodegenFnAttrs {
                 }
             }
         } else if attr.has_name(sym::instruction_set) {
-            codegen_fn_attrs.instruction_set = match attr.meta_kind() {
-                Some(MetaItemKind::List(ref items)) => match items.as_slice() {
-                    [NestedMetaItem::MetaItem(set)] => {
-                        let segments =
-                            set.path.segments.iter().map(|x| x.ident.name).collect::<Vec<_>>();
-                        match segments.as_slice() {
-                            [sym::arm, sym::a32] | [sym::arm, sym::t32] => {
-                                if !tcx.sess.target.has_thumb_interworking {
-                                    struct_span_err!(
-                                        tcx.sess.diagnostic(),
-                                        attr.span,
-                                        E0779,
-                                        "target does not support `#[instruction_set]`"
-                                    )
-                                    .emit();
-                                    None
-                                } else if segments[1] == sym::a32 {
-                                    Some(InstructionSetAttr::ArmA32)
-                                } else if segments[1] == sym::t32 {
-                                    Some(InstructionSetAttr::ArmT32)
-                                } else {
-                                    unreachable!()
-                                }
-                            }
-                            _ => {
+            codegen_fn_attrs.instruction_set = attr.meta_item_list().and_then(|l| match &l[..] {
+                [NestedMetaItem::MetaItem(set)] => {
+                    let segments =
+                        set.path.segments.iter().map(|x| x.ident.name).collect::<Vec<_>>();
+                    match segments.as_slice() {
+                        [sym::arm, sym::a32] | [sym::arm, sym::t32] => {
+                            if !tcx.sess.target.has_thumb_interworking {
                                 struct_span_err!(
                                     tcx.sess.diagnostic(),
                                     attr.span,
                                     E0779,
-                                    "invalid instruction set specified",
+                                    "target does not support `#[instruction_set]`"
                                 )
                                 .emit();
                                 None
+                            } else if segments[1] == sym::a32 {
+                                Some(InstructionSetAttr::ArmA32)
+                            } else if segments[1] == sym::t32 {
+                                Some(InstructionSetAttr::ArmT32)
+                            } else {
+                                unreachable!()
                             }
                         }
+                        _ => {
+                            struct_span_err!(
+                                tcx.sess.diagnostic(),
+                                attr.span,
+                                E0779,
+                                "invalid instruction set specified",
+                            )
+                            .emit();
+                            None
+                        }
                     }
-                    [] => {
-                        struct_span_err!(
-                            tcx.sess.diagnostic(),
-                            attr.span,
-                            E0778,
-                            "`#[instruction_set]` requires an argument"
-                        )
-                        .emit();
-                        None
-                    }
-                    _ => {
-                        struct_span_err!(
-                            tcx.sess.diagnostic(),
-                            attr.span,
-                            E0779,
-                            "cannot specify more than one instruction set"
-                        )
-                        .emit();
-                        None
-                    }
-                },
-                _ => {
+                }
+                [] => {
                     struct_span_err!(
                         tcx.sess.diagnostic(),
                         attr.span,
                         E0778,
-                        "must specify an instruction set"
+                        "`#[instruction_set]` requires an argument"
                     )
                     .emit();
                     None
                 }
-            };
+                _ => {
+                    struct_span_err!(
+                        tcx.sess.diagnostic(),
+                        attr.span,
+                        E0779,
+                        "cannot specify more than one instruction set"
+                    )
+                    .emit();
+                    None
+                }
+            })
         } else if attr.has_name(sym::repr) {
             codegen_fn_attrs.alignment = match attr.meta_item_list() {
                 Some(items) => match items.as_slice() {
diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
index 1599ccbb2..f2469fde3 100644
--- a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
+++ b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
@@ -181,16 +181,24 @@ fn push_debuginfo_type_name<'tcx>(
                 push_debuginfo_type_name(tcx, inner_type, true, output, visited);
                 match len.kind() {
                     ty::ConstKind::Param(param) => write!(output, ",{}>", param.name).unwrap(),
-                    _ => write!(output, ",{}>", len.eval_usize(tcx, ty::ParamEnv::reveal_all()))
-                        .unwrap(),
+                    _ => write!(
+                        output,
+                        ",{}>",
+                        len.eval_target_usize(tcx, ty::ParamEnv::reveal_all())
+                    )
+                    .unwrap(),
                 }
             } else {
                 output.push('[');
                 push_debuginfo_type_name(tcx, inner_type, true, output, visited);
                 match len.kind() {
                     ty::ConstKind::Param(param) => write!(output, "; {}]", param.name).unwrap(),
-                    _ => write!(output, "; {}]", len.eval_usize(tcx, ty::ParamEnv::reveal_all()))
-                        .unwrap(),
+                    _ => write!(
+                        output,
+                        "; {}]",
+                        len.eval_target_usize(tcx, ty::ParamEnv::reveal_all())
+                    )
+                    .unwrap(),
                 }
             }
         }
@@ -414,6 +422,7 @@ fn push_debuginfo_type_name<'tcx>(
         | ty::Placeholder(..)
         | ty::Alias(..)
         | ty::Bound(..)
+        | ty::GeneratorWitnessMIR(..)
         | ty::GeneratorWitness(..) => {
             bug!(
                 "debuginfo: Trying to create type name for \
diff --git a/compiler/rustc_codegen_ssa/src/errors.rs b/compiler/rustc_codegen_ssa/src/errors.rs
index d81252653..6dea7496f 100644
--- a/compiler/rustc_codegen_ssa/src/errors.rs
+++ b/compiler/rustc_codegen_ssa/src/errors.rs
@@ -1,8 +1,9 @@
 //! Errors emitted by codegen_ssa
 
 use crate::back::command::Command;
+use crate::fluent_generated as fluent;
 use rustc_errors::{
-    fluent, DiagnosticArgValue, DiagnosticBuilder, ErrorGuaranteed, Handler, IntoDiagnostic,
+    DiagnosticArgValue, DiagnosticBuilder, ErrorGuaranteed, Handler, IntoDiagnostic,
     IntoDiagnosticArg,
 };
 use rustc_macros::Diagnostic;
@@ -388,7 +389,7 @@ pub struct LinkerNotFound {
 #[derive(Diagnostic)]
 #[diag(codegen_ssa_unable_to_exe_linker)]
 #[note]
-#[note(command_note)]
+#[note(codegen_ssa_command_note)]
 pub struct UnableToExeLinker {
     pub linker_path: PathBuf,
     pub error: Error,
diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs
index 0e6596d4b..ebe9e50ff 100644
--- a/compiler/rustc_codegen_ssa/src/lib.rs
+++ b/compiler/rustc_codegen_ssa/src/lib.rs
@@ -13,7 +13,7 @@
 
 //! This crate contains codegen code that is used by all codegen backends (LLVM and others).
 //! The backend-agnostic functions of this crate use functions defined in various traits that
-//! have to be implemented by each backends.
+//! have to be implemented by each backend.
 
 #[macro_use]
 extern crate rustc_macros;
@@ -25,7 +25,9 @@ extern crate rustc_middle;
 use rustc_ast as ast;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::sync::Lrc;
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
 use rustc_hir::def_id::CrateNum;
+use rustc_macros::fluent_messages;
 use rustc_middle::dep_graph::WorkProduct;
 use rustc_middle::middle::dependency_format::Dependencies;
 use rustc_middle::middle::exported_symbols::SymbolExportKind;
@@ -54,6 +56,8 @@ pub mod mono_item;
 pub mod target_features;
 pub mod traits;
 
+fluent_messages! { "../locales/en-US.ftl" }
+
 pub struct ModuleCodegen<M> {
     /// The name of the module. When the crate may be saved between
     /// compilations, incremental compilation requires that name be
@@ -159,6 +163,7 @@ pub struct CrateInfo {
     pub dependency_formats: Lrc<Dependencies>,
     pub windows_subsystem: Option<String>,
     pub natvis_debugger_visualizers: BTreeSet<DebuggerVisualizerFile>,
+    pub feature_packed_bundled_libs: bool, // unstable feature flag.
 }
 
 #[derive(Encodable, Decodable)]
diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
index dd1ac2c74..95aad10fd 100644
--- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
@@ -36,7 +36,7 @@ pub fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 
     // Arguments get assigned to by means of the function being called
     for arg in mir.args_iter() {
-        analyzer.assign(arg, mir::START_BLOCK.start_location());
+        analyzer.assign(arg, DefLocation::Argument);
     }
 
     // If there exists a local definition that dominates all uses of that local,
@@ -64,7 +64,22 @@ enum LocalKind {
     /// A scalar or a scalar pair local that is neither defined nor used.
     Unused,
     /// A scalar or a scalar pair local with a single definition that dominates all uses.
-    SSA(mir::Location),
+    SSA(DefLocation),
+}
+
+#[derive(Copy, Clone, PartialEq, Eq)]
+enum DefLocation {
+    Argument,
+    Body(Location),
+}
+
+impl DefLocation {
+    fn dominates(self, location: Location, dominators: &Dominators<mir::BasicBlock>) -> bool {
+        match self {
+            DefLocation::Argument => true,
+            DefLocation::Body(def) => def.successor_within_block().dominates(location, dominators),
+        }
+    }
 }
 
 struct LocalAnalyzer<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
@@ -74,17 +89,13 @@ struct LocalAnalyzer<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
 }
 
 impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> LocalAnalyzer<'mir, 'a, 'tcx, Bx> {
-    fn assign(&mut self, local: mir::Local, location: Location) {
+    fn assign(&mut self, local: mir::Local, location: DefLocation) {
         let kind = &mut self.locals[local];
         match *kind {
             LocalKind::ZST => {}
             LocalKind::Memory => {}
-            LocalKind::Unused => {
-                *kind = LocalKind::SSA(location);
-            }
-            LocalKind::SSA(_) => {
-                *kind = LocalKind::Memory;
-            }
+            LocalKind::Unused => *kind = LocalKind::SSA(location),
+            LocalKind::SSA(_) => *kind = LocalKind::Memory,
         }
     }
 
@@ -166,7 +177,7 @@ impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx>
         debug!("visit_assign(place={:?}, rvalue={:?})", place, rvalue);
 
         if let Some(local) = place.as_local() {
-            self.assign(local, location);
+            self.assign(local, DefLocation::Body(location));
             if self.locals[local] != LocalKind::Memory {
                 let decl_span = self.fx.mir.local_decls[local].source_info.span;
                 if !self.fx.rvalue_creates_operand(rvalue, decl_span) {
@@ -189,7 +200,7 @@ impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx>
         match context {
             PlaceContext::MutatingUse(MutatingUseContext::Call)
             | PlaceContext::MutatingUse(MutatingUseContext::Yield) => {
-                self.assign(local, location);
+                self.assign(local, DefLocation::Body(location));
             }
 
             PlaceContext::NonUse(_) | PlaceContext::MutatingUse(MutatingUseContext::Retag) => {}
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index 978aff511..57a19a4ab 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -14,9 +14,9 @@ use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_hir::lang_items::LangItem;
 use rustc_index::vec::Idx;
 use rustc_middle::mir::{self, AssertKind, SwitchTargets};
-use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
+use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, ValidityRequirement};
 use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
-use rustc_middle::ty::{self, Instance, Ty, TypeVisitable};
+use rustc_middle::ty::{self, Instance, Ty, TypeVisitableExt};
 use rustc_session::config::OptLevel;
 use rustc_span::source_map::Span;
 use rustc_span::{sym, Symbol};
@@ -39,7 +39,6 @@ enum MergingSucc {
 struct TerminatorCodegenHelper<'tcx> {
     bb: mir::BasicBlock,
     terminator: &'tcx mir::Terminator<'tcx>,
-    funclet_bb: Option<mir::BasicBlock>,
 }
 
 impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
@@ -49,28 +48,24 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
         &self,
         fx: &'b mut FunctionCx<'a, 'tcx, Bx>,
     ) -> Option<&'b Bx::Funclet> {
-        let funclet_bb = self.funclet_bb?;
-        if base::wants_msvc_seh(fx.cx.tcx().sess) {
-            // If `landing_pad_for` hasn't been called yet to create the `Funclet`,
-            // it has to be now. This may not seem necessary, as RPO should lead
-            // to all the unwind edges being visited (and so to `landing_pad_for`
-            // getting called for them), before building any of the blocks inside
-            // the funclet itself - however, if MIR contains edges that end up not
-            // being needed in the LLVM IR after monomorphization, the funclet may
-            // be unreachable, and we don't have yet a way to skip building it in
-            // such an eventuality (which may be a better solution than this).
-            if fx.funclets[funclet_bb].is_none() {
-                fx.landing_pad_for(funclet_bb);
-            }
-
-            Some(
-                fx.funclets[funclet_bb]
-                    .as_ref()
-                    .expect("landing_pad_for didn't also create funclets entry"),
-            )
-        } else {
-            None
+        let cleanup_kinds = (&fx.cleanup_kinds).as_ref()?;
+        let funclet_bb = cleanup_kinds[self.bb].funclet_bb(self.bb)?;
+        // If `landing_pad_for` hasn't been called yet to create the `Funclet`,
+        // it has to be now. This may not seem necessary, as RPO should lead
+        // to all the unwind edges being visited (and so to `landing_pad_for`
+        // getting called for them), before building any of the blocks inside
+        // the funclet itself - however, if MIR contains edges that end up not
+        // being needed in the LLVM IR after monomorphization, the funclet may
+        // be unreachable, and we don't have yet a way to skip building it in
+        // such an eventuality (which may be a better solution than this).
+        if fx.funclets[funclet_bb].is_none() {
+            fx.landing_pad_for(funclet_bb);
         }
+        Some(
+            fx.funclets[funclet_bb]
+                .as_ref()
+                .expect("landing_pad_for didn't also create funclets entry"),
+        )
     }
 
     /// Get a basic block (creating it if necessary), possibly with cleanup
@@ -104,23 +99,24 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
         fx: &mut FunctionCx<'a, 'tcx, Bx>,
         target: mir::BasicBlock,
     ) -> (bool, bool) {
-        let target_funclet = fx.cleanup_kinds[target].funclet_bb(target);
-        let (needs_landing_pad, is_cleanupret) = match (self.funclet_bb, target_funclet) {
-            (None, None) => (false, false),
-            (None, Some(_)) => (true, false),
-            (Some(_), None) => {
-                let span = self.terminator.source_info.span;
-                span_bug!(span, "{:?} - jump out of cleanup?", self.terminator);
-            }
-            (Some(f), Some(t_f)) => {
-                if f == t_f || !base::wants_msvc_seh(fx.cx.tcx().sess) {
-                    (false, false)
-                } else {
-                    (true, true)
+        if let Some(ref cleanup_kinds) = fx.cleanup_kinds {
+            let funclet_bb = cleanup_kinds[self.bb].funclet_bb(self.bb);
+            let target_funclet = cleanup_kinds[target].funclet_bb(target);
+            let (needs_landing_pad, is_cleanupret) = match (funclet_bb, target_funclet) {
+                (None, None) => (false, false),
+                (None, Some(_)) => (true, false),
+                (Some(f), Some(t_f)) => (f != t_f, f != t_f),
+                (Some(_), None) => {
+                    let span = self.terminator.source_info.span;
+                    span_bug!(span, "{:?} - jump out of cleanup?", self.terminator);
                 }
-            }
-        };
-        (needs_landing_pad, is_cleanupret)
+            };
+            (needs_landing_pad, is_cleanupret)
+        } else {
+            let needs_landing_pad = !fx.mir[self.bb].is_cleanup && fx.mir[target].is_cleanup;
+            let is_cleanupret = false;
+            (needs_landing_pad, is_cleanupret)
+        }
     }
 
     fn funclet_br<Bx: BuilderMethods<'a, 'tcx>>(
@@ -456,86 +452,84 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             args1 = [place.llval];
             &args1[..]
         };
-        let (drop_fn, fn_abi) = match ty.kind() {
-            // FIXME(eddyb) perhaps move some of this logic into
-            // `Instance::resolve_drop_in_place`?
-            ty::Dynamic(_, _, ty::Dyn) => {
-                // IN THIS ARM, WE HAVE:
-                // ty = *mut (dyn Trait)
-                // which is: exists<T> ( *mut T,    Vtable<T: Trait> )
-                //                       args[0]    args[1]
-                //
-                // args = ( Data, Vtable )
-                //                  |
-                //                  v
-                //                /-------\
-                //                | ...   |
-                //                \-------/
-                //
-                let virtual_drop = Instance {
-                    def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0),
-                    substs: drop_fn.substs,
-                };
-                debug!("ty = {:?}", ty);
-                debug!("drop_fn = {:?}", drop_fn);
-                debug!("args = {:?}", args);
-                let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
-                let vtable = args[1];
-                // Truncate vtable off of args list
-                args = &args[..1];
-                (
-                    meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
-                        .get_fn(bx, vtable, ty, &fn_abi),
-                    fn_abi,
-                )
-            }
-            ty::Dynamic(_, _, ty::DynStar) => {
-                // IN THIS ARM, WE HAVE:
-                // ty = *mut (dyn* Trait)
-                // which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>)
-                //
-                // args = [ * ]
-                //          |
-                //          v
-                //      ( Data, Vtable )
-                //                |
-                //                v
-                //              /-------\
-                //              | ...   |
-                //              \-------/
-                //
-                //
-                // WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING
-                //
-                // data = &(*args[0]).0    // gives a pointer to Data above (really the same pointer)
-                // vtable = (*args[0]).1   // loads the vtable out
-                // (data, vtable)          // an equivalent Rust `*mut dyn Trait`
-                //
-                // SO THEN WE CAN USE THE ABOVE CODE.
-                let virtual_drop = Instance {
-                    def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0),
-                    substs: drop_fn.substs,
-                };
-                debug!("ty = {:?}", ty);
-                debug!("drop_fn = {:?}", drop_fn);
-                debug!("args = {:?}", args);
-                let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
-                let data = args[0];
-                let data_ty = bx.cx().backend_type(place.layout);
-                let vtable_ptr =
-                    bx.gep(data_ty, data, &[bx.cx().const_i32(0), bx.cx().const_i32(1)]);
-                let vtable = bx.load(bx.type_i8p(), vtable_ptr, abi::Align::ONE);
-                // Truncate vtable off of args list
-                args = &args[..1];
-                debug!("args' = {:?}", args);
-                (
-                    meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
-                        .get_fn(bx, vtable, ty, &fn_abi),
-                    fn_abi,
-                )
-            }
-            _ => (bx.get_fn_addr(drop_fn), bx.fn_abi_of_instance(drop_fn, ty::List::empty())),
-        };
+        let (drop_fn, fn_abi) =
+            match ty.kind() {
+                // FIXME(eddyb) perhaps move some of this logic into
+                // `Instance::resolve_drop_in_place`?
+                ty::Dynamic(_, _, ty::Dyn) => {
+                    // IN THIS ARM, WE HAVE:
+                    // ty = *mut (dyn Trait)
+                    // which is: exists<T> ( *mut T,    Vtable<T: Trait> )
+                    //                       args[0]    args[1]
+                    //
+                    // args = ( Data, Vtable )
+                    //                  |
+                    //                  v
+                    //                /-------\
+                    //                | ...   |
+                    //                \-------/
+                    //
+                    let virtual_drop = Instance {
+                        def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0),
+                        substs: drop_fn.substs,
+                    };
+                    debug!("ty = {:?}", ty);
+                    debug!("drop_fn = {:?}", drop_fn);
+                    debug!("args = {:?}", args);
+                    let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
+                    let vtable = args[1];
+                    // Truncate vtable off of args list
+                    args = &args[..1];
+                    (
+                        meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
+                            .get_fn(bx, vtable, ty, &fn_abi),
+                        fn_abi,
+                    )
+                }
+                ty::Dynamic(_, _, ty::DynStar) => {
+                    // IN THIS ARM, WE HAVE:
+                    // ty = *mut (dyn* Trait)
+                    // which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>)
+                    //
+                    // args = [ * ]
+                    //          |
+                    //          v
+                    //      ( Data, Vtable )
+                    //                |
+                    //                v
+                    //              /-------\
+                    //              | ...   |
+                    //              \-------/
+                    //
+                    //
+                    // WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING
+                    //
+                    // data = &(*args[0]).0    // gives a pointer to Data above (really the same pointer)
+                    // vtable = (*args[0]).1   // loads the vtable out
+                    // (data, vtable)          // an equivalent Rust `*mut dyn Trait`
+                    //
+                    // SO THEN WE CAN USE THE ABOVE CODE.
+                    let virtual_drop = Instance {
+                        def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0),
+                        substs: drop_fn.substs,
+                    };
+                    debug!("ty = {:?}", ty);
+                    debug!("drop_fn = {:?}", drop_fn);
+                    debug!("args = {:?}", args);
+                    let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
+                    let meta_ptr = place.project_field(bx, 1);
+                    let meta = bx.load_operand(meta_ptr);
+                    // Truncate vtable off of args list
+                    args = &args[..1];
+                    debug!("args' = {:?}", args);
+                    (
+                        meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
+                            .get_fn(bx, meta.immediate(), ty, &fn_abi),
+                        fn_abi,
+                    )
+                }
+                _ => (bx.get_fn_addr(drop_fn), bx.fn_abi_of_instance(drop_fn, ty::List::empty())),
+            };
         helper.do_call(
             self,
             bx,
@@ -569,11 +563,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         // with #[rustc_inherit_overflow_checks] and inlined from
         // another crate (mostly core::num generic/#[inline] fns),
         // while the current crate doesn't use overflow checks.
-        // NOTE: Unlike binops, negation doesn't have its own
-        // checked operation, just a comparison with the minimum
-        // value, so we have to check for the assert message.
-        if !bx.check_overflow() {
-            if let AssertKind::OverflowNeg(_) = *msg {
+        if !bx.cx().check_overflow() {
+            let overflow_not_to_check = match msg {
+                AssertKind::OverflowNeg(..) => true,
+                AssertKind::Overflow(op, ..) => op.is_checkable(),
+                _ => false,
+            };
+            if overflow_not_to_check {
                 const_cond = Some(expected);
             }
         }
@@ -659,35 +655,24 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         // Emit a panic or a no-op for `assert_*` intrinsics.
         // These are intrinsics that compile to panics so that we can get a message
         // which mentions the offending type, even from a const context.
-        #[derive(Debug, PartialEq)]
-        enum AssertIntrinsic {
-            Inhabited,
-            ZeroValid,
-            MemUninitializedValid,
-        }
-        let panic_intrinsic = intrinsic.and_then(|i| match i {
-            sym::assert_inhabited => Some(AssertIntrinsic::Inhabited),
-            sym::assert_zero_valid => Some(AssertIntrinsic::ZeroValid),
-            sym::assert_mem_uninitialized_valid => Some(AssertIntrinsic::MemUninitializedValid),
-            _ => None,
-        });
-        if let Some(intrinsic) = panic_intrinsic {
-            use AssertIntrinsic::*;
-
+        let panic_intrinsic = intrinsic.and_then(|s| ValidityRequirement::from_intrinsic(s));
+        if let Some(requirement) = panic_intrinsic {
             let ty = instance.unwrap().substs.type_at(0);
+
+            let do_panic = !bx
+                .tcx()
+                .check_validity_requirement((requirement, bx.param_env().and(ty)))
+                .expect("expect to have layout during codegen");
+
             let layout = bx.layout_of(ty);
-            let do_panic = match intrinsic {
-                Inhabited => layout.abi.is_uninhabited(),
-                ZeroValid => !bx.tcx().permits_zero_init(layout),
-                MemUninitializedValid => !bx.tcx().permits_uninit_init(layout),
-            };
+
             Some(if do_panic {
                 let msg_str = with_no_visible_paths!({
                     with_no_trimmed_paths!({
                         if layout.abi.is_uninhabited() {
                             // Use this error even for the other intrinsics as it is more precise.
                             format!("attempted to instantiate uninhabited type `{}`", ty)
-                        } else if intrinsic == ZeroValid {
+                        } else if requirement == ValidityRequirement::Zero {
                             format!("attempted to zero-initialize type `{}`, which is invalid", ty)
                         } else {
                             format!(
@@ -781,7 +766,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         };
 
         let extra_args = &args[sig.inputs().skip_binder().len()..];
-        let extra_args = bx.tcx().mk_type_list(extra_args.iter().map(|op_arg| {
+        let extra_args = bx.tcx().mk_type_list_from_iter(extra_args.iter().map(|op_arg| {
             let op_ty = op_arg.ty(self.mir, bx.tcx());
             self.monomorphize(op_ty)
         }));
@@ -1253,9 +1238,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     ) -> MergingSucc {
         debug!("codegen_terminator: {:?}", terminator);
 
-        // Create the cleanup bundle, if needed.
-        let funclet_bb = self.cleanup_kinds[bb].funclet_bb(bb);
-        let helper = TerminatorCodegenHelper { bb, terminator, funclet_bb };
+        let helper = TerminatorCodegenHelper { bb, terminator };
 
         let mergeable_succ = || {
             // Note: any call to `switch_to_block` will invalidate a `true` value
@@ -1547,7 +1530,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             slot
         } else {
             let layout = cx.layout_of(
-                cx.tcx().intern_tup(&[cx.tcx().mk_mut_ptr(cx.tcx().types.u8), cx.tcx().types.i32]),
+                cx.tcx().mk_tup(&[cx.tcx().mk_mut_ptr(cx.tcx().types.u8), cx.tcx().types.i32]),
             );
             let slot = PlaceRef::alloca(bx, layout);
             self.personality_slot = Some(slot);
@@ -1801,8 +1784,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         match (src.layout.abi, dst.layout.abi) {
             (abi::Abi::Scalar(src_scalar), abi::Abi::Scalar(dst_scalar)) => {
                 // HACK(eddyb) LLVM doesn't like `bitcast`s between pointers and non-pointers.
-                let src_is_ptr = src_scalar.primitive() == abi::Pointer;
-                let dst_is_ptr = dst_scalar.primitive() == abi::Pointer;
+                let src_is_ptr = matches!(src_scalar.primitive(), abi::Pointer(_));
+                let dst_is_ptr = matches!(dst_scalar.primitive(), abi::Pointer(_));
                 if src_is_ptr == dst_is_ptr {
                     assert_eq!(src.layout.size, dst.layout.size);
 
diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
index e9bc40c33..708f3bc0c 100644
--- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
@@ -385,10 +385,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     calculate_debuginfo_offset(bx, local, &var, base);
 
                 // Create a variable which will be a pointer to the actual value
-                let ptr_ty = bx.tcx().mk_ty(ty::RawPtr(ty::TypeAndMut {
-                    mutbl: mir::Mutability::Mut,
-                    ty: place.layout.ty,
-                }));
+                let ptr_ty = bx
+                    .tcx()
+                    .mk_ptr(ty::TypeAndMut { mutbl: mir::Mutability::Mut, ty: place.layout.ty });
                 let ptr_layout = bx.layout_of(ptr_ty);
                 let alloca = PlaceRef::alloca(bx, ptr_layout);
                 bx.set_var_name(alloca.llval, &(var.name.to_string() + ".dbg.spill"));
diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
index 766dc74cb..7af7fc92d 100644
--- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
@@ -218,9 +218,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 args[1].val.unaligned_volatile_store(bx, dst);
                 return;
             }
-            sym::add_with_overflow
-            | sym::sub_with_overflow
-            | sym::mul_with_overflow
             | sym::unchecked_div
             | sym::unchecked_rem
             | sym::unchecked_shl
@@ -232,28 +229,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let ty = arg_tys[0];
                 match int_type_width_signed(ty, bx.tcx()) {
                     Some((_width, signed)) => match name {
-                        sym::add_with_overflow
-                        | sym::sub_with_overflow
-                        | sym::mul_with_overflow => {
-                            let op = match name {
-                                sym::add_with_overflow => OverflowOp::Add,
-                                sym::sub_with_overflow => OverflowOp::Sub,
-                                sym::mul_with_overflow => OverflowOp::Mul,
-                                _ => bug!(),
-                            };
-                            let (val, overflow) =
-                                bx.checked_binop(op, ty, args[0].immediate(), args[1].immediate());
-                            // Convert `i1` to a `bool`, and write it to the out parameter
-                            let val = bx.from_immediate(val);
-                            let overflow = bx.from_immediate(overflow);
-
-                            let dest = result.project_field(bx, 0);
-                            bx.store(val, dest.llval, dest.align);
-                            let dest = result.project_field(bx, 1);
-                            bx.store(overflow, dest.llval, dest.align);
-
-                            return;
-                        }
                         sym::exact_div => {
                             if signed {
                                 bx.exactsdiv(args[0].immediate(), args[1].immediate())
diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs
index 79c66a955..2ec9fdbf4 100644
--- a/compiler/rustc_codegen_ssa/src/mir/mod.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs
@@ -1,8 +1,9 @@
+use crate::base;
 use crate::traits::*;
 use rustc_middle::mir;
 use rustc_middle::mir::interpret::ErrorHandled;
 use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, TyAndLayout};
-use rustc_middle::ty::{self, Instance, Ty, TypeFoldable, TypeVisitable};
+use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeFoldable, TypeVisitableExt};
 use rustc_target::abi::call::{FnAbi, PassMode};
 
 use std::iter;
@@ -58,7 +59,7 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
     cached_llbbs: IndexVec<mir::BasicBlock, CachedLlbb<Bx::BasicBlock>>,
 
     /// The funclet status of each basic block
-    cleanup_kinds: IndexVec<mir::BasicBlock, analyze::CleanupKind>,
+    cleanup_kinds: Option<IndexVec<mir::BasicBlock, analyze::CleanupKind>>,
 
     /// When targeting MSVC, this stores the cleanup info for each funclet BB.
     /// This is initialized at the same time as the `landing_pads` entry for the
@@ -104,7 +105,7 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     pub fn monomorphize<T>(&self, value: T) -> T
     where
-        T: Copy + TypeFoldable<'tcx>,
+        T: Copy + TypeFoldable<TyCtxt<'tcx>>,
     {
         debug!("monomorphize: self.instance={:?}", self.instance);
         self.instance.subst_mir_and_normalize_erasing_regions(
@@ -166,7 +167,8 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         start_bx.set_personality_fn(cx.eh_personality());
     }
 
-    let cleanup_kinds = analyze::cleanup_kinds(&mir);
+    let cleanup_kinds = base::wants_msvc_seh(cx.tcx().sess).then(|| analyze::cleanup_kinds(&mir));
+
     let cached_llbbs: IndexVec<mir::BasicBlock, CachedLlbb<Bx::BasicBlock>> =
         mir.basic_blocks
             .indices()
diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs
index fbe30154a..cf02f59f6 100644
--- a/compiler/rustc_codegen_ssa/src/mir/place.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/place.rs
@@ -9,7 +9,7 @@ use rustc_middle::mir;
 use rustc_middle::mir::tcx::PlaceTy;
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout};
 use rustc_middle::ty::{self, Ty};
-use rustc_target::abi::{Abi, Align, FieldsShape, Int, TagEncoding};
+use rustc_target::abi::{Abi, Align, FieldsShape, Int, Pointer, TagEncoding};
 use rustc_target::abi::{VariantIdx, Variants};
 
 #[derive(Copy, Clone, Debug)]
@@ -209,6 +209,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
         bx: &mut Bx,
         cast_to: Ty<'tcx>,
     ) -> V {
+        let dl = &bx.tcx().data_layout;
         let cast_to_layout = bx.cx().layout_of(cast_to);
         let cast_to_size = cast_to_layout.layout.size();
         let cast_to = bx.cx().immediate_backend_type(cast_to_layout);
@@ -250,12 +251,14 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
             TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => {
                 // Cast to an integer so we don't have to treat a pointer as a
                 // special case.
-                let (tag, tag_llty) = if tag_scalar.primitive().is_ptr() {
-                    let t = bx.type_isize();
-                    let tag = bx.ptrtoint(tag_imm, t);
-                    (tag, t)
-                } else {
-                    (tag_imm, bx.cx().immediate_backend_type(tag_op.layout))
+                let (tag, tag_llty) = match tag_scalar.primitive() {
+                    // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
+                    Pointer(_) => {
+                        let t = bx.type_from_integer(dl.ptr_sized_integer());
+                        let tag = bx.ptrtoint(tag_imm, t);
+                        (tag, t)
+                    }
+                    _ => (tag_imm, bx.cx().immediate_backend_type(tag_op.layout)),
                 };
 
                 let tag_size = tag_scalar.size(bx.cx());
diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
index 23196c8cb..3d856986f 100644
--- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
@@ -13,6 +13,7 @@ use rustc_middle::ty::cast::{CastTy, IntTy};
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
 use rustc_middle::ty::{self, adjustment::PointerCast, Instance, Ty, TyCtxt};
 use rustc_span::source_map::{Span, DUMMY_SP};
+use rustc_target::abi::VariantIdx;
 
 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     #[instrument(level = "trace", skip(self, bx))]
@@ -99,24 +100,24 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     }
                 }
 
-                let count =
-                    self.monomorphize(count).eval_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all());
+                let count = self
+                    .monomorphize(count)
+                    .eval_target_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all());
 
                 bx.write_operand_repeatedly(cg_elem, count, dest);
             }
 
             mir::Rvalue::Aggregate(ref kind, ref operands) => {
-                let (dest, active_field_index) = match **kind {
-                    mir::AggregateKind::Adt(adt_did, variant_index, _, _, active_field_index) => {
-                        dest.codegen_set_discr(bx, variant_index);
-                        if bx.tcx().adt_def(adt_did).is_enum() {
-                            (dest.project_downcast(bx, variant_index), active_field_index)
-                        } else {
-                            (dest, active_field_index)
-                        }
+                let (variant_index, variant_dest, active_field_index) = match **kind {
+                    mir::AggregateKind::Adt(_, variant_index, _, _, active_field_index) => {
+                        let variant_dest = dest.project_downcast(bx, variant_index);
+                        (variant_index, variant_dest, active_field_index)
                     }
-                    _ => (dest, None),
+                    _ => (VariantIdx::from_u32(0), dest, None),
                 };
+                if active_field_index.is_some() {
+                    assert_eq!(operands.len(), 1);
+                }
                 for (i, operand) in operands.iter().enumerate() {
                     let op = self.codegen_operand(bx, operand);
                     // Do not generate stores and GEPis for zero-sized fields.
@@ -124,13 +125,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         let field_index = active_field_index.unwrap_or(i);
                         let field = if let mir::AggregateKind::Array(_) = **kind {
                             let llindex = bx.cx().const_usize(field_index as u64);
-                            dest.project_index(bx, llindex)
+                            variant_dest.project_index(bx, llindex)
                         } else {
-                            dest.project_field(bx, field_index)
+                            variant_dest.project_field(bx, field_index)
                         };
                         op.val.store(bx, field);
                     }
                 }
+                dest.codegen_set_discr(bx, variant_index);
             }
 
             _ => {
@@ -411,7 +413,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     lhs.layout.ty,
                 );
                 let val_ty = op.ty(bx.tcx(), lhs.layout.ty, rhs.layout.ty);
-                let operand_ty = bx.tcx().intern_tup(&[val_ty, bx.tcx().types.bool]);
+                let operand_ty = bx.tcx().mk_tup(&[val_ty, bx.tcx().types.bool]);
                 OperandRef { val: result, layout: bx.cx().layout_of(operand_ty) }
             }
 
@@ -491,7 +493,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         if let Some(index) = place.as_local() {
             if let LocalRef::Operand(Some(op)) = self.locals[index] {
                 if let ty::Array(_, n) = op.layout.ty.kind() {
-                    let n = n.eval_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all());
+                    let n = n.eval_target_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all());
                     return bx.cx().const_usize(n);
                 }
             }
@@ -650,15 +652,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         rhs: Bx::Value,
         input_ty: Ty<'tcx>,
     ) -> OperandValue<Bx::Value> {
-        // This case can currently arise only from functions marked
-        // with #[rustc_inherit_overflow_checks] and inlined from
-        // another crate (mostly core::num generic/#[inline] fns),
-        // while the current crate doesn't use overflow checks.
-        if !bx.cx().check_overflow() {
-            let val = self.codegen_scalar_binop(bx, op, lhs, rhs, input_ty);
-            return OperandValue::Pair(val, bx.cx().const_bool(false));
-        }
-
         let (val, of) = match op {
             // These are checked using intrinsics
             mir::BinOp::Add | mir::BinOp::Sub | mir::BinOp::Mul => {
diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs
index 19452c8cd..60fbceb34 100644
--- a/compiler/rustc_codegen_ssa/src/mir/statement.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs
@@ -91,6 +91,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             mir::StatementKind::FakeRead(..)
             | mir::StatementKind::Retag { .. }
             | mir::StatementKind::AscribeUserType(..)
+            | mir::StatementKind::ConstEvalCounter
             | mir::StatementKind::Nop => {}
         }
     }
diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs
index 739963fff..e59fad99a 100644
--- a/compiler/rustc_codegen_ssa/src/target_features.rs
+++ b/compiler/rustc_codegen_ssa/src/target_features.rs
@@ -3,12 +3,12 @@ use rustc_attr::InstructionSetAttr;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::Applicability;
-use rustc_hir as hir;
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
 use rustc_hir::def_id::LocalDefId;
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::ty::query::Providers;
-use rustc_middle::ty::TyCtxt;
+use rustc_middle::ty::{DefIdTree, TyCtxt};
 use rustc_session::parse::feature_err;
 use rustc_session::Session;
 use rustc_span::symbol::sym;
@@ -185,7 +185,7 @@ const X86_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
     ("avx512vpopcntdq", Some(sym::avx512_target_feature)),
     ("bmi1", None),
     ("bmi2", None),
-    ("cmpxchg16b", Some(sym::cmpxchg16b_target_feature)),
+    ("cmpxchg16b", None),
     ("ermsb", Some(sym::ermsb_target_feature)),
     ("f16c", None),
     ("fma", None),
@@ -286,6 +286,7 @@ const WASM_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
     ("mutable-globals", Some(sym::wasm_target_feature)),
     ("nontrapping-fptoint", Some(sym::wasm_target_feature)),
     ("reference-types", Some(sym::wasm_target_feature)),
+    ("relaxed-simd", Some(sym::wasm_target_feature)),
     ("sign-ext", Some(sym::wasm_target_feature)),
     ("simd128", None),
     // tidy-alphabetical-end
@@ -393,7 +394,6 @@ pub fn from_target_feature(
                 Some(sym::sse4a_target_feature) => rust_features.sse4a_target_feature,
                 Some(sym::tbm_target_feature) => rust_features.tbm_target_feature,
                 Some(sym::wasm_target_feature) => rust_features.wasm_target_feature,
-                Some(sym::cmpxchg16b_target_feature) => rust_features.cmpxchg16b_target_feature,
                 Some(sym::movbe_target_feature) => rust_features.movbe_target_feature,
                 Some(sym::rtm_target_feature) => rust_features.rtm_target_feature,
                 Some(sym::ermsb_target_feature) => rust_features.ermsb_target_feature,
@@ -440,12 +440,9 @@ fn asm_target_features(tcx: TyCtxt<'_>, did: DefId) -> &FxHashSet<Symbol> {
 /// Checks the function annotated with `#[target_feature]` is not a safe
 /// trait method implementation, reporting an error if it is.
 pub fn check_target_feature_trait_unsafe(tcx: TyCtxt<'_>, id: LocalDefId, attr_span: Span) {
-    let hir_id = tcx.hir().local_def_id_to_hir_id(id);
-    let node = tcx.hir().get(hir_id);
-    if let hir::Node::ImplItem(hir::ImplItem { kind: hir::ImplItemKind::Fn(..), .. }) = node {
-        let parent_id = tcx.hir().get_parent_item(hir_id);
-        let parent_item = tcx.hir().expect_item(parent_id.def_id);
-        if let hir::ItemKind::Impl(hir::Impl { of_trait: Some(_), .. }) = parent_item.kind {
+    if let DefKind::AssocFn = tcx.def_kind(id) {
+        let parent_id = tcx.local_parent(id);
+        if let DefKind::Impl { of_trait: true } = tcx.def_kind(parent_id) {
             tcx.sess
                 .struct_span_err(
                     attr_span,
diff --git a/compiler/rustc_codegen_ssa/src/traits/backend.rs b/compiler/rustc_codegen_ssa/src/traits/backend.rs
index 5c35070ea..64bebe50d 100644
--- a/compiler/rustc_codegen_ssa/src/traits/backend.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/backend.rs
@@ -57,6 +57,10 @@ impl<'tcx, T> Backend<'tcx> for T where
 }
 
 pub trait CodegenBackend {
+    /// Locale resources for diagnostic messages - a string the content of the Fluent resource.
+    /// Called before `init` so that all other functions are able to emit translatable diagnostics.
+    fn locale_resource(&self) -> &'static str;
+
     fn init(&self, _sess: &Session) {}
     fn print(&self, _req: PrintRequest, _sess: &Session) {}
     fn target_features(&self, _sess: &Session, _allow_unstable: bool) -> Vec<Symbol> {
diff --git a/compiler/rustc_codegen_ssa/src/traits/write.rs b/compiler/rustc_codegen_ssa/src/traits/write.rs
index e0e8ffa89..9826256a4 100644
--- a/compiler/rustc_codegen_ssa/src/traits/write.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/write.rs
@@ -8,6 +8,7 @@ use rustc_middle::dep_graph::WorkProduct;
 pub trait WriteBackendMethods: 'static + Sized + Clone {
     type Module: Send + Sync;
     type TargetMachine;
+    type TargetMachineError;
     type ModuleBuffer: ModuleBufferMethods;
     type ThinData: Send + Sync;
     type ThinBuffer: ThinBufferMethods;
diff --git a/compiler/rustc_const_eval/Cargo.toml b/compiler/rustc_const_eval/Cargo.toml
index 51489e293..98ac36c1c 100644
--- a/compiler/rustc_const_eval/Cargo.toml
+++ b/compiler/rustc_const_eval/Cargo.toml
@@ -19,7 +19,6 @@ rustc_infer = { path = "../rustc_infer" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_mir_dataflow = { path = "../rustc_mir_dataflow" }
-rustc_query_system = { path = "../rustc_query_system" }
 rustc_session = { path = "../rustc_session" }
 rustc_target = { path = "../rustc_target" }
 rustc_trait_selection = { path = "../rustc_trait_selection" }
diff --git a/compiler/rustc_error_messages/locales/en-US/const_eval.ftl b/compiler/rustc_const_eval/locales/en-US.ftl
index 33bb116d6..33bb116d6 100644
--- a/compiler/rustc_error_messages/locales/en-US/const_eval.ftl
+++ b/compiler/rustc_const_eval/locales/en-US.ftl
diff --git a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
index 18e01567c..7564ba17b 100644
--- a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
+++ b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
@@ -54,7 +54,7 @@ fn eval_body_using_ecx<'mir, 'tcx>(
 
     trace!(
         "eval_body_using_ecx: pushing stack frame for global: {}{}",
-        with_no_trimmed_paths!(ty::tls::with(|tcx| tcx.def_path_str(cid.instance.def_id()))),
+        with_no_trimmed_paths!(ecx.tcx.def_path_str(cid.instance.def_id())),
         cid.promoted.map_or_else(String::new, |p| format!("::promoted[{:?}]", p))
     );
 
@@ -180,13 +180,13 @@ pub(super) fn op_to_const<'tcx>(
                         (ecx.tcx.global_alloc(alloc_id).unwrap_memory(), offset.bytes())
                     }
                     (None, _offset) => (
-                        ecx.tcx.intern_const_alloc(Allocation::from_bytes_byte_aligned_immutable(
+                        ecx.tcx.mk_const_alloc(Allocation::from_bytes_byte_aligned_immutable(
                             b"" as &[u8],
                         )),
                         0,
                     ),
                 };
-                let len = b.to_machine_usize(ecx).unwrap();
+                let len = b.to_target_usize(ecx).unwrap();
                 let start = start.try_into().unwrap();
                 let len: usize = len.try_into().unwrap();
                 ConstValue::Slice { data, start, end: start + len }
diff --git a/compiler/rustc_const_eval/src/const_eval/fn_queries.rs b/compiler/rustc_const_eval/src/const_eval/fn_queries.rs
index 351c70130..9eaab1f47 100644
--- a/compiler/rustc_const_eval/src/const_eval/fn_queries.rs
+++ b/compiler/rustc_const_eval/src/const_eval/fn_queries.rs
@@ -17,7 +17,8 @@ pub fn is_unstable_const_fn(tcx: TyCtxt<'_>, def_id: DefId) -> Option<Symbol> {
 
 pub fn is_parent_const_impl_raw(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     let parent_id = tcx.local_parent(def_id);
-    tcx.def_kind(parent_id) == DefKind::Impl && tcx.constness(parent_id) == hir::Constness::Const
+    matches!(tcx.def_kind(parent_id), DefKind::Impl { .. })
+        && tcx.constness(parent_id) == hir::Constness::Const
 }
 
 /// Checks whether an item is considered to be `const`. If it is a constructor, it is const. If
@@ -66,7 +67,7 @@ fn is_promotable_const_fn(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
                 if cfg!(debug_assertions) && stab.promotable {
                     let sig = tcx.fn_sig(def_id);
                     assert_eq!(
-                        sig.unsafety(),
+                        sig.skip_binder().unsafety(),
                         hir::Unsafety::Normal,
                         "don't mark const unsafe fns as promotable",
                         // https://github.com/rust-lang/rust/pull/53851#issuecomment-418760682
diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs
index 4709514c8..a44f70ed0 100644
--- a/compiler/rustc_const_eval/src/const_eval/machine.rs
+++ b/compiler/rustc_const_eval/src/const_eval/machine.rs
@@ -244,7 +244,7 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
         assert_eq!(args.len(), 2);
 
         let ptr = self.read_pointer(&args[0])?;
-        let target_align = self.read_scalar(&args[1])?.to_machine_usize(self)?;
+        let target_align = self.read_scalar(&args[1])?.to_target_usize(self)?;
 
         if !target_align.is_power_of_two() {
             throw_ub_format!("`align_offset` called with non-power-of-two align: {}", target_align);
@@ -276,7 +276,7 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
                     Ok(ControlFlow::Break(()))
                 } else {
                     // Not alignable in const, return `usize::MAX`.
-                    let usize_max = Scalar::from_machine_usize(self.machine_usize_max(), self);
+                    let usize_max = Scalar::from_target_usize(self.target_usize_max(), self);
                     self.write_scalar(usize_max, dest)?;
                     self.return_to_block(ret)?;
                     Ok(ControlFlow::Break(()))
@@ -470,8 +470,8 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
                 ecx.write_scalar(Scalar::from_u8(cmp), dest)?;
             }
             sym::const_allocate => {
-                let size = ecx.read_scalar(&args[0])?.to_machine_usize(ecx)?;
-                let align = ecx.read_scalar(&args[1])?.to_machine_usize(ecx)?;
+                let size = ecx.read_scalar(&args[0])?.to_target_usize(ecx)?;
+                let align = ecx.read_scalar(&args[1])?.to_target_usize(ecx)?;
 
                 let align = match Align::from_bytes(align) {
                     Ok(a) => a,
@@ -487,8 +487,8 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
             }
             sym::const_deallocate => {
                 let ptr = ecx.read_pointer(&args[0])?;
-                let size = ecx.read_scalar(&args[1])?.to_machine_usize(ecx)?;
-                let align = ecx.read_scalar(&args[2])?.to_machine_usize(ecx)?;
+                let size = ecx.read_scalar(&args[1])?.to_target_usize(ecx)?;
+                let align = ecx.read_scalar(&args[2])?.to_target_usize(ecx)?;
 
                 let size = Size::from_bytes(size);
                 let align = match Align::from_bytes(align) {
@@ -561,8 +561,8 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
         throw_unsup_format!("pointer arithmetic or comparison is not supported at compile-time");
     }
 
-    fn before_terminator(ecx: &mut InterpCx<'mir, 'tcx, Self>) -> InterpResult<'tcx> {
-        // The step limit has already been hit in a previous call to `before_terminator`.
+    fn increment_const_eval_counter(ecx: &mut InterpCx<'mir, 'tcx, Self>) -> InterpResult<'tcx> {
+        // The step limit has already been hit in a previous call to `increment_const_eval_counter`.
         if ecx.machine.steps_remaining == 0 {
             return Ok(());
         }
@@ -622,10 +622,9 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
         let alloc = alloc.inner();
         if is_write {
             // Write access. These are never allowed, but we give a targeted error message.
-            if alloc.mutability == Mutability::Not {
-                Err(err_ub!(WriteToReadOnly(alloc_id)).into())
-            } else {
-                Err(ConstEvalErrKind::ModifiedGlobal.into())
+            match alloc.mutability {
+                Mutability::Not => Err(err_ub!(WriteToReadOnly(alloc_id)).into()),
+                Mutability::Mut => Err(ConstEvalErrKind::ModifiedGlobal.into()),
             }
         } else {
             // Read access. These are usually allowed, with some exceptions.
diff --git a/compiler/rustc_const_eval/src/const_eval/mod.rs b/compiler/rustc_const_eval/src/const_eval/mod.rs
index 01b2b4b5d..3cdf1e6e3 100644
--- a/compiler/rustc_const_eval/src/const_eval/mod.rs
+++ b/compiler/rustc_const_eval/src/const_eval/mod.rs
@@ -107,7 +107,7 @@ pub(crate) fn try_destructure_mir_constant<'tcx>(
 
     // We go to `usize` as we cannot allocate anything bigger anyway.
     let (field_count, variant, down) = match val.ty().kind() {
-        ty::Array(_, len) => (len.eval_usize(tcx, param_env) as usize, None, op),
+        ty::Array(_, len) => (len.eval_target_usize(tcx, param_env) as usize, None, op),
         ty::Adt(def, _) if def.variants().is_empty() => {
             throw_ub!(Unreachable)
         }
@@ -155,7 +155,7 @@ pub(crate) fn deref_mir_constant<'tcx>(
         // In case of unsized types, figure out the real type behind.
         MemPlaceMeta::Meta(scalar) => match mplace.layout.ty.kind() {
             ty::Str => bug!("there's no sized equivalent of a `str`"),
-            ty::Slice(elem_ty) => tcx.mk_array(*elem_ty, scalar.to_machine_usize(&tcx).unwrap()),
+            ty::Slice(elem_ty) => tcx.mk_array(*elem_ty, scalar.to_target_usize(&tcx).unwrap()),
             _ => bug!(
                 "type {} should not have metadata, but had {:?}",
                 mplace.layout.ty,
diff --git a/compiler/rustc_const_eval/src/const_eval/valtrees.rs b/compiler/rustc_const_eval/src/const_eval/valtrees.rs
index 498c00873..a73f778d4 100644
--- a/compiler/rustc_const_eval/src/const_eval/valtrees.rs
+++ b/compiler/rustc_const_eval/src/const_eval/valtrees.rs
@@ -151,7 +151,7 @@ pub(crate) fn const_to_valtree_inner<'tcx>(
         // FIXME(oli-obk): we can probably encode closures just like structs
         | ty::Closure(..)
         | ty::Generator(..)
-        | ty::GeneratorWitness(..) => Err(ValTreeCreationError::NonSupportedType),
+        | ty::GeneratorWitness(..) |ty::GeneratorWitnessMIR(..)=> Err(ValTreeCreationError::NonSupportedType),
     }
 }
 
@@ -193,7 +193,7 @@ fn get_info_on_unsized_field<'tcx>(
 
     // Have to adjust type for ty::Str
     let unsized_inner_ty = match unsized_inner_ty.kind() {
-        ty::Str => tcx.mk_ty(ty::Uint(ty::UintTy::U8)),
+        ty::Str => tcx.types.u8,
         _ => unsized_inner_ty,
     };
 
@@ -216,7 +216,7 @@ fn create_pointee_place<'tcx>(
 
         let (unsized_inner_ty, num_elems) = get_info_on_unsized_field(ty, valtree, tcx);
         let unsized_inner_ty = match unsized_inner_ty.kind() {
-            ty::Str => tcx.mk_ty(ty::Uint(ty::UintTy::U8)),
+            ty::Str => tcx.types.u8,
             _ => unsized_inner_ty,
         };
         let unsized_inner_ty_size =
@@ -239,7 +239,7 @@ fn create_pointee_place<'tcx>(
         MPlaceTy::from_aligned_ptr_with_meta(
             ptr.into(),
             layout,
-            MemPlaceMeta::Meta(Scalar::from_machine_usize(num_elems as u64, &tcx)),
+            MemPlaceMeta::Meta(Scalar::from_target_usize(num_elems as u64, &tcx)),
         )
     } else {
         create_mplace_from_layout(ecx, ty)
@@ -314,6 +314,7 @@ pub fn valtree_to_const_value<'tcx>(
         | ty::Closure(..)
         | ty::Generator(..)
         | ty::GeneratorWitness(..)
+        | ty::GeneratorWitnessMIR(..)
         | ty::FnPtr(_)
         | ty::RawPtr(_)
         | ty::Str
@@ -354,7 +355,7 @@ fn valtree_into_mplace<'tcx>(
             let imm = match inner_ty.kind() {
                 ty::Slice(_) | ty::Str => {
                     let len = valtree.unwrap_branch().len();
-                    let len_scalar = Scalar::from_machine_usize(len as u64, &tcx);
+                    let len_scalar = Scalar::from_target_usize(len as u64, &tcx);
 
                     Immediate::ScalarPair(
                         Scalar::from_maybe_pointer((*pointee_place).ptr, &tcx),
@@ -425,7 +426,7 @@ fn valtree_into_mplace<'tcx>(
                         place
                             .offset_with_meta(
                                 offset,
-                                MemPlaceMeta::Meta(Scalar::from_machine_usize(
+                                MemPlaceMeta::Meta(Scalar::from_target_usize(
                                     num_elems as u64,
                                     &tcx,
                                 )),
diff --git a/compiler/rustc_const_eval/src/errors.rs b/compiler/rustc_const_eval/src/errors.rs
index 4b0550767..f8b7cc6d7 100644
--- a/compiler/rustc_const_eval/src/errors.rs
+++ b/compiler/rustc_const_eval/src/errors.rs
@@ -9,12 +9,12 @@ pub(crate) struct UnstableInStable {
     #[primary_span]
     pub span: Span,
     #[suggestion(
-        unstable_sugg,
+        const_eval_unstable_sugg,
         code = "#[rustc_const_unstable(feature = \"...\", issue = \"...\")]\n",
         applicability = "has-placeholders"
     )]
     #[suggestion(
-        bypass_sugg,
+        const_eval_bypass_sugg,
         code = "#[rustc_allow_const_fn_unstable({gate})]\n",
         applicability = "has-placeholders"
     )]
@@ -35,15 +35,15 @@ pub(crate) struct StaticAccessErr {
     #[primary_span]
     pub span: Span,
     pub kind: ConstContext,
-    #[note(teach_note)]
-    #[help(teach_help)]
+    #[note(const_eval_teach_note)]
+    #[help(const_eval_teach_help)]
     pub teach: Option<()>,
 }
 
 #[derive(Diagnostic)]
 #[diag(const_eval_raw_ptr_to_int)]
 #[note]
-#[note(note2)]
+#[note(const_eval_note2)]
 pub(crate) struct RawPtrToIntErr {
     #[primary_span]
     pub span: Span,
@@ -118,7 +118,7 @@ pub(crate) struct UnallowedMutableRefs {
     #[primary_span]
     pub span: Span,
     pub kind: ConstContext,
-    #[note(teach_note)]
+    #[note(const_eval_teach_note)]
     pub teach: Option<()>,
 }
 
@@ -128,7 +128,7 @@ pub(crate) struct UnallowedMutableRefsRaw {
     #[primary_span]
     pub span: Span,
     pub kind: ConstContext,
-    #[note(teach_note)]
+    #[note(const_eval_teach_note)]
     pub teach: Option<()>,
 }
 #[derive(Diagnostic)]
@@ -163,7 +163,7 @@ pub(crate) struct UnallowedHeapAllocations {
     #[label]
     pub span: Span,
     pub kind: ConstContext,
-    #[note(teach_note)]
+    #[note(const_eval_teach_note)]
     pub teach: Option<()>,
 }
 
@@ -184,7 +184,7 @@ pub(crate) struct InteriorMutableDataRefer {
     #[help]
     pub opt_help: Option<()>,
     pub kind: ConstContext,
-    #[note(teach_note)]
+    #[note(const_eval_teach_note)]
     pub teach: Option<()>,
 }
 
diff --git a/compiler/rustc_const_eval/src/interpret/cast.rs b/compiler/rustc_const_eval/src/interpret/cast.rs
index b2c847d3f..2be5ed896 100644
--- a/compiler/rustc_const_eval/src/interpret/cast.rs
+++ b/compiler/rustc_const_eval/src/interpret/cast.rs
@@ -126,7 +126,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                     let vtable = self.get_vtable_ptr(src.layout.ty, data.principal())?;
                     let vtable = Scalar::from_maybe_pointer(vtable, self);
                     let data = self.read_immediate(src)?.to_scalar();
-                    let _assert_pointer_sized = data.to_pointer(self)?;
+                    let _assert_pointer_like = data.to_pointer(self)?;
                     let val = Immediate::ScalarPair(data, vtable);
                     self.write_immediate(val, dest)?;
                 } else {
@@ -231,7 +231,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         // First cast to usize.
         let scalar = src.to_scalar();
         let addr = self.cast_from_int_like(scalar, src.layout, self.tcx.types.usize)?;
-        let addr = addr.to_machine_usize(self)?;
+        let addr = addr.to_target_usize(self)?;
 
         // Then turn address into pointer.
         let ptr = M::ptr_from_addr_cast(&self, addr)?;
@@ -312,6 +312,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         }
     }
 
+    /// `src` is a *pointer to* a `source_ty`, and in `dest` we should store a pointer to th same
+    /// data at type `cast_ty`.
     fn unsize_into_ptr(
         &mut self,
         src: &OpTy<'tcx, M::Provenance>,
@@ -328,11 +330,14 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             (&ty::Array(_, length), &ty::Slice(_)) => {
                 let ptr = self.read_scalar(src)?;
                 // u64 cast is from usize to u64, which is always good
-                let val =
-                    Immediate::new_slice(ptr, length.eval_usize(*self.tcx, self.param_env), self);
+                let val = Immediate::new_slice(
+                    ptr,
+                    length.eval_target_usize(*self.tcx, self.param_env),
+                    self,
+                );
                 self.write_immediate(val, dest)
             }
-            (ty::Dynamic(data_a, ..), ty::Dynamic(data_b, ..)) => {
+            (ty::Dynamic(data_a, _, ty::Dyn), ty::Dynamic(data_b, _, ty::Dyn)) => {
                 let val = self.read_immediate(src)?;
                 if data_a.principal() == data_b.principal() {
                     // A NOP cast that doesn't actually change anything, should be allowed even with mismatching vtables.
@@ -356,7 +361,12 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             }
 
             _ => {
-                span_bug!(self.cur_span(), "invalid unsizing {:?} -> {:?}", src.layout.ty, cast_ty)
+                span_bug!(
+                    self.cur_span(),
+                    "invalid pointer unsizing {:?} -> {:?}",
+                    src.layout.ty,
+                    cast_ty
+                )
             }
         }
     }
diff --git a/compiler/rustc_const_eval/src/interpret/discriminant.rs b/compiler/rustc_const_eval/src/interpret/discriminant.rs
new file mode 100644
index 000000000..557e72124
--- /dev/null
+++ b/compiler/rustc_const_eval/src/interpret/discriminant.rs
@@ -0,0 +1,238 @@
+//! Functions for reading and writing discriminants of multi-variant layouts (enums and generators).
+
+use rustc_middle::ty::layout::{LayoutOf, PrimitiveExt};
+use rustc_middle::{mir, ty};
+use rustc_target::abi::{self, TagEncoding};
+use rustc_target::abi::{VariantIdx, Variants};
+
+use super::{ImmTy, InterpCx, InterpResult, Machine, OpTy, PlaceTy, Scalar};
+
+impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
+    /// Writes the discriminant of the given variant.
+    #[instrument(skip(self), level = "trace")]
+    pub fn write_discriminant(
+        &mut self,
+        variant_index: VariantIdx,
+        dest: &PlaceTy<'tcx, M::Provenance>,
+    ) -> InterpResult<'tcx> {
+        // Layout computation excludes uninhabited variants from consideration
+        // therefore there's no way to represent those variants in the given layout.
+        // Essentially, uninhabited variants do not have a tag that corresponds to their
+        // discriminant, so we cannot do anything here.
+        // When evaluating we will always error before even getting here, but ConstProp 'executes'
+        // dead code, so we cannot ICE here.
+        if dest.layout.for_variant(self, variant_index).abi.is_uninhabited() {
+            throw_ub!(UninhabitedEnumVariantWritten)
+        }
+
+        match dest.layout.variants {
+            abi::Variants::Single { index } => {
+                assert_eq!(index, variant_index);
+            }
+            abi::Variants::Multiple {
+                tag_encoding: TagEncoding::Direct,
+                tag: tag_layout,
+                tag_field,
+                ..
+            } => {
+                // No need to validate that the discriminant here because the
+                // `TyAndLayout::for_variant()` call earlier already checks the variant is valid.
+
+                let discr_val =
+                    dest.layout.ty.discriminant_for_variant(*self.tcx, variant_index).unwrap().val;
+
+                // raw discriminants for enums are isize or bigger during
+                // their computation, but the in-memory tag is the smallest possible
+                // representation
+                let size = tag_layout.size(self);
+                let tag_val = size.truncate(discr_val);
+
+                let tag_dest = self.place_field(dest, tag_field)?;
+                self.write_scalar(Scalar::from_uint(tag_val, size), &tag_dest)?;
+            }
+            abi::Variants::Multiple {
+                tag_encoding:
+                    TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start },
+                tag: tag_layout,
+                tag_field,
+                ..
+            } => {
+                // No need to validate that the discriminant here because the
+                // `TyAndLayout::for_variant()` call earlier already checks the variant is valid.
+
+                if variant_index != untagged_variant {
+                    let variants_start = niche_variants.start().as_u32();
+                    let variant_index_relative = variant_index
+                        .as_u32()
+                        .checked_sub(variants_start)
+                        .expect("overflow computing relative variant idx");
+                    // We need to use machine arithmetic when taking into account `niche_start`:
+                    // tag_val = variant_index_relative + niche_start_val
+                    let tag_layout = self.layout_of(tag_layout.primitive().to_int_ty(*self.tcx))?;
+                    let niche_start_val = ImmTy::from_uint(niche_start, tag_layout);
+                    let variant_index_relative_val =
+                        ImmTy::from_uint(variant_index_relative, tag_layout);
+                    let tag_val = self.binary_op(
+                        mir::BinOp::Add,
+                        &variant_index_relative_val,
+                        &niche_start_val,
+                    )?;
+                    // Write result.
+                    let niche_dest = self.place_field(dest, tag_field)?;
+                    self.write_immediate(*tag_val, &niche_dest)?;
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Read discriminant, return the runtime value as well as the variant index.
+    /// Can also legally be called on non-enums (e.g. through the discriminant_value intrinsic)!
+    #[instrument(skip(self), level = "trace")]
+    pub fn read_discriminant(
+        &self,
+        op: &OpTy<'tcx, M::Provenance>,
+    ) -> InterpResult<'tcx, (Scalar<M::Provenance>, VariantIdx)> {
+        trace!("read_discriminant_value {:#?}", op.layout);
+        // Get type and layout of the discriminant.
+        let discr_layout = self.layout_of(op.layout.ty.discriminant_ty(*self.tcx))?;
+        trace!("discriminant type: {:?}", discr_layout.ty);
+
+        // We use "discriminant" to refer to the value associated with a particular enum variant.
+        // This is not to be confused with its "variant index", which is just determining its position in the
+        // declared list of variants -- they can differ with explicitly assigned discriminants.
+        // We use "tag" to refer to how the discriminant is encoded in memory, which can be either
+        // straight-forward (`TagEncoding::Direct`) or with a niche (`TagEncoding::Niche`).
+        let (tag_scalar_layout, tag_encoding, tag_field) = match op.layout.variants {
+            Variants::Single { index } => {
+                let discr = match op.layout.ty.discriminant_for_variant(*self.tcx, index) {
+                    Some(discr) => {
+                        // This type actually has discriminants.
+                        assert_eq!(discr.ty, discr_layout.ty);
+                        Scalar::from_uint(discr.val, discr_layout.size)
+                    }
+                    None => {
+                        // On a type without actual discriminants, variant is 0.
+                        assert_eq!(index.as_u32(), 0);
+                        Scalar::from_uint(index.as_u32(), discr_layout.size)
+                    }
+                };
+                return Ok((discr, index));
+            }
+            Variants::Multiple { tag, ref tag_encoding, tag_field, .. } => {
+                (tag, tag_encoding, tag_field)
+            }
+        };
+
+        // There are *three* layouts that come into play here:
+        // - The discriminant has a type for typechecking. This is `discr_layout`, and is used for
+        //   the `Scalar` we return.
+        // - The tag (encoded discriminant) has layout `tag_layout`. This is always an integer type,
+        //   and used to interpret the value we read from the tag field.
+        //   For the return value, a cast to `discr_layout` is performed.
+        // - The field storing the tag has a layout, which is very similar to `tag_layout` but
+        //   may be a pointer. This is `tag_val.layout`; we just use it for sanity checks.
+
+        // Get layout for tag.
+        let tag_layout = self.layout_of(tag_scalar_layout.primitive().to_int_ty(*self.tcx))?;
+
+        // Read tag and sanity-check `tag_layout`.
+        let tag_val = self.read_immediate(&self.operand_field(op, tag_field)?)?;
+        assert_eq!(tag_layout.size, tag_val.layout.size);
+        assert_eq!(tag_layout.abi.is_signed(), tag_val.layout.abi.is_signed());
+        trace!("tag value: {}", tag_val);
+
+        // Figure out which discriminant and variant this corresponds to.
+        Ok(match *tag_encoding {
+            TagEncoding::Direct => {
+                let scalar = tag_val.to_scalar();
+                // Generate a specific error if `tag_val` is not an integer.
+                // (`tag_bits` itself is only used for error messages below.)
+                let tag_bits = scalar
+                    .try_to_int()
+                    .map_err(|dbg_val| err_ub!(InvalidTag(dbg_val)))?
+                    .assert_bits(tag_layout.size);
+                // Cast bits from tag layout to discriminant layout.
+                // After the checks we did above, this cannot fail, as
+                // discriminants are int-like.
+                let discr_val =
+                    self.cast_from_int_like(scalar, tag_val.layout, discr_layout.ty).unwrap();
+                let discr_bits = discr_val.assert_bits(discr_layout.size);
+                // Convert discriminant to variant index, and catch invalid discriminants.
+                let index = match *op.layout.ty.kind() {
+                    ty::Adt(adt, _) => {
+                        adt.discriminants(*self.tcx).find(|(_, var)| var.val == discr_bits)
+                    }
+                    ty::Generator(def_id, substs, _) => {
+                        let substs = substs.as_generator();
+                        substs
+                            .discriminants(def_id, *self.tcx)
+                            .find(|(_, var)| var.val == discr_bits)
+                    }
+                    _ => span_bug!(self.cur_span(), "tagged layout for non-adt non-generator"),
+                }
+                .ok_or_else(|| err_ub!(InvalidTag(Scalar::from_uint(tag_bits, tag_layout.size))))?;
+                // Return the cast value, and the index.
+                (discr_val, index.0)
+            }
+            TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => {
+                let tag_val = tag_val.to_scalar();
+                // Compute the variant this niche value/"tag" corresponds to. With niche layout,
+                // discriminant (encoded in niche/tag) and variant index are the same.
+                let variants_start = niche_variants.start().as_u32();
+                let variants_end = niche_variants.end().as_u32();
+                let variant = match tag_val.try_to_int() {
+                    Err(dbg_val) => {
+                        // So this is a pointer then, and casting to an int failed.
+                        // Can only happen during CTFE.
+                        // The niche must be just 0, and the ptr not null, then we know this is
+                        // okay. Everything else, we conservatively reject.
+                        let ptr_valid = niche_start == 0
+                            && variants_start == variants_end
+                            && !self.scalar_may_be_null(tag_val)?;
+                        if !ptr_valid {
+                            throw_ub!(InvalidTag(dbg_val))
+                        }
+                        untagged_variant
+                    }
+                    Ok(tag_bits) => {
+                        let tag_bits = tag_bits.assert_bits(tag_layout.size);
+                        // We need to use machine arithmetic to get the relative variant idx:
+                        // variant_index_relative = tag_val - niche_start_val
+                        let tag_val = ImmTy::from_uint(tag_bits, tag_layout);
+                        let niche_start_val = ImmTy::from_uint(niche_start, tag_layout);
+                        let variant_index_relative_val =
+                            self.binary_op(mir::BinOp::Sub, &tag_val, &niche_start_val)?;
+                        let variant_index_relative =
+                            variant_index_relative_val.to_scalar().assert_bits(tag_val.layout.size);
+                        // Check if this is in the range that indicates an actual discriminant.
+                        if variant_index_relative <= u128::from(variants_end - variants_start) {
+                            let variant_index_relative = u32::try_from(variant_index_relative)
+                                .expect("we checked that this fits into a u32");
+                            // Then computing the absolute variant idx should not overflow any more.
+                            let variant_index = variants_start
+                                .checked_add(variant_index_relative)
+                                .expect("overflow computing absolute variant idx");
+                            let variants_len = op
+                                .layout
+                                .ty
+                                .ty_adt_def()
+                                .expect("tagged layout for non adt")
+                                .variants()
+                                .len();
+                            assert!(usize::try_from(variant_index).unwrap() < variants_len);
+                            VariantIdx::from_u32(variant_index)
+                        } else {
+                            untagged_variant
+                        }
+                    }
+                };
+                // Compute the size of the scalar we need to return.
+                // No need to cast, because the variant index directly serves as discriminant and is
+                // encoded in the tag.
+                (Scalar::from_uint(variant.as_u32(), discr_layout.size), variant)
+            }
+        })
+    }
+}
diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs
index d13fed7a9..3db102e48 100644
--- a/compiler/rustc_const_eval/src/interpret/eval_context.rs
+++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs
@@ -489,7 +489,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
     /// Call this on things you got out of the MIR (so it is as generic as the current
     /// stack frame), to bring it into the proper environment for this interpreter.
-    pub(super) fn subst_from_current_frame_and_normalize_erasing_regions<T: TypeFoldable<'tcx>>(
+    pub(super) fn subst_from_current_frame_and_normalize_erasing_regions<
+        T: TypeFoldable<TyCtxt<'tcx>>,
+    >(
         &self,
         value: T,
     ) -> Result<T, InterpError<'tcx>> {
@@ -498,7 +500,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
     /// Call this on things you got out of the MIR (so it is as generic as the provided
     /// stack frame), to bring it into the proper environment for this interpreter.
-    pub(super) fn subst_from_frame_and_normalize_erasing_regions<T: TypeFoldable<'tcx>>(
+    pub(super) fn subst_from_frame_and_normalize_erasing_regions<T: TypeFoldable<TyCtxt<'tcx>>>(
         &self,
         frame: &Frame<'mir, 'tcx, M::Provenance, M::FrameExtra>,
         value: T,
@@ -632,14 +634,14 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 }
                 Ok(Some((size, align)))
             }
-            ty::Dynamic(..) => {
+            ty::Dynamic(_, _, ty::Dyn) => {
                 let vtable = metadata.unwrap_meta().to_pointer(self)?;
                 // Read size and align from vtable (already checks size).
                 Ok(Some(self.get_vtable_size_and_align(vtable)?))
             }
 
             ty::Slice(_) | ty::Str => {
-                let len = metadata.unwrap_meta().to_machine_usize(self)?;
+                let len = metadata.unwrap_meta().to_target_usize(self)?;
                 let elem = layout.field(self, 0);
 
                 // Make sure the slice is not too big.
diff --git a/compiler/rustc_const_eval/src/interpret/intern.rs b/compiler/rustc_const_eval/src/interpret/intern.rs
index 54528b1db..b220d21f6 100644
--- a/compiler/rustc_const_eval/src/interpret/intern.rs
+++ b/compiler/rustc_const_eval/src/interpret/intern.rs
@@ -30,15 +30,15 @@ use super::{
 use crate::const_eval;
 
 pub trait CompileTimeMachine<'mir, 'tcx, T> = Machine<
-    'mir,
-    'tcx,
-    MemoryKind = T,
-    Provenance = AllocId,
-    ExtraFnVal = !,
-    FrameExtra = (),
-    AllocExtra = (),
-    MemoryMap = FxIndexMap<AllocId, (MemoryKind<T>, Allocation)>,
->;
+        'mir,
+        'tcx,
+        MemoryKind = T,
+        Provenance = AllocId,
+        ExtraFnVal = !,
+        FrameExtra = (),
+        AllocExtra = (),
+        MemoryMap = FxIndexMap<AllocId, (MemoryKind<T>, Allocation)>,
+    >;
 
 struct InternVisitor<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx, const_eval::MemoryKind>> {
     /// The ectx from which we intern.
@@ -135,7 +135,7 @@ fn intern_shallow<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx, const_eval:
     };
     // link the alloc id to the actual allocation
     leftover_allocations.extend(alloc.provenance().ptrs().iter().map(|&(_, alloc_id)| alloc_id));
-    let alloc = tcx.intern_const_alloc(alloc);
+    let alloc = tcx.mk_const_alloc(alloc);
     tcx.set_alloc_id_memory(alloc_id, alloc);
     None
 }
@@ -242,7 +242,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx, const_eval::Memory
             let mplace = self.ecx.ref_to_mplace(&value)?;
             assert_eq!(mplace.layout.ty, referenced_ty);
             // Handle trait object vtables.
-            if let ty::Dynamic(..) =
+            if let ty::Dynamic(_, _, ty::Dyn) =
                 tcx.struct_tail_erasing_lifetimes(referenced_ty, self.ecx.param_env).kind()
             {
                 let ptr = mplace.meta.unwrap_meta().to_pointer(&tcx)?;
@@ -437,7 +437,7 @@ pub fn intern_const_alloc_recursive<
                     alloc.mutability = Mutability::Not;
                 }
             }
-            let alloc = tcx.intern_const_alloc(alloc);
+            let alloc = tcx.mk_const_alloc(alloc);
             tcx.set_alloc_id_memory(alloc_id, alloc);
             for &(_, alloc_id) in alloc.inner().provenance().ptrs().iter() {
                 if leftover_allocations.insert(alloc_id) {
@@ -479,6 +479,6 @@ impl<'mir, 'tcx: 'mir, M: super::intern::CompileTimeMachine<'mir, 'tcx, !>>
         f(self, &dest.into())?;
         let mut alloc = self.memory.alloc_map.remove(&dest.ptr.provenance.unwrap()).unwrap().1;
         alloc.mutability = Mutability::Not;
-        Ok(self.tcx.intern_const_alloc(alloc))
+        Ok(self.tcx.mk_const_alloc(alloc))
     }
 }
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
index cc7b6c91b..a29cdade0 100644
--- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
@@ -11,7 +11,7 @@ use rustc_middle::mir::{
     BinOp, NonDivergingIntrinsic,
 };
 use rustc_middle::ty;
-use rustc_middle::ty::layout::LayoutOf as _;
+use rustc_middle::ty::layout::{LayoutOf as _, ValidityRequirement};
 use rustc_middle::ty::subst::SubstsRef;
 use rustc_middle::ty::{Ty, TyCtxt};
 use rustc_span::symbol::{sym, Symbol};
@@ -45,7 +45,7 @@ fn numeric_intrinsic<Prov>(name: Symbol, bits: u128, kind: Primitive) -> Scalar<
 pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ConstAllocation<'tcx> {
     let path = crate::util::type_name(tcx, ty);
     let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes());
-    tcx.intern_const_alloc(alloc)
+    tcx.mk_const_alloc(alloc)
 }
 
 /// The logic for all nullary intrinsics is implemented here. These intrinsics don't get evaluated
@@ -71,7 +71,7 @@ pub(crate) fn eval_nullary_intrinsic<'tcx>(
         sym::pref_align_of => {
             // Correctly handles non-monomorphic calls, so there is no need for ensure_monomorphic_enough.
             let layout = tcx.layout_of(param_env.and(tp_ty)).map_err(|e| err_inval!(Layout(e)))?;
-            ConstValue::from_machine_usize(layout.align.pref.bytes(), &tcx)
+            ConstValue::from_target_usize(layout.align.pref.bytes(), &tcx)
         }
         sym::type_id => {
             ensure_monomorphic_enough(tcx, tp_ty)?;
@@ -79,7 +79,7 @@ pub(crate) fn eval_nullary_intrinsic<'tcx>(
         }
         sym::variant_count => match tp_ty.kind() {
             // Correctly handles non-monomorphic calls, so there is no need for ensure_monomorphic_enough.
-            ty::Adt(adt, _) => ConstValue::from_machine_usize(adt.variants().len() as u64, &tcx),
+            ty::Adt(adt, _) => ConstValue::from_target_usize(adt.variants().len() as u64, &tcx),
             ty::Alias(..) | ty::Param(_) | ty::Placeholder(_) | ty::Infer(_) => {
                 throw_inval!(TooGeneric)
             }
@@ -101,9 +101,10 @@ pub(crate) fn eval_nullary_intrinsic<'tcx>(
             | ty::Closure(_, _)
             | ty::Generator(_, _, _)
             | ty::GeneratorWitness(_)
+            | ty::GeneratorWitnessMIR(_, _)
             | ty::Never
             | ty::Tuple(_)
-            | ty::Error(_) => ConstValue::from_machine_usize(0u64, &tcx),
+            | ty::Error(_) => ConstValue::from_target_usize(0u64, &tcx),
         },
         other => bug!("`{}` is not a zero arg intrinsic", other),
     })
@@ -155,7 +156,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                     _ => bug!(),
                 };
 
-                self.write_scalar(Scalar::from_machine_usize(result, self), dest)?;
+                self.write_scalar(Scalar::from_target_usize(result, self), dest)?;
             }
 
             sym::pref_align_of
@@ -209,19 +210,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 let out_val = numeric_intrinsic(intrinsic_name, bits, kind);
                 self.write_scalar(out_val, dest)?;
             }
-            sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
-                let lhs = self.read_immediate(&args[0])?;
-                let rhs = self.read_immediate(&args[1])?;
-                let bin_op = match intrinsic_name {
-                    sym::add_with_overflow => BinOp::Add,
-                    sym::sub_with_overflow => BinOp::Sub,
-                    sym::mul_with_overflow => BinOp::Mul,
-                    _ => bug!(),
-                };
-                self.binop_with_overflow(
-                    bin_op, /*force_overflow_checks*/ true, &lhs, &rhs, dest,
-                )?;
-            }
             sym::saturating_add | sym::saturating_sub => {
                 let l = self.read_immediate(&args[0])?;
                 let r = self.read_immediate(&args[1])?;
@@ -301,7 +289,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             }
             sym::offset => {
                 let ptr = self.read_pointer(&args[0])?;
-                let offset_count = self.read_machine_isize(&args[1])?;
+                let offset_count = self.read_target_isize(&args[1])?;
                 let pointee_ty = substs.type_at(0);
 
                 let offset_ptr = self.ptr_offset_inbounds(ptr, pointee_ty, offset_count)?;
@@ -309,7 +297,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             }
             sym::arith_offset => {
                 let ptr = self.read_pointer(&args[0])?;
-                let offset_count = self.read_machine_isize(&args[1])?;
+                let offset_count = self.read_target_isize(&args[1])?;
                 let pointee_ty = substs.type_at(0);
 
                 let pointee_size = i64::try_from(self.layout_of(pointee_ty)?.size.bytes()).unwrap();
@@ -375,7 +363,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                         // The signed form of the intrinsic allows this. If we interpret the
                         // difference as isize, we'll get the proper signed difference. If that
                         // seems *positive*, they were more than isize::MAX apart.
-                        let dist = val.to_machine_isize(self)?;
+                        let dist = val.to_target_isize(self)?;
                         if dist >= 0 {
                             throw_ub_format!(
                                 "`{}` called when first pointer is too far before second",
@@ -385,7 +373,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                         dist
                     } else {
                         // b >= a
-                        let dist = val.to_machine_isize(self)?;
+                        let dist = val.to_target_isize(self)?;
                         // If converting to isize produced a *negative* result, we had an overflow
                         // because they were more than isize::MAX apart.
                         if dist < 0 {
@@ -410,10 +398,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
                 // Perform division by size to compute return value.
                 let ret_layout = if intrinsic_name == sym::ptr_offset_from_unsigned {
-                    assert!(0 <= dist && dist <= self.machine_isize_max());
+                    assert!(0 <= dist && dist <= self.target_isize_max());
                     usize_layout
                 } else {
-                    assert!(self.machine_isize_min() <= dist && dist <= self.machine_isize_max());
+                    assert!(self.target_isize_min() <= dist && dist <= self.target_isize_max());
                     isize_layout
                 };
                 let pointee_layout = self.layout_of(substs.type_at(0))?;
@@ -430,48 +418,36 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             | sym::assert_zero_valid
             | sym::assert_mem_uninitialized_valid => {
                 let ty = instance.substs.type_at(0);
-                let layout = self.layout_of(ty)?;
-
-                // For *all* intrinsics we first check `is_uninhabited` to give a more specific
-                // error message.
-                if layout.abi.is_uninhabited() {
-                    // The run-time intrinsic panics just to get a good backtrace; here we abort
-                    // since there is no problem showing a backtrace even for aborts.
-                    M::abort(
-                        self,
-                        format!(
+                let requirement = ValidityRequirement::from_intrinsic(intrinsic_name).unwrap();
+
+                let should_panic = !self
+                    .tcx
+                    .check_validity_requirement((requirement, self.param_env.and(ty)))
+                    .map_err(|_| err_inval!(TooGeneric))?;
+
+                if should_panic {
+                    let layout = self.layout_of(ty)?;
+
+                    let msg = match requirement {
+                        // For *all* intrinsics we first check `is_uninhabited` to give a more specific
+                        // error message.
+                        _ if layout.abi.is_uninhabited() => format!(
                             "aborted execution: attempted to instantiate uninhabited type `{}`",
                             ty
                         ),
-                    )?;
-                }
-
-                if intrinsic_name == sym::assert_zero_valid {
-                    let should_panic = !self.tcx.permits_zero_init(layout);
-
-                    if should_panic {
-                        M::abort(
-                            self,
-                            format!(
-                                "aborted execution: attempted to zero-initialize type `{}`, which is invalid",
-                                ty
-                            ),
-                        )?;
-                    }
-                }
+                        ValidityRequirement::Inhabited => bug!("handled earlier"),
+                        ValidityRequirement::Zero => format!(
+                            "aborted execution: attempted to zero-initialize type `{}`, which is invalid",
+                            ty
+                        ),
+                        ValidityRequirement::UninitMitigated0x01Fill => format!(
+                            "aborted execution: attempted to leave type `{}` uninitialized, which is invalid",
+                            ty
+                        ),
+                        ValidityRequirement::Uninit => bug!("assert_uninit_valid doesn't exist"),
+                    };
 
-                if intrinsic_name == sym::assert_mem_uninitialized_valid {
-                    let should_panic = !self.tcx.permits_uninit_init(layout);
-
-                    if should_panic {
-                        M::abort(
-                            self,
-                            format!(
-                                "aborted execution: attempted to leave type `{}` uninitialized, which is invalid",
-                                ty
-                            ),
-                        )?;
-                    }
+                    M::abort(self, msg)?;
                 }
             }
             sym::simd_insert => {
@@ -482,7 +458,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 assert_eq!(input_len, dest_len, "Return vector length must match input length");
                 assert!(
                     index < dest_len,
-                    "Index `{}` must be in bounds of vector with length {}`",
+                    "Index `{}` must be in bounds of vector with length {}",
                     index,
                     dest_len
                 );
@@ -502,7 +478,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 let (input, input_len) = self.operand_to_simd(&args[0])?;
                 assert!(
                     index < input_len,
-                    "index `{}` must be in bounds of vector with length `{}`",
+                    "index `{}` must be in bounds of vector with length {}",
                     index,
                     input_len
                 );
@@ -524,12 +500,12 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             sym::vtable_size => {
                 let ptr = self.read_pointer(&args[0])?;
                 let (size, _align) = self.get_vtable_size_and_align(ptr)?;
-                self.write_scalar(Scalar::from_machine_usize(size.bytes(), self), dest)?;
+                self.write_scalar(Scalar::from_target_usize(size.bytes(), self), dest)?;
             }
             sym::vtable_align => {
                 let ptr = self.read_pointer(&args[0])?;
                 let (_size, align) = self.get_vtable_size_and_align(ptr)?;
-                self.write_scalar(Scalar::from_machine_usize(align.bytes(), self), dest)?;
+                self.write_scalar(Scalar::from_target_usize(align.bytes(), self), dest)?;
             }
 
             _ => return Ok(false),
@@ -668,10 +644,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         count: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::Provenance>,
         nonoverlapping: bool,
     ) -> InterpResult<'tcx> {
-        let count = self.read_machine_usize(&count)?;
+        let count = self.read_target_usize(&count)?;
         let layout = self.layout_of(src.layout.ty.builtin_deref(true).unwrap().ty)?;
         let (size, align) = (layout.size, layout.align.abi);
-        // `checked_mul` enforces a too small bound (the correct one would probably be machine_isize_max),
+        // `checked_mul` enforces a too small bound (the correct one would probably be target_isize_max),
         // but no actual allocation can be big enough for the difference to be noticeable.
         let size = size.checked_mul(count, self).ok_or_else(|| {
             err_ub_format!(
@@ -696,9 +672,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
         let dst = self.read_pointer(&dst)?;
         let byte = self.read_scalar(&byte)?.to_u8()?;
-        let count = self.read_machine_usize(&count)?;
+        let count = self.read_target_usize(&count)?;
 
-        // `checked_mul` enforces a too small bound (the correct one would probably be machine_isize_max),
+        // `checked_mul` enforces a too small bound (the correct one would probably be target_isize_max),
         // but no actual allocation can be big enough for the difference to be noticeable.
         let len = layout
             .size
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs
index 77c7b4bac..cf52299b7 100644
--- a/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs
@@ -78,13 +78,16 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         col: u32,
     ) -> MPlaceTy<'tcx, M::Provenance> {
         let loc_details = &self.tcx.sess.opts.unstable_opts.location_detail;
+        // This can fail if rustc runs out of memory right here. Trying to emit an error would be
+        // pointless, since that would require allocating more memory than these short strings.
         let file = if loc_details.file {
             self.allocate_str(filename.as_str(), MemoryKind::CallerLocation, Mutability::Not)
+                .unwrap()
         } else {
             // FIXME: This creates a new allocation each time. It might be preferable to
             // perform this allocation only once, and re-use the `MPlaceTy`.
             // See https://github.com/rust-lang/rust/pull/89920#discussion_r730012398
-            self.allocate_str("<redacted>", MemoryKind::CallerLocation, Mutability::Not)
+            self.allocate_str("<redacted>", MemoryKind::CallerLocation, Mutability::Not).unwrap()
         };
         let line = if loc_details.line { Scalar::from_u32(line) } else { Scalar::from_u32(0) };
         let col = if loc_details.column { Scalar::from_u32(col) } else { Scalar::from_u32(0) };
@@ -92,11 +95,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         // Allocate memory for `CallerLocation` struct.
         let loc_ty = self
             .tcx
-            .bound_type_of(self.tcx.require_lang_item(LangItem::PanicLocation, None))
-            .subst(*self.tcx, self.tcx.mk_substs([self.tcx.lifetimes.re_erased.into()].iter()));
+            .type_of(self.tcx.require_lang_item(LangItem::PanicLocation, None))
+            .subst(*self.tcx, self.tcx.mk_substs(&[self.tcx.lifetimes.re_erased.into()]));
         let loc_layout = self.layout_of(loc_ty).unwrap();
-        // This can fail if rustc runs out of memory right here. Trying to emit an error would be
-        // pointless, since that would require allocating more memory than a Location.
         let location = self.allocate(loc_layout, MemoryKind::CallerLocation).unwrap();
 
         // Initialize fields.
diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs
index 248953de8..92fa59aec 100644
--- a/compiler/rustc_const_eval/src/interpret/machine.rs
+++ b/compiler/rustc_const_eval/src/interpret/machine.rs
@@ -16,8 +16,8 @@ use rustc_target::spec::abi::Abi as CallAbi;
 use crate::const_eval::CheckAlignment;
 
 use super::{
-    AllocId, AllocRange, Allocation, ConstAllocation, Frame, ImmTy, InterpCx, InterpResult,
-    MemoryKind, OpTy, Operand, PlaceTy, Pointer, Provenance, Scalar, StackPopUnwind,
+    AllocBytes, AllocId, AllocRange, Allocation, ConstAllocation, Frame, ImmTy, InterpCx,
+    InterpResult, MemoryKind, OpTy, Operand, PlaceTy, Pointer, Provenance, Scalar, StackPopUnwind,
 };
 
 /// Data returned by Machine::stack_pop,
@@ -105,10 +105,16 @@ pub trait Machine<'mir, 'tcx>: Sized {
     /// Extra data stored in every allocation.
     type AllocExtra: Debug + Clone + 'static;
 
+    /// Type for the bytes of the allocation.
+    type Bytes: AllocBytes + 'static;
+
     /// Memory's allocation map
     type MemoryMap: AllocMap<
             AllocId,
-            (MemoryKind<Self::MemoryKind>, Allocation<Self::Provenance, Self::AllocExtra>),
+            (
+                MemoryKind<Self::MemoryKind>,
+                Allocation<Self::Provenance, Self::AllocExtra, Self::Bytes>,
+            ),
         > + Default
         + Clone;
 
@@ -147,8 +153,9 @@ pub trait Machine<'mir, 'tcx>: Sized {
         true
     }
 
-    /// Whether CheckedBinOp MIR statements should actually check for overflow.
-    fn checked_binop_checks_overflow(_ecx: &InterpCx<'mir, 'tcx, Self>) -> bool;
+    /// Whether Assert(OverflowNeg) and Assert(Overflow) MIR terminators should actually
+    /// check for overflow.
+    fn ignore_checkable_overflow_assertions(_ecx: &InterpCx<'mir, 'tcx, Self>) -> bool;
 
     /// Entry point for obtaining the MIR of anything that should get evaluated.
     /// So not just functions and shims, but also const/static initializers, anonymous
@@ -244,12 +251,18 @@ pub trait Machine<'mir, 'tcx>: Sized {
     }
 
     /// Called before a basic block terminator is executed.
-    /// You can use this to detect endlessly running programs.
     #[inline]
     fn before_terminator(_ecx: &mut InterpCx<'mir, 'tcx, Self>) -> InterpResult<'tcx> {
         Ok(())
     }
 
+    /// Called when the interpreter encounters a `StatementKind::ConstEvalCounter` instruction.
+    /// You can use this to detect long or endlessly running programs.
+    #[inline]
+    fn increment_const_eval_counter(_ecx: &mut InterpCx<'mir, 'tcx, Self>) -> InterpResult<'tcx> {
+        Ok(())
+    }
+
     /// Called before a global allocation is accessed.
     /// `def_id` is `Some` if this is the "lazy" allocation of a static.
     #[inline]
@@ -285,7 +298,7 @@ pub trait Machine<'mir, 'tcx>: Sized {
     fn adjust_alloc_base_pointer(
         ecx: &InterpCx<'mir, 'tcx, Self>,
         ptr: Pointer,
-    ) -> Pointer<Self::Provenance>;
+    ) -> InterpResult<'tcx, Pointer<Self::Provenance>>;
 
     /// "Int-to-pointer cast"
     fn ptr_from_addr_cast(
@@ -331,7 +344,7 @@ pub trait Machine<'mir, 'tcx>: Sized {
         id: AllocId,
         alloc: Cow<'b, Allocation>,
         kind: Option<MemoryKind<Self::MemoryKind>>,
-    ) -> InterpResult<'tcx, Cow<'b, Allocation<Self::Provenance, Self::AllocExtra>>>;
+    ) -> InterpResult<'tcx, Cow<'b, Allocation<Self::Provenance, Self::AllocExtra, Self::Bytes>>>;
 
     fn eval_inline_asm(
         _ecx: &mut InterpCx<'mir, 'tcx, Self>,
@@ -452,6 +465,7 @@ pub macro compile_time_machine(<$mir: lifetime, $tcx: lifetime>) {
 
     type AllocExtra = ();
     type FrameExtra = ();
+    type Bytes = Box<[u8]>;
 
     #[inline(always)]
     fn use_addr_for_alignment_check(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool {
@@ -460,8 +474,8 @@ pub macro compile_time_machine(<$mir: lifetime, $tcx: lifetime>) {
     }
 
     #[inline(always)]
-    fn checked_binop_checks_overflow(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool {
-        true
+    fn ignore_checkable_overflow_assertions(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool {
+        false
     }
 
     #[inline(always)]
@@ -499,8 +513,8 @@ pub macro compile_time_machine(<$mir: lifetime, $tcx: lifetime>) {
     fn adjust_alloc_base_pointer(
         _ecx: &InterpCx<$mir, $tcx, Self>,
         ptr: Pointer<AllocId>,
-    ) -> Pointer<AllocId> {
-        ptr
+    ) -> InterpResult<$tcx, Pointer<AllocId>> {
+        Ok(ptr)
     }
 
     #[inline(always)]
@@ -511,7 +525,7 @@ pub macro compile_time_machine(<$mir: lifetime, $tcx: lifetime>) {
         // Allow these casts, but make the pointer not dereferenceable.
         // (I.e., they behave like transmutation.)
         // This is correct because no pointers can ever be exposed in compile-time evaluation.
-        Ok(Pointer::from_addr(addr))
+        Ok(Pointer::from_addr_invalid(addr))
     }
 
     #[inline(always)]
diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs
index 291bfb2b5..a3764a7d1 100644
--- a/compiler/rustc_const_eval/src/interpret/memory.rs
+++ b/compiler/rustc_const_eval/src/interpret/memory.rs
@@ -21,8 +21,9 @@ use rustc_target::abi::{Align, HasDataLayout, Size};
 use crate::const_eval::CheckAlignment;
 
 use super::{
-    alloc_range, AllocId, AllocMap, AllocRange, Allocation, CheckInAllocMsg, GlobalAlloc, InterpCx,
-    InterpResult, Machine, MayLeak, Pointer, PointerArithmetic, Provenance, Scalar,
+    alloc_range, AllocBytes, AllocId, AllocMap, AllocRange, Allocation, CheckInAllocMsg,
+    GlobalAlloc, InterpCx, InterpResult, Machine, MayLeak, Pointer, PointerArithmetic, Provenance,
+    Scalar,
 };
 
 #[derive(Debug, PartialEq, Copy, Clone)]
@@ -114,16 +115,16 @@ pub struct Memory<'mir, 'tcx, M: Machine<'mir, 'tcx>> {
 /// A reference to some allocation that was already bounds-checked for the given region
 /// and had the on-access machine hooks run.
 #[derive(Copy, Clone)]
-pub struct AllocRef<'a, 'tcx, Prov: Provenance, Extra> {
-    alloc: &'a Allocation<Prov, Extra>,
+pub struct AllocRef<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes = Box<[u8]>> {
+    alloc: &'a Allocation<Prov, Extra, Bytes>,
     range: AllocRange,
     tcx: TyCtxt<'tcx>,
     alloc_id: AllocId,
 }
 /// A reference to some allocation that was already bounds-checked for the given region
 /// and had the on-access machine hooks run.
-pub struct AllocRefMut<'a, 'tcx, Prov: Provenance, Extra> {
-    alloc: &'a mut Allocation<Prov, Extra>,
+pub struct AllocRefMut<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes = Box<[u8]>> {
+    alloc: &'a mut Allocation<Prov, Extra, Bytes>,
     range: AllocRange,
     tcx: TyCtxt<'tcx>,
     alloc_id: AllocId,
@@ -171,7 +172,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             _ => {}
         }
         // And we need to get the provenance.
-        Ok(M::adjust_alloc_base_pointer(self, ptr))
+        M::adjust_alloc_base_pointer(self, ptr)
     }
 
     pub fn create_fn_alloc_ptr(
@@ -200,8 +201,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         kind: MemoryKind<M::MemoryKind>,
     ) -> InterpResult<'tcx, Pointer<M::Provenance>> {
         let alloc = Allocation::uninit(size, align, M::PANIC_ON_ALLOC_FAIL)?;
-        // We can `unwrap` since `alloc` contains no pointers.
-        Ok(self.allocate_raw_ptr(alloc, kind).unwrap())
+        self.allocate_raw_ptr(alloc, kind)
     }
 
     pub fn allocate_bytes_ptr(
@@ -210,10 +210,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         align: Align,
         kind: MemoryKind<M::MemoryKind>,
         mutability: Mutability,
-    ) -> Pointer<M::Provenance> {
+    ) -> InterpResult<'tcx, Pointer<M::Provenance>> {
         let alloc = Allocation::from_bytes(bytes, align, mutability);
-        // We can `unwrap` since `alloc` contains no pointers.
-        self.allocate_raw_ptr(alloc, kind).unwrap()
+        self.allocate_raw_ptr(alloc, kind)
     }
 
     /// This can fail only of `alloc` contains provenance.
@@ -230,7 +229,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         );
         let alloc = M::adjust_allocation(self, id, Cow::Owned(alloc), Some(kind))?;
         self.memory.alloc_map.insert(id, (kind, alloc.into_owned()));
-        Ok(M::adjust_alloc_base_pointer(self, Pointer::from(id)))
+        M::adjust_alloc_base_pointer(self, Pointer::from(id))
     }
 
     pub fn reallocate_ptr(
@@ -304,7 +303,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             .into());
         };
 
-        if alloc.mutability == Mutability::Not {
+        if alloc.mutability.is_not() {
             throw_ub_format!("deallocating immutable allocation {alloc_id:?}");
         }
         if alloc_kind != kind {
@@ -427,7 +426,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                     throw_ub!(PointerOutOfBounds {
                         alloc_id,
                         alloc_size,
-                        ptr_offset: self.machine_usize_to_isize(offset.bytes()),
+                        ptr_offset: self.target_usize_to_isize(offset.bytes()),
                         ptr_size: size,
                         msg,
                     })
@@ -485,7 +484,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         &self,
         id: AllocId,
         is_write: bool,
-    ) -> InterpResult<'tcx, Cow<'tcx, Allocation<M::Provenance, M::AllocExtra>>> {
+    ) -> InterpResult<'tcx, Cow<'tcx, Allocation<M::Provenance, M::AllocExtra, M::Bytes>>> {
         let (alloc, def_id) = match self.tcx.try_get_global_alloc(id) {
             Some(GlobalAlloc::Memory(mem)) => {
                 // Memory of a constant or promoted or anonymous memory referenced by a static.
@@ -528,6 +527,17 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         )
     }
 
+    /// Get the base address for the bytes in an `Allocation` specified by the
+    /// `AllocID` passed in; error if no such allocation exists.
+    ///
+    /// It is up to the caller to take sufficient care when using this address:
+    /// there could be provenance or uninit memory in there, and other memory
+    /// accesses could invalidate the exposed pointer.
+    pub fn alloc_base_addr(&self, id: AllocId) -> InterpResult<'tcx, *const u8> {
+        let alloc = self.get_alloc_raw(id)?;
+        Ok(alloc.base_addr())
+    }
+
     /// Gives raw access to the `Allocation`, without bounds or alignment checks.
     /// The caller is responsible for calling the access hooks!
     ///
@@ -535,7 +545,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     fn get_alloc_raw(
         &self,
         id: AllocId,
-    ) -> InterpResult<'tcx, &Allocation<M::Provenance, M::AllocExtra>> {
+    ) -> InterpResult<'tcx, &Allocation<M::Provenance, M::AllocExtra, M::Bytes>> {
         // The error type of the inner closure here is somewhat funny. We have two
         // ways of "erroring": An actual error, or because we got a reference from
         // `get_global_alloc` that we can actually use directly without inserting anything anywhere.
@@ -571,7 +581,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         ptr: Pointer<Option<M::Provenance>>,
         size: Size,
         align: Align,
-    ) -> InterpResult<'tcx, Option<AllocRef<'a, 'tcx, M::Provenance, M::AllocExtra>>> {
+    ) -> InterpResult<'tcx, Option<AllocRef<'a, 'tcx, M::Provenance, M::AllocExtra, M::Bytes>>>
+    {
         let ptr_and_alloc = self.check_and_deref_ptr(
             ptr,
             size,
@@ -614,7 +625,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     fn get_alloc_raw_mut(
         &mut self,
         id: AllocId,
-    ) -> InterpResult<'tcx, (&mut Allocation<M::Provenance, M::AllocExtra>, &mut M)> {
+    ) -> InterpResult<'tcx, (&mut Allocation<M::Provenance, M::AllocExtra, M::Bytes>, &mut M)> {
         // We have "NLL problem case #3" here, which cannot be worked around without loss of
         // efficiency even for the common case where the key is in the map.
         // <https://rust-lang.github.io/rfcs/2094-nll.html#problem-case-3-conditional-control-flow-across-functions>
@@ -631,7 +642,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         }
 
         let (_kind, alloc) = self.memory.alloc_map.get_mut(id).unwrap();
-        if alloc.mutability == Mutability::Not {
+        if alloc.mutability.is_not() {
             throw_ub!(WriteToReadOnly(id))
         }
         Ok((alloc, &mut self.machine))
@@ -643,7 +654,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         ptr: Pointer<Option<M::Provenance>>,
         size: Size,
         align: Align,
-    ) -> InterpResult<'tcx, Option<AllocRefMut<'a, 'tcx, M::Provenance, M::AllocExtra>>> {
+    ) -> InterpResult<'tcx, Option<AllocRefMut<'a, 'tcx, M::Provenance, M::AllocExtra, M::Bytes>>>
+    {
         let parts = self.get_ptr_access(ptr, size, align)?;
         if let Some((alloc_id, offset, prov)) = parts {
             let tcx = *self.tcx;
@@ -692,7 +704,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 assert!(self.tcx.is_static(def_id));
                 assert!(!self.tcx.is_thread_local_static(def_id));
                 // Use size and align of the type.
-                let ty = self.tcx.type_of(def_id);
+                let ty = self
+                    .tcx
+                    .type_of(def_id)
+                    .no_bound_vars()
+                    .expect("statics should not have generic parameters");
                 let layout = self.tcx.layout_of(ParamEnv::empty().and(ty)).unwrap();
                 assert!(layout.is_sized());
                 (layout.size, layout.align.abi, AllocKind::LiveData)
@@ -838,11 +854,11 @@ pub struct DumpAllocs<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> {
 impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> std::fmt::Debug for DumpAllocs<'a, 'mir, 'tcx, M> {
     fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         // Cannot be a closure because it is generic in `Prov`, `Extra`.
-        fn write_allocation_track_relocs<'tcx, Prov: Provenance, Extra>(
+        fn write_allocation_track_relocs<'tcx, Prov: Provenance, Extra, Bytes: AllocBytes>(
             fmt: &mut std::fmt::Formatter<'_>,
             tcx: TyCtxt<'tcx>,
             allocs_to_print: &mut VecDeque<AllocId>,
-            alloc: &Allocation<Prov, Extra>,
+            alloc: &Allocation<Prov, Extra, Bytes>,
         ) -> std::fmt::Result {
             for alloc_id in alloc.provenance().provenances().filter_map(|prov| prov.get_alloc_id())
             {
@@ -910,7 +926,9 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> std::fmt::Debug for DumpAllocs<'a,
 }
 
 /// Reading and writing.
-impl<'tcx, 'a, Prov: Provenance, Extra> AllocRefMut<'a, 'tcx, Prov, Extra> {
+impl<'tcx, 'a, Prov: Provenance, Extra, Bytes: AllocBytes>
+    AllocRefMut<'a, 'tcx, Prov, Extra, Bytes>
+{
     /// `range` is relative to this allocation reference, not the base of the allocation.
     pub fn write_scalar(&mut self, range: AllocRange, val: Scalar<Prov>) -> InterpResult<'tcx> {
         let range = self.range.subrange(range);
@@ -935,7 +953,7 @@ impl<'tcx, 'a, Prov: Provenance, Extra> AllocRefMut<'a, 'tcx, Prov, Extra> {
     }
 }
 
-impl<'tcx, 'a, Prov: Provenance, Extra> AllocRef<'a, 'tcx, Prov, Extra> {
+impl<'tcx, 'a, Prov: Provenance, Extra, Bytes: AllocBytes> AllocRef<'a, 'tcx, Prov, Extra, Bytes> {
     /// `range` is relative to this allocation reference, not the base of the allocation.
     pub fn read_scalar(
         &self,
diff --git a/compiler/rustc_const_eval/src/interpret/mod.rs b/compiler/rustc_const_eval/src/interpret/mod.rs
index 2e356f67b..86de4e4e3 100644
--- a/compiler/rustc_const_eval/src/interpret/mod.rs
+++ b/compiler/rustc_const_eval/src/interpret/mod.rs
@@ -1,6 +1,7 @@
 //! An interpreter for MIR used in CTFE and by miri
 
 mod cast;
+mod discriminant;
 mod eval_context;
 mod intern;
 mod intrinsics;
diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs
index befc0928f..5310ef0bb 100644
--- a/compiler/rustc_const_eval/src/interpret/operand.rs
+++ b/compiler/rustc_const_eval/src/interpret/operand.rs
@@ -4,13 +4,12 @@
 use either::{Either, Left, Right};
 
 use rustc_hir::def::Namespace;
-use rustc_middle::ty::layout::{LayoutOf, PrimitiveExt, TyAndLayout};
+use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
 use rustc_middle::ty::print::{FmtPrinter, PrettyPrinter};
 use rustc_middle::ty::{ConstInt, Ty, ValTree};
 use rustc_middle::{mir, ty};
 use rustc_span::Span;
-use rustc_target::abi::{self, Abi, Align, HasDataLayout, Size, TagEncoding};
-use rustc_target::abi::{VariantIdx, Variants};
+use rustc_target::abi::{self, Abi, Align, HasDataLayout, Size};
 
 use super::{
     alloc_range, from_known_layout, mir_assign_valid_types, AllocId, ConstValue, Frame, GlobalId,
@@ -53,7 +52,7 @@ impl<Prov: Provenance> Immediate<Prov> {
     }
 
     pub fn new_slice(val: Scalar<Prov>, len: u64, cx: &impl HasDataLayout) -> Self {
-        Immediate::ScalarPair(val, Scalar::from_machine_usize(len, cx))
+        Immediate::ScalarPair(val, Scalar::from_target_usize(len, cx))
     }
 
     pub fn new_dyn_trait(
@@ -256,7 +255,22 @@ impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
         }
     }
 
-    pub fn offset_with_meta(
+    /// Replace the layout of this operand. There's basically no sanity check that this makes sense,
+    /// you better know what you are doing! If this is an immediate, applying the wrong layout can
+    /// not just lead to invalid data, it can actually *shift the data around* since the offsets of
+    /// a ScalarPair are entirely determined by the layout, not the data.
+    pub fn transmute(&self, layout: TyAndLayout<'tcx>) -> Self {
+        assert_eq!(
+            self.layout.size, layout.size,
+            "transmuting with a size change, that doesn't seem right"
+        );
+        OpTy { layout, ..*self }
+    }
+
+    /// Offset the operand in memory (if possible) and change its metadata.
+    ///
+    /// This can go wrong very easily if you give the wrong layout for the new place!
+    pub(super) fn offset_with_meta(
         &self,
         offset: Size,
         meta: MemPlaceMeta<Prov>,
@@ -277,6 +291,9 @@ impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
         }
     }
 
+    /// Offset the operand in memory (if possible).
+    ///
+    /// This can go wrong very easily if you give the wrong layout for the new place!
     pub fn offset(
         &self,
         offset: Size,
@@ -319,7 +336,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 assert_eq!(size, mplace.layout.size, "abi::Scalar size does not match layout size");
                 let scalar = alloc.read_scalar(
                     alloc_range(Size::ZERO, size),
-                    /*read_provenance*/ s.is_ptr(),
+                    /*read_provenance*/ matches!(s, abi::Pointer(_)),
                 )?;
                 Some(ImmTy { imm: scalar.into(), layout: mplace.layout })
             }
@@ -335,11 +352,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 assert!(b_offset.bytes() > 0); // in `operand_field` we use the offset to tell apart the fields
                 let a_val = alloc.read_scalar(
                     alloc_range(Size::ZERO, a_size),
-                    /*read_provenance*/ a.is_ptr(),
+                    /*read_provenance*/ matches!(a, abi::Pointer(_)),
                 )?;
                 let b_val = alloc.read_scalar(
                     alloc_range(b_offset, b_size),
-                    /*read_provenance*/ b.is_ptr(),
+                    /*read_provenance*/ matches!(b, abi::Pointer(_)),
                 )?;
                 Some(ImmTy { imm: Immediate::ScalarPair(a_val, b_val), layout: mplace.layout })
             }
@@ -415,12 +432,12 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         self.read_scalar(op)?.to_pointer(self)
     }
     /// Read a pointer-sized unsigned integer from a place.
-    pub fn read_machine_usize(&self, op: &OpTy<'tcx, M::Provenance>) -> InterpResult<'tcx, u64> {
-        self.read_scalar(op)?.to_machine_usize(self)
+    pub fn read_target_usize(&self, op: &OpTy<'tcx, M::Provenance>) -> InterpResult<'tcx, u64> {
+        self.read_scalar(op)?.to_target_usize(self)
     }
     /// Read a pointer-sized signed integer from a place.
-    pub fn read_machine_isize(&self, op: &OpTy<'tcx, M::Provenance>) -> InterpResult<'tcx, i64> {
-        self.read_scalar(op)?.to_machine_isize(self)
+    pub fn read_target_isize(&self, op: &OpTy<'tcx, M::Provenance>) -> InterpResult<'tcx, i64> {
+        self.read_scalar(op)?.to_target_isize(self)
     }
 
     /// Turn the wide MPlace into a string (must already be dereferenced!)
@@ -595,14 +612,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         span: Option<Span>,
         layout: Option<TyAndLayout<'tcx>>,
     ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
-        // FIXME(const_prop): normalization needed b/c const prop lint in
-        // `mir_drops_elaborated_and_const_checked`, which happens before
-        // optimized MIR. Only after optimizing the MIR can we guarantee
-        // that the `RevealAll` pass has happened and that the body's consts
-        // are normalized, so any call to resolve before that needs to be
-        // manually normalized.
-        let val = self.tcx.normalize_erasing_regions(self.param_env, *val);
-        match val {
+        match *val {
             mir::ConstantKind::Ty(ct) => {
                 let ty = ct.ty();
                 let valtree = self.eval_ty_constant(ct, span)?;
@@ -657,154 +667,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         };
         Ok(OpTy { op, layout, align: Some(layout.align.abi) })
     }
-
-    /// Read discriminant, return the runtime value as well as the variant index.
-    /// Can also legally be called on non-enums (e.g. through the discriminant_value intrinsic)!
-    pub fn read_discriminant(
-        &self,
-        op: &OpTy<'tcx, M::Provenance>,
-    ) -> InterpResult<'tcx, (Scalar<M::Provenance>, VariantIdx)> {
-        trace!("read_discriminant_value {:#?}", op.layout);
-        // Get type and layout of the discriminant.
-        let discr_layout = self.layout_of(op.layout.ty.discriminant_ty(*self.tcx))?;
-        trace!("discriminant type: {:?}", discr_layout.ty);
-
-        // We use "discriminant" to refer to the value associated with a particular enum variant.
-        // This is not to be confused with its "variant index", which is just determining its position in the
-        // declared list of variants -- they can differ with explicitly assigned discriminants.
-        // We use "tag" to refer to how the discriminant is encoded in memory, which can be either
-        // straight-forward (`TagEncoding::Direct`) or with a niche (`TagEncoding::Niche`).
-        let (tag_scalar_layout, tag_encoding, tag_field) = match op.layout.variants {
-            Variants::Single { index } => {
-                let discr = match op.layout.ty.discriminant_for_variant(*self.tcx, index) {
-                    Some(discr) => {
-                        // This type actually has discriminants.
-                        assert_eq!(discr.ty, discr_layout.ty);
-                        Scalar::from_uint(discr.val, discr_layout.size)
-                    }
-                    None => {
-                        // On a type without actual discriminants, variant is 0.
-                        assert_eq!(index.as_u32(), 0);
-                        Scalar::from_uint(index.as_u32(), discr_layout.size)
-                    }
-                };
-                return Ok((discr, index));
-            }
-            Variants::Multiple { tag, ref tag_encoding, tag_field, .. } => {
-                (tag, tag_encoding, tag_field)
-            }
-        };
-
-        // There are *three* layouts that come into play here:
-        // - The discriminant has a type for typechecking. This is `discr_layout`, and is used for
-        //   the `Scalar` we return.
-        // - The tag (encoded discriminant) has layout `tag_layout`. This is always an integer type,
-        //   and used to interpret the value we read from the tag field.
-        //   For the return value, a cast to `discr_layout` is performed.
-        // - The field storing the tag has a layout, which is very similar to `tag_layout` but
-        //   may be a pointer. This is `tag_val.layout`; we just use it for sanity checks.
-
-        // Get layout for tag.
-        let tag_layout = self.layout_of(tag_scalar_layout.primitive().to_int_ty(*self.tcx))?;
-
-        // Read tag and sanity-check `tag_layout`.
-        let tag_val = self.read_immediate(&self.operand_field(op, tag_field)?)?;
-        assert_eq!(tag_layout.size, tag_val.layout.size);
-        assert_eq!(tag_layout.abi.is_signed(), tag_val.layout.abi.is_signed());
-        trace!("tag value: {}", tag_val);
-
-        // Figure out which discriminant and variant this corresponds to.
-        Ok(match *tag_encoding {
-            TagEncoding::Direct => {
-                let scalar = tag_val.to_scalar();
-                // Generate a specific error if `tag_val` is not an integer.
-                // (`tag_bits` itself is only used for error messages below.)
-                let tag_bits = scalar
-                    .try_to_int()
-                    .map_err(|dbg_val| err_ub!(InvalidTag(dbg_val)))?
-                    .assert_bits(tag_layout.size);
-                // Cast bits from tag layout to discriminant layout.
-                // After the checks we did above, this cannot fail, as
-                // discriminants are int-like.
-                let discr_val =
-                    self.cast_from_int_like(scalar, tag_val.layout, discr_layout.ty).unwrap();
-                let discr_bits = discr_val.assert_bits(discr_layout.size);
-                // Convert discriminant to variant index, and catch invalid discriminants.
-                let index = match *op.layout.ty.kind() {
-                    ty::Adt(adt, _) => {
-                        adt.discriminants(*self.tcx).find(|(_, var)| var.val == discr_bits)
-                    }
-                    ty::Generator(def_id, substs, _) => {
-                        let substs = substs.as_generator();
-                        substs
-                            .discriminants(def_id, *self.tcx)
-                            .find(|(_, var)| var.val == discr_bits)
-                    }
-                    _ => span_bug!(self.cur_span(), "tagged layout for non-adt non-generator"),
-                }
-                .ok_or_else(|| err_ub!(InvalidTag(Scalar::from_uint(tag_bits, tag_layout.size))))?;
-                // Return the cast value, and the index.
-                (discr_val, index.0)
-            }
-            TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => {
-                let tag_val = tag_val.to_scalar();
-                // Compute the variant this niche value/"tag" corresponds to. With niche layout,
-                // discriminant (encoded in niche/tag) and variant index are the same.
-                let variants_start = niche_variants.start().as_u32();
-                let variants_end = niche_variants.end().as_u32();
-                let variant = match tag_val.try_to_int() {
-                    Err(dbg_val) => {
-                        // So this is a pointer then, and casting to an int failed.
-                        // Can only happen during CTFE.
-                        // The niche must be just 0, and the ptr not null, then we know this is
-                        // okay. Everything else, we conservatively reject.
-                        let ptr_valid = niche_start == 0
-                            && variants_start == variants_end
-                            && !self.scalar_may_be_null(tag_val)?;
-                        if !ptr_valid {
-                            throw_ub!(InvalidTag(dbg_val))
-                        }
-                        untagged_variant
-                    }
-                    Ok(tag_bits) => {
-                        let tag_bits = tag_bits.assert_bits(tag_layout.size);
-                        // We need to use machine arithmetic to get the relative variant idx:
-                        // variant_index_relative = tag_val - niche_start_val
-                        let tag_val = ImmTy::from_uint(tag_bits, tag_layout);
-                        let niche_start_val = ImmTy::from_uint(niche_start, tag_layout);
-                        let variant_index_relative_val =
-                            self.binary_op(mir::BinOp::Sub, &tag_val, &niche_start_val)?;
-                        let variant_index_relative =
-                            variant_index_relative_val.to_scalar().assert_bits(tag_val.layout.size);
-                        // Check if this is in the range that indicates an actual discriminant.
-                        if variant_index_relative <= u128::from(variants_end - variants_start) {
-                            let variant_index_relative = u32::try_from(variant_index_relative)
-                                .expect("we checked that this fits into a u32");
-                            // Then computing the absolute variant idx should not overflow any more.
-                            let variant_index = variants_start
-                                .checked_add(variant_index_relative)
-                                .expect("overflow computing absolute variant idx");
-                            let variants_len = op
-                                .layout
-                                .ty
-                                .ty_adt_def()
-                                .expect("tagged layout for non adt")
-                                .variants()
-                                .len();
-                            assert!(usize::try_from(variant_index).unwrap() < variants_len);
-                            VariantIdx::from_u32(variant_index)
-                        } else {
-                            untagged_variant
-                        }
-                    }
-                };
-                // Compute the size of the scalar we need to return.
-                // No need to cast, because the variant index directly serves as discriminant and is
-                // encoded in the tag.
-                (Scalar::from_uint(variant.as_u32(), discr_layout.size), variant)
-            }
-        })
-    }
 }
 
 // Some nodes are used a lot. Make sure they don't unintentionally get bigger.
diff --git a/compiler/rustc_const_eval/src/interpret/operator.rs b/compiler/rustc_const_eval/src/interpret/operator.rs
index e8ff70e3a..4decfe863 100644
--- a/compiler/rustc_const_eval/src/interpret/operator.rs
+++ b/compiler/rustc_const_eval/src/interpret/operator.rs
@@ -10,28 +10,20 @@ use super::{ImmTy, Immediate, InterpCx, Machine, PlaceTy};
 impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     /// Applies the binary operation `op` to the two operands and writes a tuple of the result
     /// and a boolean signifying the potential overflow to the destination.
-    ///
-    /// `force_overflow_checks` indicates whether overflow checks should be done even when
-    /// `tcx.sess.overflow_checks()` is `false`.
     pub fn binop_with_overflow(
         &mut self,
         op: mir::BinOp,
-        force_overflow_checks: bool,
         left: &ImmTy<'tcx, M::Provenance>,
         right: &ImmTy<'tcx, M::Provenance>,
         dest: &PlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx> {
         let (val, overflowed, ty) = self.overflowing_binary_op(op, &left, &right)?;
         debug_assert_eq!(
-            self.tcx.intern_tup(&[ty, self.tcx.types.bool]),
+            self.tcx.mk_tup(&[ty, self.tcx.types.bool]),
             dest.layout.ty,
             "type mismatch for result of {:?}",
             op,
         );
-        // As per https://github.com/rust-lang/rust/pull/98738, we always return `false` in the 2nd
-        // component when overflow checking is disabled.
-        let overflowed =
-            overflowed && (force_overflow_checks || M::checked_binop_checks_overflow(self));
         // Write the result to `dest`.
         if let Abi::ScalarPair(..) = dest.layout.abi {
             // We can use the optimized path and avoid `place_field` (which might do
diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs
index 274af61ee..3c463500a 100644
--- a/compiler/rustc_const_eval/src/interpret/place.rs
+++ b/compiler/rustc_const_eval/src/interpret/place.rs
@@ -7,8 +7,8 @@ use either::{Either, Left, Right};
 use rustc_ast::Mutability;
 use rustc_middle::mir;
 use rustc_middle::ty;
-use rustc_middle::ty::layout::{LayoutOf, PrimitiveExt, TyAndLayout};
-use rustc_target::abi::{self, Abi, Align, HasDataLayout, Size, TagEncoding, VariantIdx};
+use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
+use rustc_target::abi::{self, Abi, Align, HasDataLayout, Size, VariantIdx};
 
 use super::{
     alloc_range, mir_assign_valid_types, AllocId, AllocRef, AllocRefMut, CheckInAllocMsg,
@@ -26,6 +26,7 @@ pub enum MemPlaceMeta<Prov: Provenance = AllocId> {
 }
 
 impl<Prov: Provenance> MemPlaceMeta<Prov> {
+    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
     pub fn unwrap_meta(self) -> Scalar<Prov> {
         match self {
             Self::Meta(s) => s,
@@ -147,12 +148,16 @@ impl<Prov: Provenance> MemPlace<Prov> {
     }
 
     #[inline]
-    pub fn offset_with_meta<'tcx>(
+    pub(super) fn offset_with_meta<'tcx>(
         self,
         offset: Size,
         meta: MemPlaceMeta<Prov>,
         cx: &impl HasDataLayout,
     ) -> InterpResult<'tcx, Self> {
+        debug_assert!(
+            !meta.has_meta() || self.meta.has_meta(),
+            "cannot use `offset_with_meta` to add metadata to a place"
+        );
         Ok(MemPlace { ptr: self.ptr.offset(offset, cx)?, meta })
     }
 }
@@ -178,12 +183,15 @@ impl<'tcx, Prov: Provenance> MPlaceTy<'tcx, Prov> {
     pub fn fake_alloc_zst(layout: TyAndLayout<'tcx>) -> Self {
         assert!(layout.is_zst());
         let align = layout.align.abi;
-        let ptr = Pointer::from_addr(align.bytes()); // no provenance, absolute address
+        let ptr = Pointer::from_addr_invalid(align.bytes()); // no provenance, absolute address
         MPlaceTy { mplace: MemPlace { ptr, meta: MemPlaceMeta::None }, layout, align }
     }
 
+    /// Offset the place in memory and change its metadata.
+    ///
+    /// This can go wrong very easily if you give the wrong layout for the new place!
     #[inline]
-    pub fn offset_with_meta(
+    pub(crate) fn offset_with_meta(
         &self,
         offset: Size,
         meta: MemPlaceMeta<Prov>,
@@ -197,6 +205,9 @@ impl<'tcx, Prov: Provenance> MPlaceTy<'tcx, Prov> {
         })
     }
 
+    /// Offset the place in memory.
+    ///
+    /// This can go wrong very easily if you give the wrong layout for the new place!
     pub fn offset(
         &self,
         offset: Size,
@@ -229,7 +240,7 @@ impl<'tcx, Prov: Provenance> MPlaceTy<'tcx, Prov> {
         if self.layout.is_unsized() {
             // We need to consult `meta` metadata
             match self.layout.ty.kind() {
-                ty::Slice(..) | ty::Str => self.mplace.meta.unwrap_meta().to_machine_usize(cx),
+                ty::Slice(..) | ty::Str => self.mplace.meta.unwrap_meta().to_target_usize(cx),
                 _ => bug!("len not supported on unsized type {:?}", self.layout.ty),
             }
         } else {
@@ -241,14 +252,6 @@ impl<'tcx, Prov: Provenance> MPlaceTy<'tcx, Prov> {
             }
         }
     }
-
-    #[inline]
-    pub(super) fn vtable(&self) -> Scalar<Prov> {
-        match self.layout.ty.kind() {
-            ty::Dynamic(..) => self.mplace.meta.unwrap_meta(),
-            _ => bug!("vtable not supported on type {:?}", self.layout.ty),
-        }
-    }
 }
 
 // These are defined here because they produce a place.
@@ -266,7 +269,12 @@ impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
     #[inline(always)]
     #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
     pub fn assert_mem_place(&self) -> MPlaceTy<'tcx, Prov> {
-        self.as_mplace_or_imm().left().unwrap()
+        self.as_mplace_or_imm().left().unwrap_or_else(|| {
+            bug!(
+                "OpTy of type {} was immediate when it was expected to be an MPlace",
+                self.layout.ty
+            )
+        })
     }
 }
 
@@ -283,7 +291,12 @@ impl<'tcx, Prov: Provenance> PlaceTy<'tcx, Prov> {
     #[inline(always)]
     #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
     pub fn assert_mem_place(&self) -> MPlaceTy<'tcx, Prov> {
-        self.as_mplace_or_local().left().unwrap()
+        self.as_mplace_or_local().left().unwrap_or_else(|| {
+            bug!(
+                "PlaceTy of type {} was a local when it was expected to be an MPlace",
+                self.layout.ty
+            )
+        })
     }
 }
 
@@ -340,7 +353,8 @@ where
     pub(super) fn get_place_alloc(
         &self,
         place: &MPlaceTy<'tcx, M::Provenance>,
-    ) -> InterpResult<'tcx, Option<AllocRef<'_, 'tcx, M::Provenance, M::AllocExtra>>> {
+    ) -> InterpResult<'tcx, Option<AllocRef<'_, 'tcx, M::Provenance, M::AllocExtra, M::Bytes>>>
+    {
         assert!(place.layout.is_sized());
         assert!(!place.meta.has_meta());
         let size = place.layout.size;
@@ -351,7 +365,8 @@ where
     pub(super) fn get_place_alloc_mut(
         &mut self,
         place: &MPlaceTy<'tcx, M::Provenance>,
-    ) -> InterpResult<'tcx, Option<AllocRefMut<'_, 'tcx, M::Provenance, M::AllocExtra>>> {
+    ) -> InterpResult<'tcx, Option<AllocRefMut<'_, 'tcx, M::Provenance, M::AllocExtra, M::Bytes>>>
+    {
         assert!(place.layout.is_sized());
         assert!(!place.meta.has_meta());
         let size = place.layout.size;
@@ -754,9 +769,9 @@ where
         str: &str,
         kind: MemoryKind<M::MemoryKind>,
         mutbl: Mutability,
-    ) -> MPlaceTy<'tcx, M::Provenance> {
-        let ptr = self.allocate_bytes_ptr(str.as_bytes(), Align::ONE, kind, mutbl);
-        let meta = Scalar::from_machine_usize(u64::try_from(str.len()).unwrap(), self);
+    ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
+        let ptr = self.allocate_bytes_ptr(str.as_bytes(), Align::ONE, kind, mutbl)?;
+        let meta = Scalar::from_target_usize(u64::try_from(str.len()).unwrap(), self);
         let mplace = MemPlace { ptr: ptr.into(), meta: MemPlaceMeta::Meta(meta) };
 
         let ty = self.tcx.mk_ref(
@@ -764,95 +779,35 @@ where
             ty::TypeAndMut { ty: self.tcx.types.str_, mutbl },
         );
         let layout = self.layout_of(ty).unwrap();
-        MPlaceTy { mplace, layout, align: layout.align.abi }
+        Ok(MPlaceTy { mplace, layout, align: layout.align.abi })
     }
 
-    /// Writes the discriminant of the given variant.
-    #[instrument(skip(self), level = "debug")]
-    pub fn write_discriminant(
+    /// Writes the aggregate to the destination.
+    #[instrument(skip(self), level = "trace")]
+    pub fn write_aggregate(
         &mut self,
-        variant_index: VariantIdx,
+        kind: &mir::AggregateKind<'tcx>,
+        operands: &[mir::Operand<'tcx>],
         dest: &PlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx> {
-        // This must be an enum or generator.
-        match dest.layout.ty.kind() {
-            ty::Adt(adt, _) => assert!(adt.is_enum()),
-            ty::Generator(..) => {}
-            _ => span_bug!(
-                self.cur_span(),
-                "write_discriminant called on non-variant-type (neither enum nor generator)"
-            ),
-        }
-        // Layout computation excludes uninhabited variants from consideration
-        // therefore there's no way to represent those variants in the given layout.
-        // Essentially, uninhabited variants do not have a tag that corresponds to their
-        // discriminant, so we cannot do anything here.
-        // When evaluating we will always error before even getting here, but ConstProp 'executes'
-        // dead code, so we cannot ICE here.
-        if dest.layout.for_variant(self, variant_index).abi.is_uninhabited() {
-            throw_ub!(UninhabitedEnumVariantWritten)
-        }
-
-        match dest.layout.variants {
-            abi::Variants::Single { index } => {
-                assert_eq!(index, variant_index);
-            }
-            abi::Variants::Multiple {
-                tag_encoding: TagEncoding::Direct,
-                tag: tag_layout,
-                tag_field,
-                ..
-            } => {
-                // No need to validate that the discriminant here because the
-                // `TyAndLayout::for_variant()` call earlier already checks the variant is valid.
-
-                let discr_val =
-                    dest.layout.ty.discriminant_for_variant(*self.tcx, variant_index).unwrap().val;
-
-                // raw discriminants for enums are isize or bigger during
-                // their computation, but the in-memory tag is the smallest possible
-                // representation
-                let size = tag_layout.size(self);
-                let tag_val = size.truncate(discr_val);
-
-                let tag_dest = self.place_field(dest, tag_field)?;
-                self.write_scalar(Scalar::from_uint(tag_val, size), &tag_dest)?;
-            }
-            abi::Variants::Multiple {
-                tag_encoding:
-                    TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start },
-                tag: tag_layout,
-                tag_field,
-                ..
-            } => {
-                // No need to validate that the discriminant here because the
-                // `TyAndLayout::for_variant()` call earlier already checks the variant is valid.
-
-                if variant_index != untagged_variant {
-                    let variants_start = niche_variants.start().as_u32();
-                    let variant_index_relative = variant_index
-                        .as_u32()
-                        .checked_sub(variants_start)
-                        .expect("overflow computing relative variant idx");
-                    // We need to use machine arithmetic when taking into account `niche_start`:
-                    // tag_val = variant_index_relative + niche_start_val
-                    let tag_layout = self.layout_of(tag_layout.primitive().to_int_ty(*self.tcx))?;
-                    let niche_start_val = ImmTy::from_uint(niche_start, tag_layout);
-                    let variant_index_relative_val =
-                        ImmTy::from_uint(variant_index_relative, tag_layout);
-                    let tag_val = self.binary_op(
-                        mir::BinOp::Add,
-                        &variant_index_relative_val,
-                        &niche_start_val,
-                    )?;
-                    // Write result.
-                    let niche_dest = self.place_field(dest, tag_field)?;
-                    self.write_immediate(*tag_val, &niche_dest)?;
-                }
+        self.write_uninit(&dest)?;
+        let (variant_index, variant_dest, active_field_index) = match *kind {
+            mir::AggregateKind::Adt(_, variant_index, _, _, active_field_index) => {
+                let variant_dest = self.place_downcast(&dest, variant_index)?;
+                (variant_index, variant_dest, active_field_index)
             }
+            _ => (VariantIdx::from_u32(0), dest.clone(), None),
+        };
+        if active_field_index.is_some() {
+            assert_eq!(operands.len(), 1);
         }
-
-        Ok(())
+        for (field_index, operand) in operands.iter().enumerate() {
+            let field_index = active_field_index.unwrap_or(field_index);
+            let field_dest = self.place_field(&variant_dest, field_index)?;
+            let op = self.eval_operand(operand, Some(field_dest.layout))?;
+            self.copy_op(&op, &field_dest, /*allow_transmute*/ false)?;
+        }
+        self.write_discriminant(variant_index, &dest)
     }
 
     pub fn raw_const_to_mplace(
@@ -867,11 +822,16 @@ where
     }
 
     /// Turn a place with a `dyn Trait` type into a place with the actual dynamic type.
+    /// Aso returns the vtable.
     pub(super) fn unpack_dyn_trait(
         &self,
         mplace: &MPlaceTy<'tcx, M::Provenance>,
-    ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
-        let vtable = mplace.vtable().to_pointer(self)?; // also sanity checks the type
+    ) -> InterpResult<'tcx, (MPlaceTy<'tcx, M::Provenance>, Pointer<Option<M::Provenance>>)> {
+        assert!(
+            matches!(mplace.layout.ty.kind(), ty::Dynamic(_, _, ty::Dyn)),
+            "`unpack_dyn_trait` only makes sense on `dyn*` types"
+        );
+        let vtable = mplace.meta.unwrap_meta().to_pointer(self)?;
         let (ty, _) = self.get_ptr_vtable(vtable)?;
         let layout = self.layout_of(ty)?;
 
@@ -880,7 +840,26 @@ where
             layout,
             align: layout.align.abi,
         };
-        Ok(mplace)
+        Ok((mplace, vtable))
+    }
+
+    /// Turn an operand with a `dyn* Trait` type into an operand with the actual dynamic type.
+    /// Aso returns the vtable.
+    pub(super) fn unpack_dyn_star(
+        &self,
+        op: &OpTy<'tcx, M::Provenance>,
+    ) -> InterpResult<'tcx, (OpTy<'tcx, M::Provenance>, Pointer<Option<M::Provenance>>)> {
+        assert!(
+            matches!(op.layout.ty.kind(), ty::Dynamic(_, _, ty::DynStar)),
+            "`unpack_dyn_star` only makes sense on `dyn*` types"
+        );
+        let data = self.operand_field(&op, 0)?;
+        let vtable = self.operand_field(&op, 1)?;
+        let vtable = self.read_pointer(&vtable)?;
+        let (ty, _) = self.get_ptr_vtable(vtable)?;
+        let layout = self.layout_of(ty)?;
+        let data = data.transmute(layout);
+        Ok((data, vtable))
     }
 }
 
diff --git a/compiler/rustc_const_eval/src/interpret/projection.rs b/compiler/rustc_const_eval/src/interpret/projection.rs
index 291464ab5..91da930db 100644
--- a/compiler/rustc_const_eval/src/interpret/projection.rs
+++ b/compiler/rustc_const_eval/src/interpret/projection.rs
@@ -319,7 +319,7 @@ where
             // implement this.
             ty::Array(inner, _) => (MemPlaceMeta::None, self.tcx.mk_array(*inner, inner_len)),
             ty::Slice(..) => {
-                let len = Scalar::from_machine_usize(inner_len, self);
+                let len = Scalar::from_target_usize(inner_len, self);
                 (MemPlaceMeta::Meta(len), base.layout.ty)
             }
             _ => {
@@ -363,7 +363,7 @@ where
             Index(local) => {
                 let layout = self.layout_of(self.tcx.types.usize)?;
                 let n = self.local_to_op(self.frame(), local, Some(layout))?;
-                let n = self.read_machine_usize(&n)?;
+                let n = self.read_target_usize(&n)?;
                 self.place_index(base, n)?
             }
             ConstantIndex { offset, min_length, from_end } => {
@@ -392,7 +392,7 @@ where
             Index(local) => {
                 let layout = self.layout_of(self.tcx.types.usize)?;
                 let n = self.local_to_op(self.frame(), local, Some(layout))?;
-                let n = self.read_machine_usize(&n)?;
+                let n = self.read_target_usize(&n)?;
                 self.operand_index(base, n)?
             }
             ConstantIndex { offset, min_length, from_end } => {
diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs
index fad4cb06c..6863435e5 100644
--- a/compiler/rustc_const_eval/src/interpret/step.rs
+++ b/compiler/rustc_const_eval/src/interpret/step.rs
@@ -129,6 +129,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             // FIXME(#73156): Handle source code coverage in const eval
             Coverage(..) => {}
 
+            ConstEvalCounter => {
+                M::increment_const_eval_counter(self)?;
+            }
+
             // Defined to do nothing. These are added by optimization passes, to avoid changing the
             // size of MIR constantly.
             Nop => {}
@@ -181,9 +185,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 let left = self.read_immediate(&self.eval_operand(left, None)?)?;
                 let layout = binop_right_homogeneous(bin_op).then_some(left.layout);
                 let right = self.read_immediate(&self.eval_operand(right, layout)?)?;
-                self.binop_with_overflow(
-                    bin_op, /*force_overflow_checks*/ false, &left, &right, &dest,
-                )?;
+                self.binop_with_overflow(bin_op, &left, &right, &dest)?;
             }
 
             UnaryOp(un_op, ref operand) => {
@@ -195,13 +197,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             }
 
             Aggregate(box ref kind, ref operands) => {
-                assert!(matches!(kind, mir::AggregateKind::Array(..)));
-
-                for (field_index, operand) in operands.iter().enumerate() {
-                    let op = self.eval_operand(operand, None)?;
-                    let field_dest = self.place_field(&dest, field_index)?;
-                    self.copy_op(&op, &field_dest, /*allow_transmute*/ false)?;
-                }
+                self.write_aggregate(kind, operands, &dest)?;
             }
 
             Repeat(ref operand, _) => {
@@ -244,7 +240,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 let src = self.eval_place(place)?;
                 let op = self.place_to_op(&src)?;
                 let len = op.len(self)?;
-                self.write_scalar(Scalar::from_machine_usize(len, self), &dest)?;
+                self.write_scalar(Scalar::from_target_usize(len, self), &dest)?;
             }
 
             Ref(_, borrow_kind, place) => {
@@ -299,7 +295,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                     mir::NullOp::SizeOf => layout.size.bytes(),
                     mir::NullOp::AlignOf => layout.align.abi.bytes(),
                 };
-                self.write_scalar(Scalar::from_machine_usize(val, self), &dest)?;
+                self.write_scalar(Scalar::from_target_usize(val, self), &dest)?;
             }
 
             ShallowInitBox(ref operand, _) => {
diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs
index da320cd1c..2aea7c79b 100644
--- a/compiler/rustc_const_eval/src/interpret/terminator.rs
+++ b/compiler/rustc_const_eval/src/interpret/terminator.rs
@@ -73,7 +73,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 let fn_sig =
                     self.tcx.normalize_erasing_late_bound_regions(self.param_env, fn_sig_binder);
                 let extra_args = &args[fn_sig.inputs().len()..];
-                let extra_args = self.tcx.mk_type_list(extra_args.iter().map(|arg| arg.layout.ty));
+                let extra_args =
+                    self.tcx.mk_type_list_from_iter(extra_args.iter().map(|arg| arg.layout.ty));
 
                 let (fn_val, fn_abi, with_caller_location) = match *func.layout.ty.kind() {
                     ty::FnPtr(_sig) => {
@@ -137,8 +138,14 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             }
 
             Assert { ref cond, expected, ref msg, target, cleanup } => {
+                let ignored = M::ignore_checkable_overflow_assertions(self)
+                    && match msg {
+                        mir::AssertKind::OverflowNeg(..) => true,
+                        mir::AssertKind::Overflow(op, ..) => op.is_checkable(),
+                        _ => false,
+                    };
                 let cond_val = self.read_scalar(&self.eval_operand(cond, None)?)?.to_bool()?;
-                if expected == cond_val {
+                if ignored || expected == cond_val {
                     self.go_to_block(target);
                 } else {
                     M::assert_panic(self, msg, cleanup)?;
@@ -541,7 +548,14 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 let receiver_place = loop {
                     match receiver.layout.ty.kind() {
                         ty::Ref(..) | ty::RawPtr(..) => break self.deref_operand(&receiver)?,
-                        ty::Dynamic(..) => break receiver.assert_mem_place(), // no immediate unsized values
+                        ty::Dynamic(.., ty::Dyn) => break receiver.assert_mem_place(), // no immediate unsized values
+                        ty::Dynamic(.., ty::DynStar) => {
+                            // Not clear how to handle this, so far we assume the receiver is always a pointer.
+                            span_bug!(
+                                self.cur_span(),
+                                "by-value calls on a `dyn*`... are those a thing?"
+                            );
+                        }
                         _ => {
                             // Not there yet, search for the only non-ZST field.
                             let mut non_zst_field = None;
@@ -567,39 +581,59 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                         }
                     }
                 };
-                // Obtain the underlying trait we are working on.
-                let receiver_tail = self
-                    .tcx
-                    .struct_tail_erasing_lifetimes(receiver_place.layout.ty, self.param_env);
-                let ty::Dynamic(data, ..) = receiver_tail.kind() else {
-                    span_bug!(self.cur_span(), "dynamic call on non-`dyn` type {}", receiver_tail)
-                };
 
-                // Get the required information from the vtable.
-                let vptr = receiver_place.meta.unwrap_meta().to_pointer(self)?;
-                let (dyn_ty, dyn_trait) = self.get_ptr_vtable(vptr)?;
-                if dyn_trait != data.principal() {
-                    throw_ub_format!(
-                        "`dyn` call on a pointer whose vtable does not match its type"
-                    );
-                }
+                // Obtain the underlying trait we are working on, and the adjusted receiver argument.
+                let (vptr, dyn_ty, adjusted_receiver) = if let ty::Dynamic(data, _, ty::DynStar) =
+                    receiver_place.layout.ty.kind()
+                {
+                    let (recv, vptr) = self.unpack_dyn_star(&receiver_place.into())?;
+                    let (dyn_ty, dyn_trait) = self.get_ptr_vtable(vptr)?;
+                    if dyn_trait != data.principal() {
+                        throw_ub_format!(
+                            "`dyn*` call on a pointer whose vtable does not match its type"
+                        );
+                    }
+                    let recv = recv.assert_mem_place(); // we passed an MPlaceTy to `unpack_dyn_star` so we definitely still have one
+
+                    (vptr, dyn_ty, recv.ptr)
+                } else {
+                    // Doesn't have to be a `dyn Trait`, but the unsized tail must be `dyn Trait`.
+                    // (For that reason we also cannot use `unpack_dyn_trait`.)
+                    let receiver_tail = self
+                        .tcx
+                        .struct_tail_erasing_lifetimes(receiver_place.layout.ty, self.param_env);
+                    let ty::Dynamic(data, _, ty::Dyn) = receiver_tail.kind() else {
+                            span_bug!(self.cur_span(), "dynamic call on non-`dyn` type {}", receiver_tail)
+                        };
+                    assert!(receiver_place.layout.is_unsized());
+
+                    // Get the required information from the vtable.
+                    let vptr = receiver_place.meta.unwrap_meta().to_pointer(self)?;
+                    let (dyn_ty, dyn_trait) = self.get_ptr_vtable(vptr)?;
+                    if dyn_trait != data.principal() {
+                        throw_ub_format!(
+                            "`dyn` call on a pointer whose vtable does not match its type"
+                        );
+                    }
+
+                    // It might be surprising that we use a pointer as the receiver even if this
+                    // is a by-val case; this works because by-val passing of an unsized `dyn
+                    // Trait` to a function is actually desugared to a pointer.
+                    (vptr, dyn_ty, receiver_place.ptr)
+                };
 
                 // Now determine the actual method to call. We can do that in two different ways and
                 // compare them to ensure everything fits.
                 let Some(ty::VtblEntry::Method(fn_inst)) = self.get_vtable_entries(vptr)?.get(idx).copied() else {
                     throw_ub_format!("`dyn` call trying to call something that is not a method")
                 };
+                trace!("Virtual call dispatches to {fn_inst:#?}");
                 if cfg!(debug_assertions) {
                     let tcx = *self.tcx;
 
                     let trait_def_id = tcx.trait_of_item(def_id).unwrap();
                     let virtual_trait_ref =
                         ty::TraitRef::from_method(tcx, trait_def_id, instance.substs);
-                    assert_eq!(
-                        receiver_tail,
-                        virtual_trait_ref.self_ty(),
-                        "mismatch in underlying dyn trait computation within Miri and MIR building",
-                    );
                     let existential_trait_ref =
                         ty::ExistentialTraitRef::erase_self_ty(tcx, virtual_trait_ref);
                     let concrete_trait_ref = existential_trait_ref.with_self_ty(tcx, dyn_ty);
@@ -614,17 +648,12 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                     assert_eq!(fn_inst, concrete_method);
                 }
 
-                // `*mut receiver_place.layout.ty` is almost the layout that we
-                // want for args[0]: We have to project to field 0 because we want
-                // a thin pointer.
-                assert!(receiver_place.layout.is_unsized());
-                let receiver_ptr_ty = self.tcx.mk_mut_ptr(receiver_place.layout.ty);
-                let this_receiver_ptr = self.layout_of(receiver_ptr_ty)?.field(self, 0);
-                // Adjust receiver argument.
-                args[0] = OpTy::from(ImmTy::from_immediate(
-                    Scalar::from_maybe_pointer(receiver_place.ptr, self).into(),
-                    this_receiver_ptr,
-                ));
+                // Adjust receiver argument. Layout can be any (thin) ptr.
+                args[0] = ImmTy::from_immediate(
+                    Scalar::from_maybe_pointer(adjusted_receiver, self).into(),
+                    self.layout_of(self.tcx.mk_mut_ptr(dyn_ty))?,
+                )
+                .into();
                 trace!("Patched receiver operand to {:#?}", args[0]);
                 // recurse with concrete function
                 self.eval_fn_call(
@@ -653,15 +682,24 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         // implementation fail -- a problem shared by rustc.
         let place = self.force_allocation(place)?;
 
-        let (instance, place) = match place.layout.ty.kind() {
-            ty::Dynamic(..) => {
+        let place = match place.layout.ty.kind() {
+            ty::Dynamic(_, _, ty::Dyn) => {
                 // Dropping a trait object. Need to find actual drop fn.
-                let place = self.unpack_dyn_trait(&place)?;
-                let instance = ty::Instance::resolve_drop_in_place(*self.tcx, place.layout.ty);
-                (instance, place)
+                self.unpack_dyn_trait(&place)?.0
+            }
+            ty::Dynamic(_, _, ty::DynStar) => {
+                // Dropping a `dyn*`. Need to find actual drop fn.
+                self.unpack_dyn_star(&place.into())?.0.assert_mem_place()
+            }
+            _ => {
+                debug_assert_eq!(
+                    instance,
+                    ty::Instance::resolve_drop_in_place(*self.tcx, place.layout.ty)
+                );
+                place
             }
-            _ => (instance, place),
         };
+        let instance = ty::Instance::resolve_drop_in_place(*self.tcx, place.layout.ty);
         let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty())?;
 
         let arg = ImmTy::from_immediate(
diff --git a/compiler/rustc_const_eval/src/interpret/util.rs b/compiler/rustc_const_eval/src/interpret/util.rs
index cabc65e2c..bf2b4ee69 100644
--- a/compiler/rustc_const_eval/src/interpret/util.rs
+++ b/compiler/rustc_const_eval/src/interpret/util.rs
@@ -1,5 +1,7 @@
 use rustc_middle::mir::interpret::InterpResult;
-use rustc_middle::ty::{self, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitor};
+use rustc_middle::ty::{
+    self, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor,
+};
 use std::ops::ControlFlow;
 
 /// Checks whether a type contains generic parameters which require substitution.
@@ -9,7 +11,7 @@ use std::ops::ControlFlow;
 /// case these parameters are unused.
 pub(crate) fn ensure_monomorphic_enough<'tcx, T>(tcx: TyCtxt<'tcx>, ty: T) -> InterpResult<'tcx>
 where
-    T: TypeVisitable<'tcx>,
+    T: TypeVisitable<TyCtxt<'tcx>>,
 {
     debug!("ensure_monomorphic_enough: ty={:?}", ty);
     if !ty.needs_subst() {
@@ -21,7 +23,7 @@ where
         tcx: TyCtxt<'tcx>,
     }
 
-    impl<'tcx> TypeVisitor<'tcx> for UsedParamsNeedSubstVisitor<'tcx> {
+    impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for UsedParamsNeedSubstVisitor<'tcx> {
         type BreakTy = FoundParam;
 
         fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs
index 19e359986..f7881c509 100644
--- a/compiler/rustc_const_eval/src/interpret/validity.rs
+++ b/compiler/rustc_const_eval/src/interpret/validity.rs
@@ -23,18 +23,18 @@ use std::hash::Hash;
 // for the validation errors
 use super::UndefinedBehaviorInfo::*;
 use super::{
-    CheckInAllocMsg, GlobalAlloc, ImmTy, Immediate, InterpCx, InterpResult, MPlaceTy, Machine,
-    MemPlaceMeta, OpTy, Scalar, ValueVisitor,
+    AllocId, CheckInAllocMsg, GlobalAlloc, ImmTy, Immediate, InterpCx, InterpResult, MPlaceTy,
+    Machine, MemPlaceMeta, OpTy, Pointer, Scalar, ValueVisitor,
 };
 
 macro_rules! throw_validation_failure {
-    ($where:expr, { $( $what_fmt:expr ),+ } $( expected { $( $expected_fmt:expr ),+ } )?) => {{
+    ($where:expr, { $( $what_fmt:tt )* } $( expected { $( $expected_fmt:tt )* } )?) => {{
         let mut msg = String::new();
         msg.push_str("encountered ");
-        write!(&mut msg, $($what_fmt),+).unwrap();
+        write!(&mut msg, $($what_fmt)*).unwrap();
         $(
             msg.push_str(", but expected ");
-            write!(&mut msg, $($expected_fmt),+).unwrap();
+            write!(&mut msg, $($expected_fmt)*).unwrap();
         )?
         let path = rustc_middle::ty::print::with_no_trimmed_paths!({
             let where_ = &$where;
@@ -82,7 +82,7 @@ macro_rules! throw_validation_failure {
 ///
 macro_rules! try_validation {
     ($e:expr, $where:expr,
-    $( $( $p:pat_param )|+ => { $( $what_fmt:expr ),+ } $( expected { $( $expected_fmt:expr ),+ } )? ),+ $(,)?
+    $( $( $p:pat_param )|+ => { $( $what_fmt:tt )* } $( expected { $( $expected_fmt:tt )* } )? ),+ $(,)?
     ) => {{
         match $e {
             Ok(x) => x,
@@ -93,7 +93,7 @@ macro_rules! try_validation {
                     InterpError::UndefinedBehavior($($p)|+) =>
                        throw_validation_failure!(
                             $where,
-                            { $( $what_fmt ),+ } $( expected { $( $expected_fmt ),+ } )?
+                            { $( $what_fmt )* } $( expected { $( $expected_fmt )* } )?
                         )
                 ),+,
                 #[allow(unreachable_patterns)]
@@ -240,10 +240,8 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
                 // FIXME this should be more descriptive i.e. CapturePlace instead of CapturedVar
                 // https://github.com/rust-lang/project-rfc-2229/issues/46
                 if let Some(local_def_id) = def_id.as_local() {
-                    let tables = self.ecx.tcx.typeck(local_def_id);
-                    if let Some(captured_place) =
-                        tables.closure_min_captures_flattened(local_def_id).nth(field)
-                    {
+                    let captures = self.ecx.tcx.closure_captures(local_def_id);
+                    if let Some(captured_place) = captures.get(field) {
                         // Sometimes the index is beyond the number of upvars (seen
                         // for a generator).
                         let var_hir_id = captured_place.get_root_variable();
@@ -335,7 +333,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
     ) -> InterpResult<'tcx> {
         let tail = self.ecx.tcx.struct_tail_erasing_lifetimes(pointee.ty, self.ecx.param_env);
         match tail.kind() {
-            ty::Dynamic(..) => {
+            ty::Dynamic(_, _, ty::Dyn) => {
                 let vtable = meta.unwrap_meta().to_pointer(self.ecx)?;
                 // Make sure it is a genuine vtable pointer.
                 let (_ty, _trait) = try_validation!(
@@ -348,7 +346,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
                 // FIXME: check if the type/trait match what ty::Dynamic says?
             }
             ty::Slice(..) | ty::Str => {
-                let _len = meta.unwrap_meta().to_machine_usize(self.ecx)?;
+                let _len = meta.unwrap_meta().to_target_usize(self.ecx)?;
                 // We do not check that `len * elem_size <= isize::MAX`:
                 // that is only required for references, and there it falls out of the
                 // "dereferenceable" check performed by Stacked Borrows.
@@ -399,12 +397,15 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
                 {
                     "an unaligned {kind} (required {} byte alignment but found {})",
                     required.bytes(),
-                    has.bytes()
+                    has.bytes(),
                 },
             DanglingIntPointer(0, _) =>
                 { "a null {kind}" },
             DanglingIntPointer(i, _) =>
-                { "a dangling {kind} (address {i:#x} is unallocated)" },
+                {
+                    "a dangling {kind} ({pointer} has no provenance)",
+                    pointer = Pointer::<Option<AllocId>>::from_addr_invalid(*i),
+                },
             PointerOutOfBounds { .. } =>
                 { "a dangling {kind} (going beyond the bounds of its allocation)" },
             // This cannot happen during const-eval (because interning already detects
@@ -602,6 +603,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
             | ty::Bound(..)
             | ty::Param(..)
             | ty::Alias(..)
+            | ty::GeneratorWitnessMIR(..)
             | ty::GeneratorWitness(..) => bug!("Encountered invalid type {:?}", ty),
         }
     }
diff --git a/compiler/rustc_const_eval/src/interpret/visitor.rs b/compiler/rustc_const_eval/src/interpret/visitor.rs
index f9efc2418..7a1445939 100644
--- a/compiler/rustc_const_eval/src/interpret/visitor.rs
+++ b/compiler/rustc_const_eval/src/interpret/visitor.rs
@@ -284,7 +284,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueMut<'mir, 'tcx, M>
         &self,
         ecx: &InterpCx<'mir, 'tcx, M>,
     ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
-        // We `force_allocation` here so that `from_op` below can work.
+        // No need for `force_allocation` since we are just going to read from this.
         ecx.place_to_op(self)
     }
 
@@ -421,15 +421,25 @@ macro_rules! make_value_visitor {
                 // Special treatment for special types, where the (static) layout is not sufficient.
                 match *ty.kind() {
                     // If it is a trait object, switch to the real type that was used to create it.
-                    ty::Dynamic(..) => {
+                    ty::Dynamic(_, _, ty::Dyn) => {
+                        // Dyn types. This is unsized, and the actual dynamic type of the data is given by the
+                        // vtable stored in the place metadata.
                         // unsized values are never immediate, so we can assert_mem_place
                         let op = v.to_op_for_read(self.ecx())?;
                         let dest = op.assert_mem_place();
-                        let inner_mplace = self.ecx().unpack_dyn_trait(&dest)?;
+                        let inner_mplace = self.ecx().unpack_dyn_trait(&dest)?.0;
                         trace!("walk_value: dyn object layout: {:#?}", inner_mplace.layout);
                         // recurse with the inner type
                         return self.visit_field(&v, 0, &$value_trait::from_op(&inner_mplace.into()));
                     },
+                    ty::Dynamic(_, _, ty::DynStar) => {
+                        // DynStar types. Very different from a dyn type (but strangely part of the
+                        // same variant in `TyKind`): These are pairs where the 2nd component is the
+                        // vtable, and the first component is the data (which must be ptr-sized).
+                        let op = v.to_op_for_proj(self.ecx())?;
+                        let data = self.ecx().unpack_dyn_star(&op)?.0;
+                        return self.visit_field(&v, 0, &$value_trait::from_op(&data));
+                    }
                     // Slices do not need special handling here: they have `Array` field
                     // placement with length 0, so we enter the `Array` case below which
                     // indirectly uses the metadata to determine the actual length.
diff --git a/compiler/rustc_const_eval/src/lib.rs b/compiler/rustc_const_eval/src/lib.rs
index 57b91df2d..ed9efe568 100644
--- a/compiler/rustc_const_eval/src/lib.rs
+++ b/compiler/rustc_const_eval/src/lib.rs
@@ -34,9 +34,12 @@ pub mod interpret;
 pub mod transform;
 pub mod util;
 
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
+use rustc_macros::fluent_messages;
 use rustc_middle::ty;
 use rustc_middle::ty::query::Providers;
-use rustc_target::abi::InitKind;
+
+fluent_messages! { "../locales/en-US.ftl" }
 
 pub fn provide(providers: &mut Providers) {
     const_eval::provide(providers);
@@ -58,7 +61,7 @@ pub fn provide(providers: &mut Providers) {
         let (param_env, value) = param_env_and_value.into_parts();
         const_eval::deref_mir_constant(tcx, param_env, value)
     };
-    providers.permits_uninit_init =
-        |tcx, ty| util::might_permit_raw_init(tcx, ty, InitKind::UninitMitigated0x01Fill);
-    providers.permits_zero_init = |tcx, ty| util::might_permit_raw_init(tcx, ty, InitKind::Zero);
+    providers.check_validity_requirement = |tcx, (init_kind, param_env_and_ty)| {
+        util::check_validity_requirement(tcx, init_kind, param_env_and_ty)
+    };
 }
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/check.rs b/compiler/rustc_const_eval/src/transform/check_consts/check.rs
index 79f1737e3..aa24d9053 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/check.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/check.rs
@@ -10,7 +10,7 @@ use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceC
 use rustc_middle::mir::*;
 use rustc_middle::ty::subst::{GenericArgKind, InternalSubsts};
 use rustc_middle::ty::{self, adjustment::PointerCast, Instance, InstanceDef, Ty, TyCtxt};
-use rustc_middle::ty::{Binder, TraitRef, TypeVisitable};
+use rustc_middle::ty::{Binder, TraitRef, TypeVisitableExt};
 use rustc_mir_dataflow::{self, Analysis};
 use rustc_span::{sym, Span, Symbol};
 use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt as _;
@@ -332,7 +332,7 @@ impl<'mir, 'tcx> Checker<'mir, 'tcx> {
 
     fn check_static(&mut self, def_id: DefId, span: Span) {
         if self.tcx.is_thread_local_static(def_id) {
-            self.tcx.sess.delay_span_bug(span, "tls access is checked in `Rvalue::ThreadLocalRef");
+            self.tcx.sess.delay_span_bug(span, "tls access is checked in `Rvalue::ThreadLocalRef`");
         }
         self.check_op_spanned(ops::StaticAccess, span)
     }
@@ -453,7 +453,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
 
             Rvalue::Aggregate(kind, ..) => {
                 if let AggregateKind::Generator(def_id, ..) = kind.as_ref()
-                    && let Some(generator_kind @ hir::GeneratorKind::Async(..)) = self.tcx.generator_kind(def_id.to_def_id())
+                    && let Some(generator_kind @ hir::GeneratorKind::Async(..)) = self.tcx.generator_kind(def_id)
                 {
                     self.check_op(ops::Generator(generator_kind));
                 }
@@ -473,7 +473,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                     // that this is merely a ZST and it is already eligible for promotion.
                     // This may require an RFC?
                     /*
-                    ty::Array(_, len) if len.try_eval_usize(cx.tcx, cx.param_env) == Some(0)
+                    ty::Array(_, len) if len.try_eval_target_usize(cx.tcx, cx.param_env) == Some(0)
                         => true,
                     */
                     _ => false,
@@ -693,6 +693,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
             | StatementKind::AscribeUserType(..)
             | StatementKind::Coverage(..)
             | StatementKind::Intrinsic(..)
+            | StatementKind::ConstEvalCounter
             | StatementKind::Nop => {}
         }
     }
@@ -754,12 +755,9 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                         let ocx = ObligationCtxt::new(&infcx);
 
                         let predicates = tcx.predicates_of(callee).instantiate(tcx, substs);
-                        let hir_id = tcx
-                            .hir()
-                            .local_def_id_to_hir_id(self.body.source.def_id().expect_local());
                         let cause = ObligationCause::new(
                             terminator.source_info.span,
-                            hir_id,
+                            self.body.source.def_id().expect_local(),
                             ObligationCauseCode::ItemObligation(callee),
                         );
                         let normalized_predicates = ocx.normalize(&cause, param_env, predicates);
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/mod.rs b/compiler/rustc_const_eval/src/transform/check_consts/mod.rs
index 54868e418..0e4501922 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/mod.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/mod.rs
@@ -68,11 +68,11 @@ impl<'mir, 'tcx> ConstCx<'mir, 'tcx> {
     pub fn fn_sig(&self) -> PolyFnSig<'tcx> {
         let did = self.def_id().to_def_id();
         if self.tcx.is_closure(did) {
-            let ty = self.tcx.type_of(did);
+            let ty = self.tcx.type_of(did).subst_identity();
             let ty::Closure(_, substs) = ty.kind() else { bug!("type_of closure not ty::Closure") };
             substs.as_closure().sig()
         } else {
-            self.tcx.fn_sig(did)
+            self.tcx.fn_sig(did).subst_identity()
         }
     }
 }
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/ops.rs b/compiler/rustc_const_eval/src/transform/check_consts/ops.rs
index 0cb5d2ff8..3e416b89c 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/ops.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/ops.rs
@@ -22,13 +22,7 @@ use rustc_span::{BytePos, Pos, Span, Symbol};
 use rustc_trait_selection::traits::SelectionContext;
 
 use super::ConstCx;
-use crate::errors::{
-    InteriorMutabilityBorrow, InteriorMutableDataRefer, MutDerefErr, NonConstFmtMacroCall,
-    NonConstFnCall, NonConstOpErr, PanicNonStrErr, RawPtrToIntErr, StaticAccessErr,
-    TransientMutBorrowErr, TransientMutBorrowErrRaw, UnallowedFnPointerCall,
-    UnallowedHeapAllocations, UnallowedInlineAsm, UnallowedMutableRefs, UnallowedMutableRefsRaw,
-    UnallowedOpInConstContext, UnstableConstFn,
-};
+use crate::errors;
 use crate::util::{call_kind, CallDesugaringKind, CallKind};
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -99,7 +93,7 @@ impl<'tcx> NonConstOp<'tcx> for FnCallIndirect {
         ccx: &ConstCx<'_, 'tcx>,
         span: Span,
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
-        ccx.tcx.sess.create_err(UnallowedFnPointerCall { span, kind: ccx.const_kind() })
+        ccx.tcx.sess.create_err(errors::UnallowedFnPointerCall { span, kind: ccx.const_kind() })
     }
 }
 
@@ -142,6 +136,7 @@ impl<'tcx> NonConstOp<'tcx> for FnCallNonConst<'tcx> {
                             &param_ty.name.as_str(),
                             &constraint,
                             None,
+                            None,
                         );
                     }
                 }
@@ -303,10 +298,11 @@ impl<'tcx> NonConstOp<'tcx> for FnCallNonConst<'tcx> {
                 diag_trait(&mut err, self_ty, tcx.require_lang_item(LangItem::Deref, Some(span)));
                 err
             }
-            _ if tcx.opt_parent(callee) == tcx.get_diagnostic_item(sym::ArgumentV1Methods) => {
-                ccx.tcx.sess.create_err(NonConstFmtMacroCall { span, kind: ccx.const_kind() })
-            }
-            _ => ccx.tcx.sess.create_err(NonConstFnCall {
+            _ if tcx.opt_parent(callee) == tcx.get_diagnostic_item(sym::ArgumentV1Methods) => ccx
+                .tcx
+                .sess
+                .create_err(errors::NonConstFmtMacroCall { span, kind: ccx.const_kind() }),
+            _ => ccx.tcx.sess.create_err(errors::NonConstFnCall {
                 span,
                 def_path_str: ccx.tcx.def_path_str_with_substs(callee, substs),
                 kind: ccx.const_kind(),
@@ -351,7 +347,7 @@ impl<'tcx> NonConstOp<'tcx> for FnCallUnstable {
         let mut err = ccx
             .tcx
             .sess
-            .create_err(UnstableConstFn { span, def_path: ccx.tcx.def_path_str(def_id) });
+            .create_err(errors::UnstableConstFn { span, def_path: ccx.tcx.def_path_str(def_id) });
 
         if ccx.is_const_stable_const_fn() {
             err.help("const-stable functions can only call other const-stable functions");
@@ -387,11 +383,11 @@ impl<'tcx> NonConstOp<'tcx> for Generator {
         let msg = format!("{}s are not allowed in {}s", self.0.descr(), ccx.const_kind());
         if let hir::GeneratorKind::Async(hir::AsyncGeneratorKind::Block) = self.0 {
             ccx.tcx.sess.create_feature_err(
-                UnallowedOpInConstContext { span, msg },
+                errors::UnallowedOpInConstContext { span, msg },
                 sym::const_async_blocks,
             )
         } else {
-            ccx.tcx.sess.create_err(UnallowedOpInConstContext { span, msg })
+            ccx.tcx.sess.create_err(errors::UnallowedOpInConstContext { span, msg })
         }
     }
 }
@@ -404,7 +400,7 @@ impl<'tcx> NonConstOp<'tcx> for HeapAllocation {
         ccx: &ConstCx<'_, 'tcx>,
         span: Span,
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
-        ccx.tcx.sess.create_err(UnallowedHeapAllocations {
+        ccx.tcx.sess.create_err(errors::UnallowedHeapAllocations {
             span,
             kind: ccx.const_kind(),
             teach: ccx.tcx.sess.teach(&error_code!(E0010)).then_some(()),
@@ -420,7 +416,7 @@ impl<'tcx> NonConstOp<'tcx> for InlineAsm {
         ccx: &ConstCx<'_, 'tcx>,
         span: Span,
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
-        ccx.tcx.sess.create_err(UnallowedInlineAsm { span, kind: ccx.const_kind() })
+        ccx.tcx.sess.create_err(errors::UnallowedInlineAsm { span, kind: ccx.const_kind() })
     }
 }
 
@@ -471,7 +467,9 @@ impl<'tcx> NonConstOp<'tcx> for TransientCellBorrow {
         ccx: &ConstCx<'_, 'tcx>,
         span: Span,
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
-        ccx.tcx.sess.create_feature_err(InteriorMutabilityBorrow { span }, sym::const_refs_to_cell)
+        ccx.tcx
+            .sess
+            .create_feature_err(errors::InteriorMutabilityBorrow { span }, sym::const_refs_to_cell)
     }
 }
 
@@ -488,14 +486,14 @@ impl<'tcx> NonConstOp<'tcx> for CellBorrow {
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
         // FIXME: Maybe a more elegant solution to this if else case
         if let hir::ConstContext::Static(_) = ccx.const_kind() {
-            ccx.tcx.sess.create_err(InteriorMutableDataRefer {
+            ccx.tcx.sess.create_err(errors::InteriorMutableDataRefer {
                 span,
                 opt_help: Some(()),
                 kind: ccx.const_kind(),
                 teach: ccx.tcx.sess.teach(&error_code!(E0492)).then_some(()),
             })
         } else {
-            ccx.tcx.sess.create_err(InteriorMutableDataRefer {
+            ccx.tcx.sess.create_err(errors::InteriorMutableDataRefer {
                 span,
                 opt_help: None,
                 kind: ccx.const_kind(),
@@ -528,12 +526,12 @@ impl<'tcx> NonConstOp<'tcx> for MutBorrow {
         span: Span,
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
         match self.0 {
-            hir::BorrowKind::Raw => ccx.tcx.sess.create_err(UnallowedMutableRefsRaw {
+            hir::BorrowKind::Raw => ccx.tcx.sess.create_err(errors::UnallowedMutableRefsRaw {
                 span,
                 kind: ccx.const_kind(),
                 teach: ccx.tcx.sess.teach(&error_code!(E0764)).then_some(()),
             }),
-            hir::BorrowKind::Ref => ccx.tcx.sess.create_err(UnallowedMutableRefs {
+            hir::BorrowKind::Ref => ccx.tcx.sess.create_err(errors::UnallowedMutableRefs {
                 span,
                 kind: ccx.const_kind(),
                 teach: ccx.tcx.sess.teach(&error_code!(E0764)).then_some(()),
@@ -557,14 +555,14 @@ impl<'tcx> NonConstOp<'tcx> for TransientMutBorrow {
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
         let kind = ccx.const_kind();
         match self.0 {
-            hir::BorrowKind::Raw => ccx
-                .tcx
-                .sess
-                .create_feature_err(TransientMutBorrowErrRaw { span, kind }, sym::const_mut_refs),
-            hir::BorrowKind::Ref => ccx
-                .tcx
-                .sess
-                .create_feature_err(TransientMutBorrowErr { span, kind }, sym::const_mut_refs),
+            hir::BorrowKind::Raw => ccx.tcx.sess.create_feature_err(
+                errors::TransientMutBorrowErrRaw { span, kind },
+                sym::const_mut_refs,
+            ),
+            hir::BorrowKind::Ref => ccx.tcx.sess.create_feature_err(
+                errors::TransientMutBorrowErr { span, kind },
+                sym::const_mut_refs,
+            ),
         }
     }
 }
@@ -586,9 +584,10 @@ impl<'tcx> NonConstOp<'tcx> for MutDeref {
         ccx: &ConstCx<'_, 'tcx>,
         span: Span,
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
-        ccx.tcx
-            .sess
-            .create_feature_err(MutDerefErr { span, kind: ccx.const_kind() }, sym::const_mut_refs)
+        ccx.tcx.sess.create_feature_err(
+            errors::MutDerefErr { span, kind: ccx.const_kind() },
+            sym::const_mut_refs,
+        )
     }
 }
 
@@ -601,7 +600,7 @@ impl<'tcx> NonConstOp<'tcx> for PanicNonStr {
         ccx: &ConstCx<'_, 'tcx>,
         span: Span,
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
-        ccx.tcx.sess.create_err(PanicNonStrErr { span })
+        ccx.tcx.sess.create_err(errors::PanicNonStrErr { span })
     }
 }
 
@@ -652,7 +651,7 @@ impl<'tcx> NonConstOp<'tcx> for RawPtrToIntCast {
         ccx: &ConstCx<'_, 'tcx>,
         span: Span,
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
-        ccx.tcx.sess.create_err(RawPtrToIntErr { span })
+        ccx.tcx.sess.create_err(errors::RawPtrToIntErr { span })
     }
 }
 
@@ -673,7 +672,7 @@ impl<'tcx> NonConstOp<'tcx> for StaticAccess {
         ccx: &ConstCx<'_, 'tcx>,
         span: Span,
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
-        ccx.tcx.sess.create_err(StaticAccessErr {
+        ccx.tcx.sess.create_err(errors::StaticAccessErr {
             span,
             kind: ccx.const_kind(),
             teach: ccx.tcx.sess.teach(&error_code!(E0013)).then_some(()),
@@ -690,7 +689,7 @@ impl<'tcx> NonConstOp<'tcx> for ThreadLocalAccess {
         ccx: &ConstCx<'_, 'tcx>,
         span: Span,
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
-        ccx.tcx.sess.create_err(NonConstOpErr { span })
+        ccx.tcx.sess.create_err(errors::NonConstOpErr { span })
     }
 }
 
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs b/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs
index 8ca3fdf40..bb4b7ad50 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs
@@ -217,10 +217,10 @@ impl Qualif for CustomEq {
 
     fn in_adt_inherently<'tcx>(
         cx: &ConstCx<'_, 'tcx>,
-        adt: AdtDef<'tcx>,
+        def: AdtDef<'tcx>,
         substs: SubstsRef<'tcx>,
     ) -> bool {
-        let ty = cx.tcx.mk_ty(ty::Adt(adt, substs));
+        let ty = cx.tcx.mk_adt(def, substs);
         !ty.is_structural_eq_shallow(cx.tcx)
     }
 }
diff --git a/compiler/rustc_const_eval/src/transform/promote_consts.rs b/compiler/rustc_const_eval/src/transform/promote_consts.rs
index fae6117f8..3f3b66b06 100644
--- a/compiler/rustc_const_eval/src/transform/promote_consts.rs
+++ b/compiler/rustc_const_eval/src/transform/promote_consts.rs
@@ -18,7 +18,7 @@ use rustc_middle::mir::traversal::ReversePostorderIter;
 use rustc_middle::mir::visit::{MutVisitor, MutatingUseContext, PlaceContext, Visitor};
 use rustc_middle::mir::*;
 use rustc_middle::ty::subst::InternalSubsts;
-use rustc_middle::ty::{self, List, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, List, TyCtxt, TypeVisitableExt};
 use rustc_span::Span;
 
 use rustc_index::vec::{Idx, IndexVec};
@@ -364,31 +364,33 @@ impl<'tcx> Validator<'_, 'tcx> {
                     ProjectionElem::Index(local) => {
                         let mut promotable = false;
                         // Only accept if we can predict the index and are indexing an array.
-                        let val =
-                            if let TempState::Defined { location: loc, .. } = self.temps[local] {
-                                let block = &self.body[loc.block];
-                                if loc.statement_index < block.statements.len() {
-                                    let statement = &block.statements[loc.statement_index];
-                                    match &statement.kind {
-                                        StatementKind::Assign(box (
-                                            _,
-                                            Rvalue::Use(Operand::Constant(c)),
-                                        )) => c.literal.try_eval_usize(self.tcx, self.param_env),
-                                        _ => None,
-                                    }
-                                } else {
-                                    None
+                        let val = if let TempState::Defined { location: loc, .. } =
+                            self.temps[local]
+                        {
+                            let block = &self.body[loc.block];
+                            if loc.statement_index < block.statements.len() {
+                                let statement = &block.statements[loc.statement_index];
+                                match &statement.kind {
+                                    StatementKind::Assign(box (
+                                        _,
+                                        Rvalue::Use(Operand::Constant(c)),
+                                    )) => c.literal.try_eval_target_usize(self.tcx, self.param_env),
+                                    _ => None,
                                 }
                             } else {
                                 None
-                            };
+                            }
+                        } else {
+                            None
+                        };
                         if let Some(idx) = val {
                             // Determine the type of the thing we are indexing.
                             let ty = place_base.ty(self.body, self.tcx).ty;
                             match ty.kind() {
                                 ty::Array(_, len) => {
                                     // It's an array; determine its length.
-                                    if let Some(len) = len.try_eval_usize(self.tcx, self.param_env)
+                                    if let Some(len) =
+                                        len.try_eval_target_usize(self.tcx, self.param_env)
                                     {
                                         // If the index is in-bounds, go ahead.
                                         if idx < len {
@@ -470,7 +472,7 @@ impl<'tcx> Validator<'_, 'tcx> {
                 // mutably without consequences. However, only &mut []
                 // is allowed right now.
                 if let ty::Array(_, len) = ty.kind() {
-                    match len.try_eval_usize(self.tcx, self.param_env) {
+                    match len.try_eval_target_usize(self.tcx, self.param_env) {
                         Some(0) => {}
                         _ => return Err(Unpromotable),
                     }
@@ -864,7 +866,7 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> {
 
             let mut projection = vec![PlaceElem::Deref];
             projection.extend(place.projection);
-            place.projection = tcx.intern_place_elems(&projection);
+            place.projection = tcx.mk_place_elems(&projection);
 
             // Create a temp to hold the promoted reference.
             // This is because `*r` requires `r` to be a local,
@@ -896,7 +898,7 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> {
         assert_eq!(self.new_block(), START_BLOCK);
         self.visit_rvalue(
             &mut rvalue,
-            Location { block: BasicBlock::new(0), statement_index: usize::MAX },
+            Location { block: START_BLOCK, statement_index: usize::MAX },
         );
 
         let span = self.promoted.span;
diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs
index dd168a9ac..fb37eb79a 100644
--- a/compiler/rustc_const_eval/src/transform/validate.rs
+++ b/compiler/rustc_const_eval/src/transform/validate.rs
@@ -8,12 +8,12 @@ use rustc_middle::mir::interpret::Scalar;
 use rustc_middle::mir::visit::NonUseContext::VarDebugInfo;
 use rustc_middle::mir::visit::{PlaceContext, Visitor};
 use rustc_middle::mir::{
-    traversal, AggregateKind, BasicBlock, BinOp, Body, BorrowKind, CastKind, CopyNonOverlapping,
-    Local, Location, MirPass, MirPhase, NonDivergingIntrinsic, Operand, Place, PlaceElem, PlaceRef,
-    ProjectionElem, RetagKind, RuntimePhase, Rvalue, SourceScope, Statement, StatementKind,
-    Terminator, TerminatorKind, UnOp, START_BLOCK,
+    traversal, BasicBlock, BinOp, Body, BorrowKind, CastKind, CopyNonOverlapping, Local, Location,
+    MirPass, MirPhase, NonDivergingIntrinsic, Operand, Place, PlaceElem, PlaceRef, ProjectionElem,
+    RetagKind, RuntimePhase, Rvalue, SourceScope, Statement, StatementKind, Terminator,
+    TerminatorKind, UnOp, START_BLOCK,
 };
-use rustc_middle::ty::{self, InstanceDef, ParamEnv, Ty, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, InstanceDef, ParamEnv, Ty, TyCtxt, TypeVisitableExt};
 use rustc_mir_dataflow::impls::MaybeStorageLive;
 use rustc_mir_dataflow::storage::always_storage_live_locals;
 use rustc_mir_dataflow::{Analysis, ResultsCursor};
@@ -230,8 +230,12 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             // Equal types, all is good.
             return true;
         }
-        // Normalization reveals opaque types, but we may be validating MIR while computing
-        // said opaque types, causing cycles.
+
+        // We sometimes have to use `defining_opaque_types` for subtyping
+        // to succeed here and figuring out how exactly that should work
+        // is annoying. It is harmless enough to just not validate anything
+        // in that case. We still check this after analysis as all opque
+        // types have been revealed at this point.
         if (src, dest).has_opaque_types() {
             return true;
         }
@@ -311,7 +315,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                 }
             }
             ProjectionElem::Field(f, ty) => {
-                let parent = Place { local, projection: self.tcx.intern_place_elems(proj_base) };
+                let parent = Place { local, projection: self.tcx.mk_place_elems(proj_base) };
                 let parent_ty = parent.ty(&self.body.local_decls, self.tcx);
                 let fail_out_of_bounds = |this: &Self, location| {
                     this.fail(location, format!("Out of bounds field {:?} for {:?}", f, parent_ty));
@@ -330,7 +334,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
 
                 let kind = match parent_ty.ty.kind() {
                     &ty::Alias(ty::Opaque, ty::AliasTy { def_id, substs, .. }) => {
-                        self.tcx.bound_type_of(def_id).subst(self.tcx, substs).kind()
+                        self.tcx.type_of(def_id).subst(self.tcx, substs).kind()
                     }
                     kind => kind,
                 };
@@ -377,12 +381,12 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                                 return;
                             };
 
-                            let Some(&f_ty) = layout.field_tys.get(local) else {
+                            let Some(f_ty) = layout.field_tys.get(local) else {
                                 self.fail(location, format!("Out of bounds local {:?} for {:?}", local, parent_ty));
                                 return;
                             };
 
-                            f_ty
+                            f_ty.ty
                         } else {
                             let Some(f_ty) = substs.as_generator().prefix_tys().nth(f.index()) else {
                                 fail_out_of_bounds(self, location);
@@ -428,19 +432,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
             };
         }
         match rvalue {
-            Rvalue::Use(_) | Rvalue::CopyForDeref(_) => {}
-            Rvalue::Aggregate(agg_kind, _) => {
-                let disallowed = match **agg_kind {
-                    AggregateKind::Array(..) => false,
-                    _ => self.mir_phase >= MirPhase::Runtime(RuntimePhase::PostCleanup),
-                };
-                if disallowed {
-                    self.fail(
-                        location,
-                        format!("{:?} have been lowered to field assignments", rvalue),
-                    )
-                }
-            }
+            Rvalue::Use(_) | Rvalue::CopyForDeref(_) | Rvalue::Aggregate(..) => {}
             Rvalue::Ref(_, BorrowKind::Shallow, _) => {
                 if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {
                     self.fail(
@@ -759,13 +751,32 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                 // FIXME(JakobDegen) The validator should check that `self.mir_phase <
                 // DropsLowered`. However, this causes ICEs with generation of drop shims, which
                 // seem to fail to set their `MirPhase` correctly.
-                if *kind == RetagKind::Raw || *kind == RetagKind::TwoPhase {
+                if matches!(kind, RetagKind::Raw | RetagKind::TwoPhase) {
                     self.fail(location, format!("explicit `{:?}` is forbidden", kind));
                 }
             }
-            StatementKind::StorageLive(..)
-            | StatementKind::StorageDead(..)
+            StatementKind::StorageLive(local) => {
+                // We check that the local is not live when entering a `StorageLive` for it.
+                // Technically, violating this restriction is only UB and not actually indicative
+                // of not well-formed MIR. This means that an optimization which turns MIR that
+                // already has UB into MIR that fails this check is not necessarily wrong. However,
+                // we have no such optimizations at the moment, and so we include this check anyway
+                // to help us catch bugs. If you happen to write an optimization that might cause
+                // this to incorrectly fire, feel free to remove this check.
+                if self.reachable_blocks.contains(location.block) {
+                    self.storage_liveness.seek_before_primary_effect(location);
+                    let locals_with_storage = self.storage_liveness.get();
+                    if locals_with_storage.contains(*local) {
+                        self.fail(
+                            location,
+                            format!("StorageLive({local:?}) which already has storage here"),
+                        );
+                    }
+                }
+            }
+            StatementKind::StorageDead(_)
             | StatementKind::Coverage(_)
+            | StatementKind::ConstEvalCounter
             | StatementKind::Nop => {}
         }
 
diff --git a/compiler/rustc_const_eval/src/util/aggregate.rs b/compiler/rustc_const_eval/src/util/aggregate.rs
deleted file mode 100644
index 10783c5ed..000000000
--- a/compiler/rustc_const_eval/src/util/aggregate.rs
+++ /dev/null
@@ -1,77 +0,0 @@
-use rustc_index::vec::Idx;
-use rustc_middle::mir::*;
-use rustc_middle::ty::{Ty, TyCtxt};
-use rustc_target::abi::VariantIdx;
-
-use std::iter::TrustedLen;
-
-/// Expand `lhs = Rvalue::Aggregate(kind, operands)` into assignments to the fields.
-///
-/// Produces something like
-/// ```ignore (ilustrative)
-/// (lhs as Variant).field0 = arg0;     // We only have a downcast if this is an enum
-/// (lhs as Variant).field1 = arg1;
-/// discriminant(lhs) = variant_index;  // If lhs is an enum or generator.
-/// ```
-pub fn expand_aggregate<'tcx>(
-    orig_lhs: Place<'tcx>,
-    operands: impl Iterator<Item = (Operand<'tcx>, Ty<'tcx>)> + TrustedLen,
-    kind: AggregateKind<'tcx>,
-    source_info: SourceInfo,
-    tcx: TyCtxt<'tcx>,
-) -> impl Iterator<Item = Statement<'tcx>> + TrustedLen {
-    let mut lhs = orig_lhs;
-    let mut set_discriminant = None;
-    let active_field_index = match kind {
-        AggregateKind::Adt(adt_did, variant_index, _, _, active_field_index) => {
-            let adt_def = tcx.adt_def(adt_did);
-            if adt_def.is_enum() {
-                set_discriminant = Some(Statement {
-                    kind: StatementKind::SetDiscriminant {
-                        place: Box::new(orig_lhs),
-                        variant_index,
-                    },
-                    source_info,
-                });
-                lhs = tcx.mk_place_downcast(orig_lhs, adt_def, variant_index);
-            }
-            active_field_index
-        }
-        AggregateKind::Generator(..) => {
-            // Right now we only support initializing generators to
-            // variant 0 (Unresumed).
-            let variant_index = VariantIdx::new(0);
-            set_discriminant = Some(Statement {
-                kind: StatementKind::SetDiscriminant { place: Box::new(orig_lhs), variant_index },
-                source_info,
-            });
-
-            // Operands are upvars stored on the base place, so no
-            // downcast is necessary.
-
-            None
-        }
-        _ => None,
-    };
-
-    let operands = operands.enumerate().map(move |(i, (op, ty))| {
-        let lhs_field = if let AggregateKind::Array(_) = kind {
-            let offset = u64::try_from(i).unwrap();
-            tcx.mk_place_elem(
-                lhs,
-                ProjectionElem::ConstantIndex { offset, min_length: offset + 1, from_end: false },
-            )
-        } else {
-            let field = Field::new(active_field_index.unwrap_or(i));
-            tcx.mk_place_field(lhs, field, ty)
-        };
-        Statement {
-            source_info,
-            kind: StatementKind::Assign(Box::new((lhs_field, Rvalue::Use(op)))),
-        }
-    });
-    [Statement { source_info, kind: StatementKind::Deinit(Box::new(orig_lhs)) }]
-        .into_iter()
-        .chain(operands)
-        .chain(set_discriminant)
-}
diff --git a/compiler/rustc_const_eval/src/util/call_kind.rs b/compiler/rustc_const_eval/src/util/call_kind.rs
index 38d9b0449..995363c0e 100644
--- a/compiler/rustc_const_eval/src/util/call_kind.rs
+++ b/compiler/rustc_const_eval/src/util/call_kind.rs
@@ -40,6 +40,7 @@ pub enum CallKind<'tcx> {
         self_arg: Option<Ident>,
         desugaring: Option<(CallDesugaringKind, Ty<'tcx>)>,
         method_did: DefId,
+        method_substs: SubstsRef<'tcx>,
     },
     /// A call to `Fn(..)::call(..)`, desugared from `my_closure(a, b, c)`
     FnCall { fn_trait_id: DefId, self_ty: Ty<'tcx> },
@@ -131,6 +132,6 @@ pub fn call_kind<'tcx>(
         } else {
             None
         };
-        CallKind::Normal { self_arg, desugaring, method_did }
+        CallKind::Normal { self_arg, desugaring, method_did, method_substs }
     })
 }
diff --git a/compiler/rustc_const_eval/src/util/might_permit_raw_init.rs b/compiler/rustc_const_eval/src/util/check_validity_requirement.rs
index 4ce107ea6..23fcd22c5 100644
--- a/compiler/rustc_const_eval/src/util/might_permit_raw_init.rs
+++ b/compiler/rustc_const_eval/src/util/check_validity_requirement.rs
@@ -1,7 +1,7 @@
-use rustc_middle::ty::layout::{LayoutCx, LayoutOf, TyAndLayout};
-use rustc_middle::ty::{ParamEnv, TyCtxt};
+use rustc_middle::ty::layout::{LayoutCx, LayoutError, LayoutOf, TyAndLayout, ValidityRequirement};
+use rustc_middle::ty::{ParamEnv, ParamEnvAnd, Ty, TyCtxt};
 use rustc_session::Limit;
-use rustc_target::abi::{Abi, FieldsShape, InitKind, Scalar, Variants};
+use rustc_target::abi::{Abi, FieldsShape, Scalar, Variants};
 
 use crate::const_eval::{CheckAlignment, CompileTimeInterpreter};
 use crate::interpret::{InterpCx, MemoryKind, OpTy};
@@ -18,16 +18,23 @@ use crate::interpret::{InterpCx, MemoryKind, OpTy};
 /// Rust UB as long as there is no risk of miscompilations. The `strict_init_checks` can be set to
 /// do a full check against Rust UB instead (in which case we will also ignore the 0x01-filling and
 /// to the full uninit check).
-pub fn might_permit_raw_init<'tcx>(
+pub fn check_validity_requirement<'tcx>(
     tcx: TyCtxt<'tcx>,
-    ty: TyAndLayout<'tcx>,
-    kind: InitKind,
-) -> bool {
-    if tcx.sess.opts.unstable_opts.strict_init_checks {
-        might_permit_raw_init_strict(ty, tcx, kind)
+    kind: ValidityRequirement,
+    param_env_and_ty: ParamEnvAnd<'tcx, Ty<'tcx>>,
+) -> Result<bool, LayoutError<'tcx>> {
+    let layout = tcx.layout_of(param_env_and_ty)?;
+
+    // There is nothing strict or lax about inhabitedness.
+    if kind == ValidityRequirement::Inhabited {
+        return Ok(!layout.abi.is_uninhabited());
+    }
+
+    if kind == ValidityRequirement::Uninit || tcx.sess.opts.unstable_opts.strict_init_checks {
+        might_permit_raw_init_strict(layout, tcx, kind)
     } else {
-        let layout_cx = LayoutCx { tcx, param_env: ParamEnv::reveal_all() };
-        might_permit_raw_init_lax(ty, &layout_cx, kind)
+        let layout_cx = LayoutCx { tcx, param_env: param_env_and_ty.param_env };
+        might_permit_raw_init_lax(layout, &layout_cx, kind)
     }
 }
 
@@ -36,8 +43,8 @@ pub fn might_permit_raw_init<'tcx>(
 fn might_permit_raw_init_strict<'tcx>(
     ty: TyAndLayout<'tcx>,
     tcx: TyCtxt<'tcx>,
-    kind: InitKind,
-) -> bool {
+    kind: ValidityRequirement,
+) -> Result<bool, LayoutError<'tcx>> {
     let machine = CompileTimeInterpreter::new(
         Limit::new(0),
         /*can_access_statics:*/ false,
@@ -50,7 +57,7 @@ fn might_permit_raw_init_strict<'tcx>(
         .allocate(ty, MemoryKind::Machine(crate::const_eval::MemoryKind::Heap))
         .expect("OOM: failed to allocate for uninit check");
 
-    if kind == InitKind::Zero {
+    if kind == ValidityRequirement::Zero {
         cx.write_bytes_ptr(
             allocated.ptr,
             std::iter::repeat(0_u8).take(ty.layout.size().bytes_usize()),
@@ -64,7 +71,7 @@ fn might_permit_raw_init_strict<'tcx>(
     // This does *not* actually check that references are dereferenceable, but since all types that
     // require dereferenceability also require non-null, we don't actually get any false negatives
     // due to this.
-    cx.validate_operand(&ot).is_ok()
+    Ok(cx.validate_operand(&ot).is_ok())
 }
 
 /// Implements the 'lax' (default) version of the `might_permit_raw_init` checks; see that function for
@@ -72,15 +79,18 @@ fn might_permit_raw_init_strict<'tcx>(
 fn might_permit_raw_init_lax<'tcx>(
     this: TyAndLayout<'tcx>,
     cx: &LayoutCx<'tcx, TyCtxt<'tcx>>,
-    init_kind: InitKind,
-) -> bool {
+    init_kind: ValidityRequirement,
+) -> Result<bool, LayoutError<'tcx>> {
     let scalar_allows_raw_init = move |s: Scalar| -> bool {
         match init_kind {
-            InitKind::Zero => {
+            ValidityRequirement::Inhabited => {
+                bug!("ValidityRequirement::Inhabited should have been handled above")
+            }
+            ValidityRequirement::Zero => {
                 // The range must contain 0.
                 s.valid_range(cx).contains(0)
             }
-            InitKind::UninitMitigated0x01Fill => {
+            ValidityRequirement::UninitMitigated0x01Fill => {
                 // The range must include an 0x01-filled buffer.
                 let mut val: u128 = 0x01;
                 for _ in 1..s.size(cx).bytes() {
@@ -89,6 +99,9 @@ fn might_permit_raw_init_lax<'tcx>(
                 }
                 s.valid_range(cx).contains(val)
             }
+            ValidityRequirement::Uninit => {
+                bug!("ValidityRequirement::Uninit should have been handled above")
+            }
         }
     };
 
@@ -102,20 +115,20 @@ fn might_permit_raw_init_lax<'tcx>(
     };
     if !valid {
         // This is definitely not okay.
-        return false;
+        return Ok(false);
     }
 
     // Special magic check for references and boxes (i.e., special pointer types).
     if let Some(pointee) = this.ty.builtin_deref(false) {
-        let pointee = cx.layout_of(pointee.ty).expect("need to be able to compute layouts");
+        let pointee = cx.layout_of(pointee.ty)?;
         // We need to ensure that the LLVM attributes `aligned` and `dereferenceable(size)` are satisfied.
         if pointee.align.abi.bytes() > 1 {
             // 0x01-filling is not aligned.
-            return false;
+            return Ok(false);
         }
         if pointee.size.bytes() > 0 {
             // A 'fake' integer pointer is not sufficiently dereferenceable.
-            return false;
+            return Ok(false);
         }
     }
 
@@ -128,9 +141,9 @@ fn might_permit_raw_init_lax<'tcx>(
         }
         FieldsShape::Arbitrary { offsets, .. } => {
             for idx in 0..offsets.len() {
-                if !might_permit_raw_init_lax(this.field(cx, idx), cx, init_kind) {
+                if !might_permit_raw_init_lax(this.field(cx, idx), cx, init_kind)? {
                     // We found a field that is unhappy with this kind of initialization.
-                    return false;
+                    return Ok(false);
                 }
             }
         }
@@ -147,5 +160,5 @@ fn might_permit_raw_init_lax<'tcx>(
         }
     }
 
-    true
+    Ok(true)
 }
diff --git a/compiler/rustc_const_eval/src/util/mod.rs b/compiler/rustc_const_eval/src/util/mod.rs
index 76ea5a24e..c0aabd77c 100644
--- a/compiler/rustc_const_eval/src/util/mod.rs
+++ b/compiler/rustc_const_eval/src/util/mod.rs
@@ -1,16 +1,14 @@
-pub mod aggregate;
 mod alignment;
 mod call_kind;
+mod check_validity_requirement;
 pub mod collect_writes;
 mod compare_types;
 mod find_self_call;
-mod might_permit_raw_init;
 mod type_name;
 
-pub use self::aggregate::expand_aggregate;
 pub use self::alignment::is_disaligned;
 pub use self::call_kind::{call_kind, CallDesugaringKind, CallKind};
+pub use self::check_validity_requirement::check_validity_requirement;
 pub use self::compare_types::{is_equal_up_to_subtyping, is_subtype};
 pub use self::find_self_call::find_self_call;
-pub use self::might_permit_raw_init::might_permit_raw_init;
 pub use self::type_name::type_name;
diff --git a/compiler/rustc_const_eval/src/util/type_name.rs b/compiler/rustc_const_eval/src/util/type_name.rs
index c4122f664..4e80a2851 100644
--- a/compiler/rustc_const_eval/src/util/type_name.rs
+++ b/compiler/rustc_const_eval/src/util/type_name.rs
@@ -64,6 +64,7 @@ impl<'tcx> Printer<'tcx> for AbsolutePathPrinter<'tcx> {
             ty::Foreign(def_id) => self.print_def_path(def_id, &[]),
 
             ty::GeneratorWitness(_) => bug!("type_name: unexpected `GeneratorWitness`"),
+            ty::GeneratorWitnessMIR(..) => bug!("type_name: unexpected `GeneratorWitnessMIR`"),
         }
     }
 
diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml
index 0366fb0a1..29cb2c0a3 100644
--- a/compiler/rustc_data_structures/Cargo.toml
+++ b/compiler/rustc_data_structures/Cargo.toml
@@ -9,7 +9,7 @@ edition = "2021"
 arrayvec = { version = "0.7", default-features = false }
 bitflags = "1.2.1"
 cfg-if = "1.0"
-ena = "0.14"
+ena = "0.14.1"
 indexmap = { version = "1.9.1" }
 jobserver_crate = { version = "0.1.13", package = "jobserver" }
 libc = "0.2"
@@ -29,8 +29,9 @@ smallvec = { version = "1.8.1", features = [
 stable_deref_trait = "1.0.0"
 stacker = "0.1.15"
 tempfile = "3.2"
-thin-vec = "0.2.9"
+thin-vec = "0.2.12"
 tracing = "0.1"
+elsa = "1.8"
 
 [dependencies.parking_lot]
 version = "0.11"
diff --git a/compiler/rustc_data_structures/src/functor.rs b/compiler/rustc_data_structures/src/functor.rs
index 84cb417dd..28fcf80b3 100644
--- a/compiler/rustc_data_structures/src/functor.rs
+++ b/compiler/rustc_data_structures/src/functor.rs
@@ -1,5 +1,5 @@
 use rustc_index::vec::{Idx, IndexVec};
-use std::mem;
+use std::{mem, rc::Rc, sync::Arc};
 
 pub trait IdFunctor: Sized {
     type Inner;
@@ -65,3 +65,52 @@ impl<I: Idx, T> IdFunctor for IndexVec<I, T> {
         self.raw.try_map_id(f).map(IndexVec::from_raw)
     }
 }
+
+macro_rules! rc {
+    ($($rc:ident),+) => {$(
+        impl<T: Clone> IdFunctor for $rc<T> {
+            type Inner = T;
+
+            #[inline]
+            fn try_map_id<F, E>(mut self, mut f: F) -> Result<Self, E>
+            where
+                F: FnMut(Self::Inner) -> Result<Self::Inner, E>,
+            {
+                // We merely want to replace the contained `T`, if at all possible,
+                // so that we don't needlessly allocate a new `$rc` or indeed clone
+                // the contained type.
+                unsafe {
+                    // First step is to ensure that we have a unique reference to
+                    // the contained type, which `$rc::make_mut` will accomplish (by
+                    // allocating a new `$rc` and cloning the `T` only if required).
+                    // This is done *before* casting to `$rc<ManuallyDrop<T>>` so that
+                    // panicking during `make_mut` does not leak the `T`.
+                    $rc::make_mut(&mut self);
+
+                    // Casting to `$rc<ManuallyDrop<T>>` is safe because `ManuallyDrop`
+                    // is `repr(transparent)`.
+                    let ptr = $rc::into_raw(self).cast::<mem::ManuallyDrop<T>>();
+                    let mut unique = $rc::from_raw(ptr);
+
+                    // Call to `$rc::make_mut` above guarantees that `unique` is the
+                    // sole reference to the contained value, so we can avoid doing
+                    // a checked `get_mut` here.
+                    let slot = $rc::get_mut_unchecked(&mut unique);
+
+                    // Semantically move the contained type out from `unique`, fold
+                    // it, then move the folded value back into `unique`. Should
+                    // folding fail, `ManuallyDrop` ensures that the "moved-out"
+                    // value is not re-dropped.
+                    let owned = mem::ManuallyDrop::take(slot);
+                    let folded = f(owned)?;
+                    *slot = mem::ManuallyDrop::new(folded);
+
+                    // Cast back to `$rc<T>`.
+                    Ok($rc::from_raw($rc::into_raw(unique).cast()))
+                }
+            }
+        }
+    )+};
+}
+
+rc! { Rc, Arc }
diff --git a/compiler/rustc_data_structures/src/graph/dominators/mod.rs b/compiler/rustc_data_structures/src/graph/dominators/mod.rs
index 471457f61..0a21a4249 100644
--- a/compiler/rustc_data_structures/src/graph/dominators/mod.rs
+++ b/compiler/rustc_data_structures/src/graph/dominators/mod.rs
@@ -135,7 +135,47 @@ pub fn dominators<G: ControlFlowGraph>(graph: G) -> Dominators<G::Node> {
         // This loop computes the semi[w] for w.
         semi[w] = w;
         for v in graph.predecessors(pre_order_to_real[w]) {
-            // Reachable vertices may have unreachable predecessors, so ignore any of them
+            // TL;DR: Reachable vertices may have unreachable predecessors, so ignore any of them.
+            //
+            // Ignore blocks which are not connected to the entry block.
+            //
+            // The algorithm that was used to traverse the graph and build the
+            // `pre_order_to_real` and `real_to_pre_order` vectors does so by
+            // starting from the entry block and following the successors.
+            // Therefore, any blocks not reachable from the entry block will be
+            // set to `None` in the `pre_order_to_real` vector.
+            //
+            // For example, in this graph, A and B should be skipped:
+            //
+            //           ┌─────┐
+            //           │     │
+            //           └──┬──┘
+            //              │
+            //           ┌──▼──┐              ┌─────┐
+            //           │     │              │  A  │
+            //           └──┬──┘              └──┬──┘
+            //              │                    │
+            //      ┌───────┴───────┐            │
+            //      │               │            │
+            //   ┌──▼──┐         ┌──▼──┐      ┌──▼──┐
+            //   │     │         │     │      │  B  │
+            //   └──┬──┘         └──┬──┘      └──┬──┘
+            //      │               └──────┬─────┘
+            //   ┌──▼──┐                   │
+            //   │     │                   │
+            //   └──┬──┘                ┌──▼──┐
+            //      │                   │     │
+            //      │                   └─────┘
+            //   ┌──▼──┐
+            //   │     │
+            //   └──┬──┘
+            //      │
+            //   ┌──▼──┐
+            //   │     │
+            //   └─────┘
+            //
+            // ...this may be the case if a MirPass modifies the CFG to remove
+            // or rearrange certain blocks/edges.
             let Some(v) = real_to_pre_order[v] else {
                 continue
             };
@@ -264,13 +304,18 @@ fn compress(
     }
 }
 
+/// Tracks the list of dominators for each node.
 #[derive(Clone, Debug)]
 pub struct Dominators<N: Idx> {
     post_order_rank: IndexVec<N, usize>,
+    // Even though we track only the immediate dominator of each node, it's
+    // possible to get its full list of dominators by looking up the dominator
+    // of each dominator. (See the `impl Iterator for Iter` definition).
     immediate_dominators: IndexVec<N, Option<N>>,
 }
 
 impl<Node: Idx> Dominators<Node> {
+    /// Whether the given Node has an immediate dominator.
     pub fn is_reachable(&self, node: Node) -> bool {
         self.immediate_dominators[node].is_some()
     }
@@ -280,12 +325,14 @@ impl<Node: Idx> Dominators<Node> {
         self.immediate_dominators[node].unwrap()
     }
 
+    /// Provides an iterator over each dominator up the CFG, for the given Node.
+    /// See the `impl Iterator for Iter` definition to understand how this works.
     pub fn dominators(&self, node: Node) -> Iter<'_, Node> {
         assert!(self.is_reachable(node), "node {node:?} is not reachable");
         Iter { dominators: self, node: Some(node) }
     }
 
-    pub fn is_dominated_by(&self, node: Node, dom: Node) -> bool {
+    pub fn dominates(&self, dom: Node, node: Node) -> bool {
         // FIXME -- could be optimized by using post-order-rank
         self.dominators(node).any(|n| n == dom)
     }
diff --git a/compiler/rustc_data_structures/src/graph/scc/mod.rs b/compiler/rustc_data_structures/src/graph/scc/mod.rs
index c8e66eb67..c4b11951a 100644
--- a/compiler/rustc_data_structures/src/graph/scc/mod.rs
+++ b/compiler/rustc_data_structures/src/graph/scc/mod.rs
@@ -27,7 +27,7 @@ pub struct Sccs<N: Idx, S: Idx> {
     scc_data: SccData<S>,
 }
 
-struct SccData<S: Idx> {
+pub struct SccData<S: Idx> {
     /// For each SCC, the range of `all_successors` where its
     /// successors can be found.
     ranges: IndexVec<S, Range<usize>>,
@@ -43,6 +43,14 @@ impl<N: Idx, S: Idx + Ord> Sccs<N, S> {
         SccsConstruction::construct(graph)
     }
 
+    pub fn scc_indices(&self) -> &IndexVec<N, S> {
+        &self.scc_indices
+    }
+
+    pub fn scc_data(&self) -> &SccData<S> {
+        &self.scc_data
+    }
+
     /// Returns the number of SCCs in the graph.
     pub fn num_sccs(&self) -> usize {
         self.scc_data.len()
@@ -115,6 +123,14 @@ impl<S: Idx> SccData<S> {
         self.ranges.len()
     }
 
+    pub fn ranges(&self) -> &IndexVec<S, Range<usize>> {
+        &self.ranges
+    }
+
+    pub fn all_successors(&self) -> &Vec<S> {
+        &self.all_successors
+    }
+
     /// Returns the successors of the given SCC.
     fn successors(&self, scc: S) -> &[S] {
         // Annoyingly, `range` does not implement `Copy`, so we have
diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs
index 954e84c30..a94e52fdf 100644
--- a/compiler/rustc_data_structures/src/lib.rs
+++ b/compiler/rustc_data_structures/src/lib.rs
@@ -11,6 +11,7 @@
 #![feature(associated_type_bounds)]
 #![feature(auto_traits)]
 #![feature(cell_leak)]
+#![feature(core_intrinsics)]
 #![feature(extend_one)]
 #![feature(hash_raw_entry)]
 #![feature(hasher_prefixfree_extras)]
@@ -25,6 +26,7 @@
 #![feature(test)]
 #![feature(thread_id_value)]
 #![feature(vec_into_raw_parts)]
+#![feature(get_mut_unchecked)]
 #![allow(rustc::default_hash_types)]
 #![allow(rustc::potential_query_instability)]
 #![deny(rustc::untranslatable_diagnostic)]
diff --git a/compiler/rustc_data_structures/src/obligation_forest/mod.rs b/compiler/rustc_data_structures/src/obligation_forest/mod.rs
index 10e673cd9..91abdaada 100644
--- a/compiler/rustc_data_structures/src/obligation_forest/mod.rs
+++ b/compiler/rustc_data_structures/src/obligation_forest/mod.rs
@@ -95,10 +95,7 @@ pub trait ForestObligation: Clone + Debug {
 pub trait ObligationProcessor {
     type Obligation: ForestObligation;
     type Error: Debug;
-    type OUT: OutcomeTrait<
-        Obligation = Self::Obligation,
-        Error = Error<Self::Obligation, Self::Error>,
-    >;
+    type OUT: OutcomeTrait<Obligation = Self::Obligation, Error = Error<Self::Obligation, Self::Error>>;
 
     fn needs_process_obligation(&self, obligation: &Self::Obligation) -> bool;
 
@@ -139,8 +136,7 @@ pub enum ProcessResult<O, E> {
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
 struct ObligationTreeId(usize);
 
-type ObligationTreeIdGenerator =
-    std::iter::Map<std::ops::RangeFrom<usize>, fn(usize) -> ObligationTreeId>;
+type ObligationTreeIdGenerator = impl Iterator<Item = ObligationTreeId>;
 
 pub struct ObligationForest<O: ForestObligation> {
     /// The list of obligations. In between calls to [Self::process_obligations],
@@ -430,6 +426,7 @@ impl<O: ForestObligation> ObligationForest<O> {
             // nodes. Therefore we use a `while` loop.
             let mut index = 0;
             while let Some(node) = self.nodes.get_mut(index) {
+                // This test is extremely hot.
                 if node.state.get() != NodeState::Pending
                     || !processor.needs_process_obligation(&node.obligation)
                 {
@@ -443,6 +440,7 @@ impl<O: ForestObligation> ObligationForest<O> {
                 // out of sync with `nodes`. It's not very common, but it does
                 // happen, and code in `compress` has to allow for it.
 
+                // This code is much less hot.
                 match processor.process_obligation(&mut node.obligation) {
                     ProcessResult::Unchanged => {
                         // No change in state.
diff --git a/compiler/rustc_data_structures/src/profiling.rs b/compiler/rustc_data_structures/src/profiling.rs
index 393f17390..443316836 100644
--- a/compiler/rustc_data_structures/src/profiling.rs
+++ b/compiler/rustc_data_structures/src/profiling.rs
@@ -88,6 +88,7 @@ use std::borrow::Borrow;
 use std::collections::hash_map::Entry;
 use std::error::Error;
 use std::fs;
+use std::intrinsics::unlikely;
 use std::path::Path;
 use std::process;
 use std::sync::Arc;
@@ -206,8 +207,7 @@ impl SelfProfilerRef {
     /// a measureme event, "verbose" generic activities also print a timing entry to
     /// stderr if the compiler is invoked with -Ztime-passes.
     pub fn verbose_generic_activity(&self, event_label: &'static str) -> VerboseTimingGuard<'_> {
-        let message =
-            if self.print_verbose_generic_activities { Some(event_label.to_owned()) } else { None };
+        let message = self.print_verbose_generic_activities.then(|| event_label.to_owned());
 
         VerboseTimingGuard::start(message, self.generic_activity(event_label))
     }
@@ -221,11 +221,9 @@ impl SelfProfilerRef {
     where
         A: Borrow<str> + Into<String>,
     {
-        let message = if self.print_verbose_generic_activities {
-            Some(format!("{}({})", event_label, event_arg.borrow()))
-        } else {
-            None
-        };
+        let message = self
+            .print_verbose_generic_activities
+            .then(|| format!("{}({})", event_label, event_arg.borrow()));
 
         VerboseTimingGuard::start(message, self.generic_activity_with_arg(event_label, event_arg))
     }
@@ -395,11 +393,18 @@ impl SelfProfilerRef {
     /// Record a query in-memory cache hit.
     #[inline(always)]
     pub fn query_cache_hit(&self, query_invocation_id: QueryInvocationId) {
-        self.instant_query_event(
-            |profiler| profiler.query_cache_hit_event_kind,
-            query_invocation_id,
-            EventFilter::QUERY_CACHE_HITS,
-        );
+        #[inline(never)]
+        #[cold]
+        fn cold_call(profiler_ref: &SelfProfilerRef, query_invocation_id: QueryInvocationId) {
+            profiler_ref.instant_query_event(
+                |profiler| profiler.query_cache_hit_event_kind,
+                query_invocation_id,
+            );
+        }
+
+        if unlikely(self.event_filter_mask.contains(EventFilter::QUERY_CACHE_HITS)) {
+            cold_call(self, query_invocation_id);
+        }
     }
 
     /// Start profiling a query being blocked on a concurrent execution.
@@ -444,20 +449,15 @@ impl SelfProfilerRef {
         &self,
         event_kind: fn(&SelfProfiler) -> StringId,
         query_invocation_id: QueryInvocationId,
-        event_filter: EventFilter,
     ) {
-        drop(self.exec(event_filter, |profiler| {
-            let event_id = StringId::new_virtual(query_invocation_id.0);
-            let thread_id = get_thread_id();
-
-            profiler.profiler.record_instant_event(
-                event_kind(profiler),
-                EventId::from_virtual(event_id),
-                thread_id,
-            );
-
-            TimingGuard::none()
-        }));
+        let event_id = StringId::new_virtual(query_invocation_id.0);
+        let thread_id = get_thread_id();
+        let profiler = self.profiler.as_ref().unwrap();
+        profiler.profiler.record_instant_event(
+            event_kind(profiler),
+            EventId::from_virtual(event_id),
+            thread_id,
+        );
     }
 
     pub fn with_profiler(&self, f: impl FnOnce(&SelfProfiler)) {
diff --git a/compiler/rustc_data_structures/src/sorted_map/index_map.rs b/compiler/rustc_data_structures/src/sorted_map/index_map.rs
index 814e7c7fb..7d23ff519 100644
--- a/compiler/rustc_data_structures/src/sorted_map/index_map.rs
+++ b/compiler/rustc_data_structures/src/sorted_map/index_map.rs
@@ -100,6 +100,11 @@ impl<I: Idx, K: Ord, V> SortedIndexMultiMap<I, K, V> {
             (k == &key).then_some((i, v))
         })
     }
+
+    #[inline]
+    pub fn contains_key(&self, key: K) -> bool {
+        self.get_by_key(key).next().is_some()
+    }
 }
 
 impl<I: Idx, K: Eq, V: Eq> Eq for SortedIndexMultiMap<I, K, V> {}
diff --git a/compiler/rustc_data_structures/src/sorted_map/tests.rs b/compiler/rustc_data_structures/src/sorted_map/tests.rs
index 3cc250862..def7a7112 100644
--- a/compiler/rustc_data_structures/src/sorted_map/tests.rs
+++ b/compiler/rustc_data_structures/src/sorted_map/tests.rs
@@ -17,6 +17,10 @@ fn test_sorted_index_multi_map() {
     assert_eq!(set.get_by_key(3).copied().collect::<Vec<_>>(), vec![0]);
     assert!(set.get_by_key(4).next().is_none());
 
+    // `contains_key` works
+    assert!(set.contains_key(3));
+    assert!(!set.contains_key(4));
+
     // `get_by_key` returns items in insertion order.
     let twos: Vec<_> = set.get_by_key_enumerated(2).collect();
     let idxs: Vec<usize> = twos.iter().map(|(i, _)| *i).collect();
diff --git a/compiler/rustc_data_structures/src/stable_hasher.rs b/compiler/rustc_data_structures/src/stable_hasher.rs
index ae4836645..e0d77cdae 100644
--- a/compiler/rustc_data_structures/src/stable_hasher.rs
+++ b/compiler/rustc_data_structures/src/stable_hasher.rs
@@ -486,6 +486,14 @@ impl<HCX> ToStableHashKey<HCX> for String {
     }
 }
 
+impl<HCX, T1: ToStableHashKey<HCX>, T2: ToStableHashKey<HCX>> ToStableHashKey<HCX> for (T1, T2) {
+    type KeyType = (T1::KeyType, T2::KeyType);
+    #[inline]
+    fn to_stable_hash_key(&self, hcx: &HCX) -> Self::KeyType {
+        (self.0.to_stable_hash_key(hcx), self.1.to_stable_hash_key(hcx))
+    }
+}
+
 impl<CTX> HashStable<CTX> for bool {
     #[inline]
     fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) {
diff --git a/compiler/rustc_data_structures/src/stable_hasher/tests.rs b/compiler/rustc_data_structures/src/stable_hasher/tests.rs
index b0d66c32a..724be5888 100644
--- a/compiler/rustc_data_structures/src/stable_hasher/tests.rs
+++ b/compiler/rustc_data_structures/src/stable_hasher/tests.rs
@@ -150,7 +150,7 @@ fn test_isize_compression() {
         let hash_b = hash(&(b as isize, a as isize));
         assert_ne!(
             hash_a, hash_b,
-            "The hash stayed the same when permuting values `{a}` and `{b}!",
+            "The hash stayed the same when permuting values `{a}` and `{b}`!",
         );
     }
 
diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs
index ed5341c40..31323c21d 100644
--- a/compiler/rustc_data_structures/src/sync.rs
+++ b/compiler/rustc_data_structures/src/sync.rs
@@ -26,13 +26,17 @@ use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe};
 pub use std::sync::atomic::Ordering;
 pub use std::sync::atomic::Ordering::SeqCst;
 
+pub use vec::AppendOnlyVec;
+
+mod vec;
+
 cfg_if! {
     if #[cfg(not(parallel_compiler))] {
         pub auto trait Send {}
         pub auto trait Sync {}
 
-        impl<T: ?Sized> Send for T {}
-        impl<T: ?Sized> Sync for T {}
+        impl<T> Send for T {}
+        impl<T> Sync for T {}
 
         #[macro_export]
         macro_rules! rustc_erase_owner {
diff --git a/compiler/rustc_data_structures/src/sync/vec.rs b/compiler/rustc_data_structures/src/sync/vec.rs
new file mode 100644
index 000000000..cbea4f059
--- /dev/null
+++ b/compiler/rustc_data_structures/src/sync/vec.rs
@@ -0,0 +1,41 @@
+use std::marker::PhantomData;
+
+use rustc_index::vec::Idx;
+
+pub struct AppendOnlyVec<I: Idx, T: Copy> {
+    #[cfg(not(parallel_compiler))]
+    vec: elsa::vec::FrozenVec<T>,
+    #[cfg(parallel_compiler)]
+    vec: elsa::sync::LockFreeFrozenVec<T>,
+    _marker: PhantomData<fn(&I)>,
+}
+
+impl<I: Idx, T: Copy> AppendOnlyVec<I, T> {
+    pub fn new() -> Self {
+        Self {
+            #[cfg(not(parallel_compiler))]
+            vec: elsa::vec::FrozenVec::new(),
+            #[cfg(parallel_compiler)]
+            vec: elsa::sync::LockFreeFrozenVec::new(),
+            _marker: PhantomData,
+        }
+    }
+
+    pub fn push(&self, val: T) -> I {
+        #[cfg(not(parallel_compiler))]
+        let i = self.vec.len();
+        #[cfg(not(parallel_compiler))]
+        self.vec.push(val);
+        #[cfg(parallel_compiler)]
+        let i = self.vec.push(val);
+        I::new(i)
+    }
+
+    pub fn get(&self, i: I) -> Option<T> {
+        let i = i.index();
+        #[cfg(not(parallel_compiler))]
+        return self.vec.get_copy(i);
+        #[cfg(parallel_compiler)]
+        return self.vec.get(i);
+    }
+}
diff --git a/compiler/rustc_driver/Cargo.toml b/compiler/rustc_driver/Cargo.toml
index 59e937777..d7c295418 100644
--- a/compiler/rustc_driver/Cargo.toml
+++ b/compiler/rustc_driver/Cargo.toml
@@ -7,39 +7,4 @@ edition = "2021"
 crate-type = ["dylib"]
 
 [dependencies]
-tracing = { version = "0.1.35" }
-serde_json = "1.0.59"
-rustc_log = { path = "../rustc_log" }
-rustc_middle = { path = "../rustc_middle" }
-rustc_ast_pretty = { path = "../rustc_ast_pretty" }
-rustc_target = { path = "../rustc_target" }
-rustc_lint = { path = "../rustc_lint" }
-rustc_data_structures = { path = "../rustc_data_structures" }
-rustc_errors = { path = "../rustc_errors" }
-rustc_feature = { path = "../rustc_feature" }
-rustc_hir = { path = "../rustc_hir" }
-rustc_hir_pretty = { path = "../rustc_hir_pretty" }
-rustc_macros = { path = "../rustc_macros" }
-rustc_metadata = { path = "../rustc_metadata" }
-rustc_parse = { path = "../rustc_parse" }
-rustc_plugin_impl = { path = "../rustc_plugin_impl" }
-rustc_save_analysis = { path = "../rustc_save_analysis" }
-rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }
-rustc_session = { path = "../rustc_session" }
-rustc_error_codes = { path = "../rustc_error_codes" }
-rustc_interface = { path = "../rustc_interface" }
-rustc_ast = { path = "../rustc_ast" }
-rustc_span = { path = "../rustc_span" }
-rustc_hir_analysis = { path = "../rustc_hir_analysis" }
-
-[target.'cfg(unix)'.dependencies]
-libc = "0.2"
-
-[target.'cfg(windows)'.dependencies]
-winapi = { version = "0.3", features = ["consoleapi", "debugapi", "processenv"] }
-
-[features]
-llvm = ['rustc_interface/llvm']
-max_level_info = ['rustc_log/max_level_info']
-rustc_use_parallel_compiler = ['rustc_data_structures/rustc_use_parallel_compiler', 'rustc_interface/rustc_use_parallel_compiler',
-    'rustc_middle/rustc_use_parallel_compiler']
+rustc_driver_impl = { path = "../rustc_driver_impl" }
diff --git a/compiler/rustc_driver/src/lib.rs b/compiler/rustc_driver/src/lib.rs
index f50ad0137..4eabba575 100644
--- a/compiler/rustc_driver/src/lib.rs
+++ b/compiler/rustc_driver/src/lib.rs
@@ -1,1358 +1,4 @@
-//! The Rust compiler.
-//!
-//! # Note
-//!
-//! This API is completely unstable and subject to change.
+// This crate is intentionally empty and a rexport of `rustc_driver_impl` to allow the code in
+// `rustc_driver_impl` to be compiled in parallel with other crates.
 
-#![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![feature(is_terminal)]
-#![feature(once_cell)]
-#![feature(decl_macro)]
-#![recursion_limit = "256"]
-#![allow(rustc::potential_query_instability)]
-#![deny(rustc::untranslatable_diagnostic)]
-#![deny(rustc::diagnostic_outside_of_impl)]
-
-#[macro_use]
-extern crate tracing;
-
-pub extern crate rustc_plugin_impl as plugin;
-
-use rustc_ast as ast;
-use rustc_codegen_ssa::{traits::CodegenBackend, CodegenErrors, CodegenResults};
-use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry};
-use rustc_data_structures::sync::SeqCst;
-use rustc_errors::registry::{InvalidErrorCode, Registry};
-use rustc_errors::{ErrorGuaranteed, PResult};
-use rustc_feature::find_gated_cfg;
-use rustc_hir::def_id::LOCAL_CRATE;
-use rustc_interface::util::{self, collect_crate_types, get_codegen_backend};
-use rustc_interface::{interface, Queries};
-use rustc_lint::LintStore;
-use rustc_metadata::locator;
-use rustc_save_analysis as save;
-use rustc_save_analysis::DumpHandler;
-use rustc_session::config::{nightly_options, CG_OPTIONS, Z_OPTIONS};
-use rustc_session::config::{ErrorOutputType, Input, OutputType, PrintRequest, TrimmedDefPaths};
-use rustc_session::cstore::MetadataLoader;
-use rustc_session::getopts;
-use rustc_session::lint::{Lint, LintId};
-use rustc_session::{config, Session};
-use rustc_session::{early_error, early_error_no_abort, early_warn};
-use rustc_span::source_map::{FileLoader, FileName};
-use rustc_span::symbol::sym;
-use rustc_target::json::ToJson;
-
-use std::borrow::Cow;
-use std::cmp::max;
-use std::env;
-use std::ffi::OsString;
-use std::fs;
-use std::io::{self, IsTerminal, Read, Write};
-use std::panic::{self, catch_unwind};
-use std::path::PathBuf;
-use std::process::{self, Command, Stdio};
-use std::str;
-use std::sync::LazyLock;
-use std::time::Instant;
-
-pub mod args;
-pub mod pretty;
-mod session_diagnostics;
-
-use crate::session_diagnostics::{
-    RLinkEmptyVersionNumber, RLinkEncodingVersionMismatch, RLinkRustcVersionMismatch,
-    RLinkWrongFileType, RlinkNotAFile, RlinkUnableToRead,
-};
-
-/// Exit status code used for successful compilation and help output.
-pub const EXIT_SUCCESS: i32 = 0;
-
-/// Exit status code used for compilation failures and invalid flags.
-pub const EXIT_FAILURE: i32 = 1;
-
-const BUG_REPORT_URL: &str = "https://github.com/rust-lang/rust/issues/new\
-    ?labels=C-bug%2C+I-ICE%2C+T-compiler&template=ice.md";
-
-const ICE_REPORT_COMPILER_FLAGS: &[&str] = &["-Z", "-C", "--crate-type"];
-
-const ICE_REPORT_COMPILER_FLAGS_EXCLUDE: &[&str] = &["metadata", "extra-filename"];
-
-const ICE_REPORT_COMPILER_FLAGS_STRIP_VALUE: &[&str] = &["incremental"];
-
-pub fn abort_on_err<T>(result: Result<T, ErrorGuaranteed>, sess: &Session) -> T {
-    match result {
-        Err(..) => {
-            sess.abort_if_errors();
-            panic!("error reported but abort_if_errors didn't abort???");
-        }
-        Ok(x) => x,
-    }
-}
-
-pub trait Callbacks {
-    /// Called before creating the compiler instance
-    fn config(&mut self, _config: &mut interface::Config) {}
-    /// Called after parsing. Return value instructs the compiler whether to
-    /// continue the compilation afterwards (defaults to `Compilation::Continue`)
-    fn after_parsing<'tcx>(
-        &mut self,
-        _compiler: &interface::Compiler,
-        _queries: &'tcx Queries<'tcx>,
-    ) -> Compilation {
-        Compilation::Continue
-    }
-    /// Called after expansion. Return value instructs the compiler whether to
-    /// continue the compilation afterwards (defaults to `Compilation::Continue`)
-    fn after_expansion<'tcx>(
-        &mut self,
-        _compiler: &interface::Compiler,
-        _queries: &'tcx Queries<'tcx>,
-    ) -> Compilation {
-        Compilation::Continue
-    }
-    /// Called after analysis. Return value instructs the compiler whether to
-    /// continue the compilation afterwards (defaults to `Compilation::Continue`)
-    fn after_analysis<'tcx>(
-        &mut self,
-        _compiler: &interface::Compiler,
-        _queries: &'tcx Queries<'tcx>,
-    ) -> Compilation {
-        Compilation::Continue
-    }
-}
-
-#[derive(Default)]
-pub struct TimePassesCallbacks {
-    time_passes: bool,
-}
-
-impl Callbacks for TimePassesCallbacks {
-    // JUSTIFICATION: the session doesn't exist at this point.
-    #[allow(rustc::bad_opt_access)]
-    fn config(&mut self, config: &mut interface::Config) {
-        // If a --print=... option has been given, we don't print the "total"
-        // time because it will mess up the --print output. See #64339.
-        //
-        self.time_passes = config.opts.prints.is_empty() && config.opts.unstable_opts.time_passes;
-        config.opts.trimmed_def_paths = TrimmedDefPaths::GoodPath;
-    }
-}
-
-pub fn diagnostics_registry() -> Registry {
-    Registry::new(rustc_error_codes::DIAGNOSTICS)
-}
-
-/// This is the primary entry point for rustc.
-pub struct RunCompiler<'a, 'b> {
-    at_args: &'a [String],
-    callbacks: &'b mut (dyn Callbacks + Send),
-    file_loader: Option<Box<dyn FileLoader + Send + Sync>>,
-    make_codegen_backend:
-        Option<Box<dyn FnOnce(&config::Options) -> Box<dyn CodegenBackend> + Send>>,
-}
-
-impl<'a, 'b> RunCompiler<'a, 'b> {
-    pub fn new(at_args: &'a [String], callbacks: &'b mut (dyn Callbacks + Send)) -> Self {
-        Self { at_args, callbacks, file_loader: None, make_codegen_backend: None }
-    }
-
-    /// Set a custom codegen backend.
-    ///
-    /// Has no uses within this repository, but is used by bjorn3 for "the
-    /// hotswapping branch of cg_clif" for "setting the codegen backend from a
-    /// custom driver where the custom codegen backend has arbitrary data."
-    /// (See #102759.)
-    pub fn set_make_codegen_backend(
-        &mut self,
-        make_codegen_backend: Option<
-            Box<dyn FnOnce(&config::Options) -> Box<dyn CodegenBackend> + Send>,
-        >,
-    ) -> &mut Self {
-        self.make_codegen_backend = make_codegen_backend;
-        self
-    }
-
-    /// Load files from sources other than the file system.
-    ///
-    /// Has no uses within this repository, but may be used in the future by
-    /// bjorn3 for "hooking rust-analyzer's VFS into rustc at some point for
-    /// running rustc without having to save". (See #102759.)
-    pub fn set_file_loader(
-        &mut self,
-        file_loader: Option<Box<dyn FileLoader + Send + Sync>>,
-    ) -> &mut Self {
-        self.file_loader = file_loader;
-        self
-    }
-
-    /// Parse args and run the compiler.
-    pub fn run(self) -> interface::Result<()> {
-        run_compiler(self.at_args, self.callbacks, self.file_loader, self.make_codegen_backend)
-    }
-}
-
-fn run_compiler(
-    at_args: &[String],
-    callbacks: &mut (dyn Callbacks + Send),
-    file_loader: Option<Box<dyn FileLoader + Send + Sync>>,
-    make_codegen_backend: Option<
-        Box<dyn FnOnce(&config::Options) -> Box<dyn CodegenBackend> + Send>,
-    >,
-) -> interface::Result<()> {
-    let args = args::arg_expand_all(at_args);
-
-    let Some(matches) = handle_options(&args) else { return Ok(()) };
-
-    let sopts = config::build_session_options(&matches);
-
-    if let Some(ref code) = matches.opt_str("explain") {
-        handle_explain(diagnostics_registry(), code, sopts.error_format);
-        return Ok(());
-    }
-
-    let cfg = interface::parse_cfgspecs(matches.opt_strs("cfg"));
-    let check_cfg = interface::parse_check_cfg(matches.opt_strs("check-cfg"));
-    let (odir, ofile) = make_output(&matches);
-    let mut config = interface::Config {
-        opts: sopts,
-        crate_cfg: cfg,
-        crate_check_cfg: check_cfg,
-        input: Input::File(PathBuf::new()),
-        output_file: ofile,
-        output_dir: odir,
-        file_loader,
-        lint_caps: Default::default(),
-        parse_sess_created: None,
-        register_lints: None,
-        override_queries: None,
-        make_codegen_backend,
-        registry: diagnostics_registry(),
-    };
-
-    if !tracing::dispatcher::has_been_set() {
-        init_rustc_env_logger_with_backtrace_option(&config.opts.unstable_opts.log_backtrace);
-    }
-
-    match make_input(config.opts.error_format, &matches.free) {
-        Err(reported) => return Err(reported),
-        Ok(Some(input)) => {
-            config.input = input;
-
-            callbacks.config(&mut config);
-        }
-        Ok(None) => match matches.free.len() {
-            0 => {
-                callbacks.config(&mut config);
-                interface::run_compiler(config, |compiler| {
-                    let sopts = &compiler.session().opts;
-                    if sopts.describe_lints {
-                        let mut lint_store =
-                            rustc_lint::new_lint_store(compiler.session().enable_internal_lints());
-                        let registered_lints =
-                            if let Some(register_lints) = compiler.register_lints() {
-                                register_lints(compiler.session(), &mut lint_store);
-                                true
-                            } else {
-                                false
-                            };
-                        describe_lints(compiler.session(), &lint_store, registered_lints);
-                        return;
-                    }
-                    let should_stop =
-                        print_crate_info(&***compiler.codegen_backend(), compiler.session(), false);
-
-                    if should_stop == Compilation::Stop {
-                        return;
-                    }
-                    early_error(sopts.error_format, "no input filename given")
-                });
-                return Ok(());
-            }
-            1 => panic!("make_input should have provided valid inputs"),
-            _ => early_error(
-                config.opts.error_format,
-                &format!(
-                    "multiple input filenames provided (first two filenames are `{}` and `{}`)",
-                    matches.free[0], matches.free[1],
-                ),
-            ),
-        },
-    };
-
-    interface::run_compiler(config, |compiler| {
-        let sess = compiler.session();
-        let should_stop = print_crate_info(&***compiler.codegen_backend(), sess, true)
-            .and_then(|| list_metadata(sess, &*compiler.codegen_backend().metadata_loader()))
-            .and_then(|| try_process_rlink(sess, compiler));
-
-        if should_stop == Compilation::Stop {
-            return sess.compile_status();
-        }
-
-        let linker = compiler.enter(|queries| {
-            let early_exit = || sess.compile_status().map(|_| None);
-            queries.parse()?;
-
-            if let Some(ppm) = &sess.opts.pretty {
-                if ppm.needs_ast_map() {
-                    let expanded_crate = queries.expansion()?.borrow().0.clone();
-                    queries.global_ctxt()?.enter(|tcx| {
-                        pretty::print_after_hir_lowering(tcx, &*expanded_crate, *ppm);
-                        Ok(())
-                    })?;
-                } else {
-                    let krate = queries.parse()?.steal();
-                    pretty::print_after_parsing(sess, &krate, *ppm);
-                }
-                trace!("finished pretty-printing");
-                return early_exit();
-            }
-
-            if callbacks.after_parsing(compiler, queries) == Compilation::Stop {
-                return early_exit();
-            }
-
-            if sess.opts.unstable_opts.parse_only || sess.opts.unstable_opts.show_span.is_some() {
-                return early_exit();
-            }
-
-            {
-                let plugins = queries.register_plugins()?;
-                let (_, lint_store) = &*plugins.borrow();
-
-                // Lint plugins are registered; now we can process command line flags.
-                if sess.opts.describe_lints {
-                    describe_lints(sess, lint_store, true);
-                    return early_exit();
-                }
-            }
-
-            queries.global_ctxt()?;
-            if callbacks.after_expansion(compiler, queries) == Compilation::Stop {
-                return early_exit();
-            }
-
-            if sess.opts.output_types.contains_key(&OutputType::DepInfo)
-                && sess.opts.output_types.len() == 1
-            {
-                return early_exit();
-            }
-
-            if sess.opts.unstable_opts.no_analysis {
-                return early_exit();
-            }
-
-            queries.global_ctxt()?.enter(|tcx| {
-                let result = tcx.analysis(());
-                if sess.opts.unstable_opts.save_analysis {
-                    let crate_name = tcx.crate_name(LOCAL_CRATE);
-                    sess.time("save_analysis", || {
-                        save::process_crate(
-                            tcx,
-                            crate_name,
-                            &sess.io.input,
-                            None,
-                            DumpHandler::new(sess.io.output_dir.as_deref(), crate_name),
-                        )
-                    });
-                }
-                result
-            })?;
-
-            if callbacks.after_analysis(compiler, queries) == Compilation::Stop {
-                return early_exit();
-            }
-
-            queries.ongoing_codegen()?;
-
-            if sess.opts.unstable_opts.print_type_sizes {
-                sess.code_stats.print_type_sizes();
-            }
-
-            let linker = queries.linker()?;
-            Ok(Some(linker))
-        })?;
-
-        if let Some(linker) = linker {
-            let _timer = sess.timer("link");
-            linker.link()?
-        }
-
-        if sess.opts.unstable_opts.perf_stats {
-            sess.print_perf_stats();
-        }
-
-        if sess.opts.unstable_opts.print_fuel.is_some() {
-            eprintln!(
-                "Fuel used by {}: {}",
-                sess.opts.unstable_opts.print_fuel.as_ref().unwrap(),
-                sess.print_fuel.load(SeqCst)
-            );
-        }
-
-        Ok(())
-    })
-}
-
-// Extract output directory and file from matches.
-fn make_output(matches: &getopts::Matches) -> (Option<PathBuf>, Option<PathBuf>) {
-    let odir = matches.opt_str("out-dir").map(|o| PathBuf::from(&o));
-    let ofile = matches.opt_str("o").map(|o| PathBuf::from(&o));
-    (odir, ofile)
-}
-
-// Extract input (string or file and optional path) from matches.
-fn make_input(
-    error_format: ErrorOutputType,
-    free_matches: &[String],
-) -> Result<Option<Input>, ErrorGuaranteed> {
-    if free_matches.len() == 1 {
-        let ifile = &free_matches[0];
-        if ifile == "-" {
-            let mut src = String::new();
-            if io::stdin().read_to_string(&mut src).is_err() {
-                // Immediately stop compilation if there was an issue reading
-                // the input (for example if the input stream is not UTF-8).
-                let reported = early_error_no_abort(
-                    error_format,
-                    "couldn't read from stdin, as it did not contain valid UTF-8",
-                );
-                return Err(reported);
-            }
-            if let Ok(path) = env::var("UNSTABLE_RUSTDOC_TEST_PATH") {
-                let line = env::var("UNSTABLE_RUSTDOC_TEST_LINE").expect(
-                    "when UNSTABLE_RUSTDOC_TEST_PATH is set \
-                                    UNSTABLE_RUSTDOC_TEST_LINE also needs to be set",
-                );
-                let line = isize::from_str_radix(&line, 10)
-                    .expect("UNSTABLE_RUSTDOC_TEST_LINE needs to be an number");
-                let file_name = FileName::doc_test_source_code(PathBuf::from(path), line);
-                Ok(Some(Input::Str { name: file_name, input: src }))
-            } else {
-                Ok(Some(Input::Str { name: FileName::anon_source_code(&src), input: src }))
-            }
-        } else {
-            Ok(Some(Input::File(PathBuf::from(ifile))))
-        }
-    } else {
-        Ok(None)
-    }
-}
-
-/// Whether to stop or continue compilation.
-#[derive(Copy, Clone, Debug, Eq, PartialEq)]
-pub enum Compilation {
-    Stop,
-    Continue,
-}
-
-impl Compilation {
-    pub fn and_then<F: FnOnce() -> Compilation>(self, next: F) -> Compilation {
-        match self {
-            Compilation::Stop => Compilation::Stop,
-            Compilation::Continue => next(),
-        }
-    }
-}
-
-fn handle_explain(registry: Registry, code: &str, output: ErrorOutputType) {
-    let upper_cased_code = code.to_ascii_uppercase();
-    let normalised =
-        if upper_cased_code.starts_with('E') { upper_cased_code } else { format!("E{code:0>4}") };
-    match registry.try_find_description(&normalised) {
-        Ok(Some(description)) => {
-            let mut is_in_code_block = false;
-            let mut text = String::new();
-            // Slice off the leading newline and print.
-            for line in description.lines() {
-                let indent_level =
-                    line.find(|c: char| !c.is_whitespace()).unwrap_or_else(|| line.len());
-                let dedented_line = &line[indent_level..];
-                if dedented_line.starts_with("```") {
-                    is_in_code_block = !is_in_code_block;
-                    text.push_str(&line[..(indent_level + 3)]);
-                } else if is_in_code_block && dedented_line.starts_with("# ") {
-                    continue;
-                } else {
-                    text.push_str(line);
-                }
-                text.push('\n');
-            }
-            if io::stdout().is_terminal() {
-                show_content_with_pager(&text);
-            } else {
-                print!("{text}");
-            }
-        }
-        Ok(None) => {
-            early_error(output, &format!("no extended information for {code}"));
-        }
-        Err(InvalidErrorCode) => {
-            early_error(output, &format!("{code} is not a valid error code"));
-        }
-    }
-}
-
-fn show_content_with_pager(content: &str) {
-    let pager_name = env::var_os("PAGER").unwrap_or_else(|| {
-        if cfg!(windows) { OsString::from("more.com") } else { OsString::from("less") }
-    });
-
-    let mut fallback_to_println = false;
-
-    match Command::new(pager_name).stdin(Stdio::piped()).spawn() {
-        Ok(mut pager) => {
-            if let Some(pipe) = pager.stdin.as_mut() {
-                if pipe.write_all(content.as_bytes()).is_err() {
-                    fallback_to_println = true;
-                }
-            }
-
-            if pager.wait().is_err() {
-                fallback_to_println = true;
-            }
-        }
-        Err(_) => {
-            fallback_to_println = true;
-        }
-    }
-
-    // If pager fails for whatever reason, we should still print the content
-    // to standard output
-    if fallback_to_println {
-        print!("{content}");
-    }
-}
-
-pub fn try_process_rlink(sess: &Session, compiler: &interface::Compiler) -> Compilation {
-    if sess.opts.unstable_opts.link_only {
-        if let Input::File(file) = &sess.io.input {
-            // FIXME: #![crate_type] and #![crate_name] support not implemented yet
-            sess.init_crate_types(collect_crate_types(sess, &[]));
-            let outputs = compiler.build_output_filenames(sess, &[]);
-            let rlink_data = fs::read(file).unwrap_or_else(|err| {
-                sess.emit_fatal(RlinkUnableToRead { err });
-            });
-            let codegen_results = match CodegenResults::deserialize_rlink(rlink_data) {
-                Ok(codegen) => codegen,
-                Err(err) => {
-                    match err {
-                        CodegenErrors::WrongFileType => sess.emit_fatal(RLinkWrongFileType),
-                        CodegenErrors::EmptyVersionNumber => {
-                            sess.emit_fatal(RLinkEmptyVersionNumber)
-                        }
-                        CodegenErrors::EncodingVersionMismatch { version_array, rlink_version } => {
-                            sess.emit_fatal(RLinkEncodingVersionMismatch {
-                                version_array,
-                                rlink_version,
-                            })
-                        }
-                        CodegenErrors::RustcVersionMismatch { rustc_version, current_version } => {
-                            sess.emit_fatal(RLinkRustcVersionMismatch {
-                                rustc_version,
-                                current_version,
-                            })
-                        }
-                    };
-                }
-            };
-            let result = compiler.codegen_backend().link(sess, codegen_results, &outputs);
-            abort_on_err(result, sess);
-        } else {
-            sess.emit_fatal(RlinkNotAFile {})
-        }
-        Compilation::Stop
-    } else {
-        Compilation::Continue
-    }
-}
-
-pub fn list_metadata(sess: &Session, metadata_loader: &dyn MetadataLoader) -> Compilation {
-    if sess.opts.unstable_opts.ls {
-        match sess.io.input {
-            Input::File(ref ifile) => {
-                let path = &(*ifile);
-                let mut v = Vec::new();
-                locator::list_file_metadata(&sess.target, path, metadata_loader, &mut v).unwrap();
-                println!("{}", String::from_utf8(v).unwrap());
-            }
-            Input::Str { .. } => {
-                early_error(ErrorOutputType::default(), "cannot list metadata for stdin");
-            }
-        }
-        return Compilation::Stop;
-    }
-
-    Compilation::Continue
-}
-
-fn print_crate_info(
-    codegen_backend: &dyn CodegenBackend,
-    sess: &Session,
-    parse_attrs: bool,
-) -> Compilation {
-    use rustc_session::config::PrintRequest::*;
-    // NativeStaticLibs and LinkArgs are special - printed during linking
-    // (empty iterator returns true)
-    if sess.opts.prints.iter().all(|&p| p == NativeStaticLibs || p == LinkArgs) {
-        return Compilation::Continue;
-    }
-
-    let attrs = if parse_attrs {
-        let result = parse_crate_attrs(sess);
-        match result {
-            Ok(attrs) => Some(attrs),
-            Err(mut parse_error) => {
-                parse_error.emit();
-                return Compilation::Stop;
-            }
-        }
-    } else {
-        None
-    };
-    for req in &sess.opts.prints {
-        match *req {
-            TargetList => {
-                let mut targets = rustc_target::spec::TARGETS.to_vec();
-                targets.sort_unstable();
-                println!("{}", targets.join("\n"));
-            }
-            Sysroot => println!("{}", sess.sysroot.display()),
-            TargetLibdir => println!("{}", sess.target_tlib_path.dir.display()),
-            TargetSpec => {
-                println!("{}", serde_json::to_string_pretty(&sess.target.to_json()).unwrap());
-            }
-            FileNames | CrateName => {
-                let attrs = attrs.as_ref().unwrap();
-                let t_outputs = rustc_interface::util::build_output_filenames(attrs, sess);
-                let id = rustc_session::output::find_crate_name(sess, attrs);
-                if *req == PrintRequest::CrateName {
-                    println!("{id}");
-                    continue;
-                }
-                let crate_types = collect_crate_types(sess, attrs);
-                for &style in &crate_types {
-                    let fname =
-                        rustc_session::output::filename_for_input(sess, style, id, &t_outputs);
-                    println!("{}", fname.file_name().unwrap().to_string_lossy());
-                }
-            }
-            Cfg => {
-                let mut cfgs = sess
-                    .parse_sess
-                    .config
-                    .iter()
-                    .filter_map(|&(name, value)| {
-                        // Note that crt-static is a specially recognized cfg
-                        // directive that's printed out here as part of
-                        // rust-lang/rust#37406, but in general the
-                        // `target_feature` cfg is gated under
-                        // rust-lang/rust#29717. For now this is just
-                        // specifically allowing the crt-static cfg and that's
-                        // it, this is intended to get into Cargo and then go
-                        // through to build scripts.
-                        if (name != sym::target_feature || value != Some(sym::crt_dash_static))
-                            && !sess.is_nightly_build()
-                            && find_gated_cfg(|cfg_sym| cfg_sym == name).is_some()
-                        {
-                            return None;
-                        }
-
-                        if let Some(value) = value {
-                            Some(format!("{name}=\"{value}\""))
-                        } else {
-                            Some(name.to_string())
-                        }
-                    })
-                    .collect::<Vec<String>>();
-
-                cfgs.sort();
-                for cfg in cfgs {
-                    println!("{cfg}");
-                }
-            }
-            CallingConventions => {
-                let mut calling_conventions = rustc_target::spec::abi::all_names();
-                calling_conventions.sort_unstable();
-                println!("{}", calling_conventions.join("\n"));
-            }
-            RelocationModels
-            | CodeModels
-            | TlsModels
-            | TargetCPUs
-            | StackProtectorStrategies
-            | TargetFeatures => {
-                codegen_backend.print(*req, sess);
-            }
-            // Any output here interferes with Cargo's parsing of other printed output
-            NativeStaticLibs => {}
-            LinkArgs => {}
-            SplitDebuginfo => {
-                use rustc_target::spec::SplitDebuginfo::{Off, Packed, Unpacked};
-
-                for split in &[Off, Packed, Unpacked] {
-                    let stable = sess.target.options.supported_split_debuginfo.contains(split);
-                    let unstable_ok = sess.unstable_options();
-                    if stable || unstable_ok {
-                        println!("{split}");
-                    }
-                }
-            }
-        }
-    }
-    Compilation::Stop
-}
-
-/// Prints version information
-///
-/// NOTE: this is a macro to support drivers built at a different time than the main `rustc_driver` crate.
-pub macro version($binary: literal, $matches: expr) {
-    fn unw(x: Option<&str>) -> &str {
-        x.unwrap_or("unknown")
-    }
-    $crate::version_at_macro_invocation(
-        $binary,
-        $matches,
-        unw(option_env!("CFG_VERSION")),
-        unw(option_env!("CFG_VER_HASH")),
-        unw(option_env!("CFG_VER_DATE")),
-        unw(option_env!("CFG_RELEASE")),
-    )
-}
-
-#[doc(hidden)] // use the macro instead
-pub fn version_at_macro_invocation(
-    binary: &str,
-    matches: &getopts::Matches,
-    version: &str,
-    commit_hash: &str,
-    commit_date: &str,
-    release: &str,
-) {
-    let verbose = matches.opt_present("verbose");
-
-    println!("{binary} {version}");
-
-    if verbose {
-        println!("binary: {binary}");
-        println!("commit-hash: {commit_hash}");
-        println!("commit-date: {commit_date}");
-        println!("host: {}", config::host_triple());
-        println!("release: {release}");
-
-        let debug_flags = matches.opt_strs("Z");
-        let backend_name = debug_flags.iter().find_map(|x| x.strip_prefix("codegen-backend="));
-        get_codegen_backend(&None, backend_name).print_version();
-    }
-}
-
-fn usage(verbose: bool, include_unstable_options: bool, nightly_build: bool) {
-    let groups = if verbose { config::rustc_optgroups() } else { config::rustc_short_optgroups() };
-    let mut options = getopts::Options::new();
-    for option in groups.iter().filter(|x| include_unstable_options || x.is_stable()) {
-        (option.apply)(&mut options);
-    }
-    let message = "Usage: rustc [OPTIONS] INPUT";
-    let nightly_help = if nightly_build {
-        "\n    -Z help             Print unstable compiler options"
-    } else {
-        ""
-    };
-    let verbose_help = if verbose {
-        ""
-    } else {
-        "\n    --help -v           Print the full set of options rustc accepts"
-    };
-    let at_path = if verbose {
-        "    @path               Read newline separated options from `path`\n"
-    } else {
-        ""
-    };
-    println!(
-        "{options}{at_path}\nAdditional help:
-    -C help             Print codegen options
-    -W help             \
-              Print 'lint' options and default settings{nightly}{verbose}\n",
-        options = options.usage(message),
-        at_path = at_path,
-        nightly = nightly_help,
-        verbose = verbose_help
-    );
-}
-
-fn print_wall_help() {
-    println!(
-        "
-The flag `-Wall` does not exist in `rustc`. Most useful lints are enabled by
-default. Use `rustc -W help` to see all available lints. It's more common to put
-warning settings in the crate root using `#![warn(LINT_NAME)]` instead of using
-the command line flag directly.
-"
-    );
-}
-
-/// Write to stdout lint command options, together with a list of all available lints
-pub fn describe_lints(sess: &Session, lint_store: &LintStore, loaded_plugins: bool) {
-    println!(
-        "
-Available lint options:
-    -W <foo>           Warn about <foo>
-    -A <foo>           \
-              Allow <foo>
-    -D <foo>           Deny <foo>
-    -F <foo>           Forbid <foo> \
-              (deny <foo> and all attempts to override)
-
-"
-    );
-
-    fn sort_lints(sess: &Session, mut lints: Vec<&'static Lint>) -> Vec<&'static Lint> {
-        // The sort doesn't case-fold but it's doubtful we care.
-        lints.sort_by_cached_key(|x: &&Lint| (x.default_level(sess.edition()), x.name));
-        lints
-    }
-
-    fn sort_lint_groups(
-        lints: Vec<(&'static str, Vec<LintId>, bool)>,
-    ) -> Vec<(&'static str, Vec<LintId>)> {
-        let mut lints: Vec<_> = lints.into_iter().map(|(x, y, _)| (x, y)).collect();
-        lints.sort_by_key(|l| l.0);
-        lints
-    }
-
-    let (plugin, builtin): (Vec<_>, _) =
-        lint_store.get_lints().iter().cloned().partition(|&lint| lint.is_plugin);
-    let plugin = sort_lints(sess, plugin);
-    let builtin = sort_lints(sess, builtin);
-
-    let (plugin_groups, builtin_groups): (Vec<_>, _) =
-        lint_store.get_lint_groups().partition(|&(.., p)| p);
-    let plugin_groups = sort_lint_groups(plugin_groups);
-    let builtin_groups = sort_lint_groups(builtin_groups);
-
-    let max_name_len =
-        plugin.iter().chain(&builtin).map(|&s| s.name.chars().count()).max().unwrap_or(0);
-    let padded = |x: &str| {
-        let mut s = " ".repeat(max_name_len - x.chars().count());
-        s.push_str(x);
-        s
-    };
-
-    println!("Lint checks provided by rustc:\n");
-
-    let print_lints = |lints: Vec<&Lint>| {
-        println!("    {}  {:7.7}  {}", padded("name"), "default", "meaning");
-        println!("    {}  {:7.7}  {}", padded("----"), "-------", "-------");
-        for lint in lints {
-            let name = lint.name_lower().replace('_', "-");
-            println!(
-                "    {}  {:7.7}  {}",
-                padded(&name),
-                lint.default_level(sess.edition()).as_str(),
-                lint.desc
-            );
-        }
-        println!("\n");
-    };
-
-    print_lints(builtin);
-
-    let max_name_len = max(
-        "warnings".len(),
-        plugin_groups
-            .iter()
-            .chain(&builtin_groups)
-            .map(|&(s, _)| s.chars().count())
-            .max()
-            .unwrap_or(0),
-    );
-
-    let padded = |x: &str| {
-        let mut s = " ".repeat(max_name_len - x.chars().count());
-        s.push_str(x);
-        s
-    };
-
-    println!("Lint groups provided by rustc:\n");
-
-    let print_lint_groups = |lints: Vec<(&'static str, Vec<LintId>)>, all_warnings| {
-        println!("    {}  sub-lints", padded("name"));
-        println!("    {}  ---------", padded("----"));
-
-        if all_warnings {
-            println!("    {}  all lints that are set to issue warnings", padded("warnings"));
-        }
-
-        for (name, to) in lints {
-            let name = name.to_lowercase().replace('_', "-");
-            let desc = to
-                .into_iter()
-                .map(|x| x.to_string().replace('_', "-"))
-                .collect::<Vec<String>>()
-                .join(", ");
-            println!("    {}  {}", padded(&name), desc);
-        }
-        println!("\n");
-    };
-
-    print_lint_groups(builtin_groups, true);
-
-    match (loaded_plugins, plugin.len(), plugin_groups.len()) {
-        (false, 0, _) | (false, _, 0) => {
-            println!("Lint tools like Clippy can provide additional lints and lint groups.");
-        }
-        (false, ..) => panic!("didn't load lint plugins but got them anyway!"),
-        (true, 0, 0) => println!("This crate does not load any lint plugins or lint groups."),
-        (true, l, g) => {
-            if l > 0 {
-                println!("Lint checks provided by plugins loaded by this crate:\n");
-                print_lints(plugin);
-            }
-            if g > 0 {
-                println!("Lint groups provided by plugins loaded by this crate:\n");
-                print_lint_groups(plugin_groups, false);
-            }
-        }
-    }
-}
-
-fn describe_debug_flags() {
-    println!("\nAvailable options:\n");
-    print_flag_list("-Z", config::Z_OPTIONS);
-}
-
-fn describe_codegen_flags() {
-    println!("\nAvailable codegen options:\n");
-    print_flag_list("-C", config::CG_OPTIONS);
-}
-
-pub fn print_flag_list<T>(
-    cmdline_opt: &str,
-    flag_list: &[(&'static str, T, &'static str, &'static str)],
-) {
-    let max_len = flag_list.iter().map(|&(name, _, _, _)| name.chars().count()).max().unwrap_or(0);
-
-    for &(name, _, _, desc) in flag_list {
-        println!(
-            "    {} {:>width$}=val -- {}",
-            cmdline_opt,
-            name.replace('_', "-"),
-            desc,
-            width = max_len
-        );
-    }
-}
-
-/// Process command line options. Emits messages as appropriate. If compilation
-/// should continue, returns a getopts::Matches object parsed from args,
-/// otherwise returns `None`.
-///
-/// The compiler's handling of options is a little complicated as it ties into
-/// our stability story. The current intention of each compiler option is to
-/// have one of two modes:
-///
-/// 1. An option is stable and can be used everywhere.
-/// 2. An option is unstable, and can only be used on nightly.
-///
-/// Like unstable library and language features, however, unstable options have
-/// always required a form of "opt in" to indicate that you're using them. This
-/// provides the easy ability to scan a code base to check to see if anything
-/// unstable is being used. Currently, this "opt in" is the `-Z` "zed" flag.
-///
-/// All options behind `-Z` are considered unstable by default. Other top-level
-/// options can also be considered unstable, and they were unlocked through the
-/// `-Z unstable-options` flag. Note that `-Z` remains to be the root of
-/// instability in both cases, though.
-///
-/// So with all that in mind, the comments below have some more detail about the
-/// contortions done here to get things to work out correctly.
-pub fn handle_options(args: &[String]) -> Option<getopts::Matches> {
-    // Throw away the first argument, the name of the binary
-    let args = &args[1..];
-
-    if args.is_empty() {
-        // user did not write `-v` nor `-Z unstable-options`, so do not
-        // include that extra information.
-        let nightly_build =
-            rustc_feature::UnstableFeatures::from_environment(None).is_nightly_build();
-        usage(false, false, nightly_build);
-        return None;
-    }
-
-    // Parse with *all* options defined in the compiler, we don't worry about
-    // option stability here we just want to parse as much as possible.
-    let mut options = getopts::Options::new();
-    for option in config::rustc_optgroups() {
-        (option.apply)(&mut options);
-    }
-    let matches = options.parse(args).unwrap_or_else(|e| {
-        let msg = match e {
-            getopts::Fail::UnrecognizedOption(ref opt) => CG_OPTIONS
-                .iter()
-                .map(|&(name, ..)| ('C', name))
-                .chain(Z_OPTIONS.iter().map(|&(name, ..)| ('Z', name)))
-                .find(|&(_, name)| *opt == name.replace('_', "-"))
-                .map(|(flag, _)| format!("{e}. Did you mean `-{flag} {opt}`?")),
-            _ => None,
-        };
-        early_error(ErrorOutputType::default(), &msg.unwrap_or_else(|| e.to_string()));
-    });
-
-    // For all options we just parsed, we check a few aspects:
-    //
-    // * If the option is stable, we're all good
-    // * If the option wasn't passed, we're all good
-    // * If `-Z unstable-options` wasn't passed (and we're not a -Z option
-    //   ourselves), then we require the `-Z unstable-options` flag to unlock
-    //   this option that was passed.
-    // * If we're a nightly compiler, then unstable options are now unlocked, so
-    //   we're good to go.
-    // * Otherwise, if we're an unstable option then we generate an error
-    //   (unstable option being used on stable)
-    nightly_options::check_nightly_options(&matches, &config::rustc_optgroups());
-
-    if matches.opt_present("h") || matches.opt_present("help") {
-        // Only show unstable options in --help if we accept unstable options.
-        let unstable_enabled = nightly_options::is_unstable_enabled(&matches);
-        let nightly_build = nightly_options::match_is_nightly_build(&matches);
-        usage(matches.opt_present("verbose"), unstable_enabled, nightly_build);
-        return None;
-    }
-
-    // Handle the special case of -Wall.
-    let wall = matches.opt_strs("W");
-    if wall.iter().any(|x| *x == "all") {
-        print_wall_help();
-        rustc_errors::FatalError.raise();
-    }
-
-    // Don't handle -W help here, because we might first load plugins.
-    let debug_flags = matches.opt_strs("Z");
-    if debug_flags.iter().any(|x| *x == "help") {
-        describe_debug_flags();
-        return None;
-    }
-
-    let cg_flags = matches.opt_strs("C");
-
-    if cg_flags.iter().any(|x| *x == "help") {
-        describe_codegen_flags();
-        return None;
-    }
-
-    if cg_flags.iter().any(|x| *x == "no-stack-check") {
-        early_warn(
-            ErrorOutputType::default(),
-            "the --no-stack-check flag is deprecated and does nothing",
-        );
-    }
-
-    if cg_flags.iter().any(|x| *x == "passes=list") {
-        let backend_name = debug_flags.iter().find_map(|x| x.strip_prefix("codegen-backend="));
-        get_codegen_backend(&None, backend_name).print_passes();
-        return None;
-    }
-
-    if matches.opt_present("version") {
-        version!("rustc", &matches);
-        return None;
-    }
-
-    Some(matches)
-}
-
-fn parse_crate_attrs<'a>(sess: &'a Session) -> PResult<'a, ast::AttrVec> {
-    match &sess.io.input {
-        Input::File(ifile) => rustc_parse::parse_crate_attrs_from_file(ifile, &sess.parse_sess),
-        Input::Str { name, input } => rustc_parse::parse_crate_attrs_from_source_str(
-            name.clone(),
-            input.clone(),
-            &sess.parse_sess,
-        ),
-    }
-}
-
-/// Gets a list of extra command-line flags provided by the user, as strings.
-///
-/// This function is used during ICEs to show more information useful for
-/// debugging, since some ICEs only happens with non-default compiler flags
-/// (and the users don't always report them).
-fn extra_compiler_flags() -> Option<(Vec<String>, bool)> {
-    let mut args = env::args_os().map(|arg| arg.to_string_lossy().to_string()).peekable();
-
-    let mut result = Vec::new();
-    let mut excluded_cargo_defaults = false;
-    while let Some(arg) = args.next() {
-        if let Some(a) = ICE_REPORT_COMPILER_FLAGS.iter().find(|a| arg.starts_with(*a)) {
-            let content = if arg.len() == a.len() {
-                // A space-separated option, like `-C incremental=foo` or `--crate-type rlib`
-                match args.next() {
-                    Some(arg) => arg.to_string(),
-                    None => continue,
-                }
-            } else if arg.get(a.len()..a.len() + 1) == Some("=") {
-                // An equals option, like `--crate-type=rlib`
-                arg[a.len() + 1..].to_string()
-            } else {
-                // A non-space option, like `-Cincremental=foo`
-                arg[a.len()..].to_string()
-            };
-            let option = content.split_once('=').map(|s| s.0).unwrap_or(&content);
-            if ICE_REPORT_COMPILER_FLAGS_EXCLUDE.iter().any(|exc| option == *exc) {
-                excluded_cargo_defaults = true;
-            } else {
-                result.push(a.to_string());
-                match ICE_REPORT_COMPILER_FLAGS_STRIP_VALUE.iter().find(|s| option == **s) {
-                    Some(s) => result.push(format!("{s}=[REDACTED]")),
-                    None => result.push(content),
-                }
-            }
-        }
-    }
-
-    if !result.is_empty() { Some((result, excluded_cargo_defaults)) } else { None }
-}
-
-/// Runs a closure and catches unwinds triggered by fatal errors.
-///
-/// The compiler currently unwinds with a special sentinel value to abort
-/// compilation on fatal errors. This function catches that sentinel and turns
-/// the panic into a `Result` instead.
-pub fn catch_fatal_errors<F: FnOnce() -> R, R>(f: F) -> Result<R, ErrorGuaranteed> {
-    catch_unwind(panic::AssertUnwindSafe(f)).map_err(|value| {
-        if value.is::<rustc_errors::FatalErrorMarker>() {
-            ErrorGuaranteed::unchecked_claim_error_was_emitted()
-        } else {
-            panic::resume_unwind(value);
-        }
-    })
-}
-
-/// Variant of `catch_fatal_errors` for the `interface::Result` return type
-/// that also computes the exit code.
-pub fn catch_with_exit_code(f: impl FnOnce() -> interface::Result<()>) -> i32 {
-    let result = catch_fatal_errors(f).and_then(|result| result);
-    match result {
-        Ok(()) => EXIT_SUCCESS,
-        Err(_) => EXIT_FAILURE,
-    }
-}
-
-static DEFAULT_HOOK: LazyLock<Box<dyn Fn(&panic::PanicInfo<'_>) + Sync + Send + 'static>> =
-    LazyLock::new(|| {
-        let hook = panic::take_hook();
-        panic::set_hook(Box::new(|info| {
-            // If the error was caused by a broken pipe then this is not a bug.
-            // Write the error and return immediately. See #98700.
-            #[cfg(windows)]
-            if let Some(msg) = info.payload().downcast_ref::<String>() {
-                if msg.starts_with("failed printing to stdout: ") && msg.ends_with("(os error 232)")
-                {
-                    early_error_no_abort(ErrorOutputType::default(), &msg);
-                    return;
-                }
-            };
-
-            // Invoke the default handler, which prints the actual panic message and optionally a backtrace
-            // Don't do this for delayed bugs, which already emit their own more useful backtrace.
-            if !info.payload().is::<rustc_errors::DelayedBugPanic>() {
-                (*DEFAULT_HOOK)(info);
-
-                // Separate the output with an empty line
-                eprintln!();
-            }
-
-            // Print the ICE message
-            report_ice(info, BUG_REPORT_URL);
-        }));
-        hook
-    });
-
-/// Prints the ICE message, including query stack, but without backtrace.
-///
-/// The message will point the user at `bug_report_url` to report the ICE.
-///
-/// When `install_ice_hook` is called, this function will be called as the panic
-/// hook.
-pub fn report_ice(info: &panic::PanicInfo<'_>, bug_report_url: &str) {
-    let fallback_bundle =
-        rustc_errors::fallback_fluent_bundle(rustc_errors::DEFAULT_LOCALE_RESOURCES, false);
-    let emitter = Box::new(rustc_errors::emitter::EmitterWriter::stderr(
-        rustc_errors::ColorConfig::Auto,
-        None,
-        None,
-        fallback_bundle,
-        false,
-        false,
-        None,
-        false,
-        false,
-    ));
-    let handler = rustc_errors::Handler::with_emitter(true, None, emitter);
-
-    // a .span_bug or .bug call has already printed what
-    // it wants to print.
-    if !info.payload().is::<rustc_errors::ExplicitBug>()
-        && !info.payload().is::<rustc_errors::DelayedBugPanic>()
-    {
-        let mut d = rustc_errors::Diagnostic::new(rustc_errors::Level::Bug, "unexpected panic");
-        handler.emit_diagnostic(&mut d);
-    }
-
-    let mut xs: Vec<Cow<'static, str>> = vec![
-        "the compiler unexpectedly panicked. this is a bug.".into(),
-        format!("we would appreciate a bug report: {bug_report_url}").into(),
-        format!(
-            "rustc {} running on {}",
-            util::version_str!().unwrap_or("unknown_version"),
-            config::host_triple()
-        )
-        .into(),
-    ];
-
-    if let Some((flags, excluded_cargo_defaults)) = extra_compiler_flags() {
-        xs.push(format!("compiler flags: {}", flags.join(" ")).into());
-
-        if excluded_cargo_defaults {
-            xs.push("some of the compiler flags provided by cargo are hidden".into());
-        }
-    }
-
-    for note in &xs {
-        handler.note_without_error(note.as_ref());
-    }
-
-    // If backtraces are enabled, also print the query stack
-    let backtrace = env::var_os("RUST_BACKTRACE").map_or(false, |x| &x != "0");
-
-    let num_frames = if backtrace { None } else { Some(2) };
-
-    interface::try_print_query_stack(&handler, num_frames);
-
-    #[cfg(windows)]
-    unsafe {
-        if env::var("RUSTC_BREAK_ON_ICE").is_ok() {
-            // Trigger a debugger if we crashed during bootstrap
-            winapi::um::debugapi::DebugBreak();
-        }
-    }
-}
-
-/// Installs a panic hook that will print the ICE message on unexpected panics.
-///
-/// A custom rustc driver can skip calling this to set up a custom ICE hook.
-pub fn install_ice_hook() {
-    // If the user has not explicitly overridden "RUST_BACKTRACE", then produce
-    // full backtraces. When a compiler ICE happens, we want to gather
-    // as much information as possible to present in the issue opened
-    // by the user. Compiler developers and other rustc users can
-    // opt in to less-verbose backtraces by manually setting "RUST_BACKTRACE"
-    // (e.g. `RUST_BACKTRACE=1`)
-    if std::env::var("RUST_BACKTRACE").is_err() {
-        std::env::set_var("RUST_BACKTRACE", "full");
-    }
-    LazyLock::force(&DEFAULT_HOOK);
-}
-
-/// This allows tools to enable rust logging without having to magically match rustc's
-/// tracing crate version.
-pub fn init_rustc_env_logger() {
-    init_rustc_env_logger_with_backtrace_option(&None);
-}
-
-/// This allows tools to enable rust logging without having to magically match rustc's
-/// tracing crate version. In contrast to `init_rustc_env_logger` it allows you to
-/// choose a target module you wish to show backtraces along with its logging.
-pub fn init_rustc_env_logger_with_backtrace_option(backtrace_target: &Option<String>) {
-    if let Err(error) = rustc_log::init_rustc_env_logger_with_backtrace_option(backtrace_target) {
-        early_error(ErrorOutputType::default(), &error.to_string());
-    }
-}
-
-/// This allows tools to enable rust logging without having to magically match rustc's
-/// tracing crate version. In contrast to `init_rustc_env_logger` it allows you to choose an env var
-/// other than `RUSTC_LOG`.
-pub fn init_env_logger(env: &str) {
-    if let Err(error) = rustc_log::init_env_logger(env) {
-        early_error(ErrorOutputType::default(), &error.to_string());
-    }
-}
-
-#[cfg(all(unix, any(target_env = "gnu", target_os = "macos")))]
-mod signal_handler {
-    extern "C" {
-        fn backtrace_symbols_fd(
-            buffer: *const *mut libc::c_void,
-            size: libc::c_int,
-            fd: libc::c_int,
-        );
-    }
-
-    extern "C" fn print_stack_trace(_: libc::c_int) {
-        const MAX_FRAMES: usize = 256;
-        static mut STACK_TRACE: [*mut libc::c_void; MAX_FRAMES] =
-            [std::ptr::null_mut(); MAX_FRAMES];
-        unsafe {
-            let depth = libc::backtrace(STACK_TRACE.as_mut_ptr(), MAX_FRAMES as i32);
-            if depth == 0 {
-                return;
-            }
-            backtrace_symbols_fd(STACK_TRACE.as_ptr(), depth, 2);
-        }
-    }
-
-    /// When an error signal (such as SIGABRT or SIGSEGV) is delivered to the
-    /// process, print a stack trace and then exit.
-    pub(super) fn install() {
-        unsafe {
-            const ALT_STACK_SIZE: usize = libc::MINSIGSTKSZ + 64 * 1024;
-            let mut alt_stack: libc::stack_t = std::mem::zeroed();
-            alt_stack.ss_sp =
-                std::alloc::alloc(std::alloc::Layout::from_size_align(ALT_STACK_SIZE, 1).unwrap())
-                    as *mut libc::c_void;
-            alt_stack.ss_size = ALT_STACK_SIZE;
-            libc::sigaltstack(&alt_stack, std::ptr::null_mut());
-
-            let mut sa: libc::sigaction = std::mem::zeroed();
-            sa.sa_sigaction = print_stack_trace as libc::sighandler_t;
-            sa.sa_flags = libc::SA_NODEFER | libc::SA_RESETHAND | libc::SA_ONSTACK;
-            libc::sigemptyset(&mut sa.sa_mask);
-            libc::sigaction(libc::SIGSEGV, &sa, std::ptr::null_mut());
-        }
-    }
-}
-
-#[cfg(not(all(unix, any(target_env = "gnu", target_os = "macos"))))]
-mod signal_handler {
-    pub(super) fn install() {}
-}
-
-pub fn main() -> ! {
-    let start_time = Instant::now();
-    let start_rss = get_resident_set_size();
-    signal_handler::install();
-    let mut callbacks = TimePassesCallbacks::default();
-    install_ice_hook();
-    let exit_code = catch_with_exit_code(|| {
-        let args = env::args_os()
-            .enumerate()
-            .map(|(i, arg)| {
-                arg.into_string().unwrap_or_else(|arg| {
-                    early_error(
-                        ErrorOutputType::default(),
-                        &format!("argument {i} is not valid Unicode: {arg:?}"),
-                    )
-                })
-            })
-            .collect::<Vec<_>>();
-        RunCompiler::new(&args, &mut callbacks).run()
-    });
-
-    if callbacks.time_passes {
-        let end_rss = get_resident_set_size();
-        print_time_passes_entry("total", start_time.elapsed(), start_rss, end_rss);
-    }
-
-    process::exit(exit_code)
-}
+pub use rustc_driver_impl::*;
diff --git a/compiler/rustc_driver/src/session_diagnostics.rs b/compiler/rustc_driver/src/session_diagnostics.rs
deleted file mode 100644
index c1bc10891..000000000
--- a/compiler/rustc_driver/src/session_diagnostics.rs
+++ /dev/null
@@ -1,40 +0,0 @@
-use rustc_macros::Diagnostic;
-
-#[derive(Diagnostic)]
-#[diag(driver_rlink_unable_to_read)]
-pub(crate) struct RlinkUnableToRead {
-    pub err: std::io::Error,
-}
-
-#[derive(Diagnostic)]
-#[diag(driver_rlink_wrong_file_type)]
-pub(crate) struct RLinkWrongFileType;
-
-#[derive(Diagnostic)]
-#[diag(driver_rlink_empty_version_number)]
-pub(crate) struct RLinkEmptyVersionNumber;
-
-#[derive(Diagnostic)]
-#[diag(driver_rlink_encoding_version_mismatch)]
-pub(crate) struct RLinkEncodingVersionMismatch {
-    pub version_array: String,
-    pub rlink_version: u32,
-}
-
-#[derive(Diagnostic)]
-#[diag(driver_rlink_rustc_version_mismatch)]
-pub(crate) struct RLinkRustcVersionMismatch<'a> {
-    pub rustc_version: String,
-    pub current_version: &'a str,
-}
-
-#[derive(Diagnostic)]
-#[diag(driver_rlink_no_a_file)]
-pub(crate) struct RlinkNotAFile;
-
-#[derive(Diagnostic)]
-#[diag(driver_unpretty_dump_fail)]
-pub(crate) struct UnprettyDumpFail {
-    pub path: String,
-    pub err: String,
-}
diff --git a/compiler/rustc_driver_impl/Cargo.toml b/compiler/rustc_driver_impl/Cargo.toml
new file mode 100644
index 000000000..7b59a52cf
--- /dev/null
+++ b/compiler/rustc_driver_impl/Cargo.toml
@@ -0,0 +1,64 @@
+[package]
+name = "rustc_driver_impl"
+version = "0.0.0"
+edition = "2021"
+
+[lib]
+
+[dependencies]
+tracing = { version = "0.1.35" }
+serde_json = "1.0.59"
+rustc_log = { path = "../rustc_log" }
+rustc_ast_lowering = { path = "../rustc_ast_lowering" }
+rustc_ast_passes = { path = "../rustc_ast_passes" }
+rustc_attr = { path = "../rustc_attr" }
+rustc_borrowck = { path = "../rustc_borrowck" }
+rustc_builtin_macros = { path = "../rustc_builtin_macros" }
+rustc_const_eval = { path = "../rustc_const_eval" }
+rustc_error_messages = { path = "../rustc_error_messages" }
+rustc_expand = { path = "../rustc_expand" }
+rustc_hir_typeck = { path = "../rustc_hir_typeck" }
+rustc_incremental = { path = "../rustc_incremental" }
+rustc_infer = { path = "../rustc_infer" }
+rustc_mir_build = { path = "../rustc_mir_build" }
+rustc_mir_dataflow = { path = "../rustc_mir_dataflow" }
+rustc_monomorphize = { path = "../rustc_monomorphize" }
+rustc_passes = { path = "../rustc_passes" }
+rustc_privacy = { path = "../rustc_privacy" }
+rustc_query_system = { path = "../rustc_query_system" }
+rustc_resolve = { path = "../rustc_resolve" }
+rustc_symbol_mangling = { path = "../rustc_symbol_mangling" }
+rustc_trait_selection = { path = "../rustc_trait_selection" }
+rustc_ty_utils = { path = "../rustc_ty_utils" }
+rustc_middle = { path = "../rustc_middle" }
+rustc_ast_pretty = { path = "../rustc_ast_pretty" }
+rustc_target = { path = "../rustc_target" }
+rustc_lint = { path = "../rustc_lint" }
+rustc_data_structures = { path = "../rustc_data_structures" }
+rustc_errors = { path = "../rustc_errors" }
+rustc_feature = { path = "../rustc_feature" }
+rustc_hir = { path = "../rustc_hir" }
+rustc_hir_pretty = { path = "../rustc_hir_pretty" }
+rustc_macros = { path = "../rustc_macros" }
+rustc_metadata = { path = "../rustc_metadata" }
+rustc_parse = { path = "../rustc_parse" }
+rustc_plugin_impl = { path = "../rustc_plugin_impl" }
+rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }
+rustc_session = { path = "../rustc_session" }
+rustc_error_codes = { path = "../rustc_error_codes" }
+rustc_interface = { path = "../rustc_interface" }
+rustc_ast = { path = "../rustc_ast" }
+rustc_span = { path = "../rustc_span" }
+rustc_hir_analysis = { path = "../rustc_hir_analysis" }
+
+[target.'cfg(unix)'.dependencies]
+libc = "0.2"
+
+[target.'cfg(windows)'.dependencies]
+winapi = { version = "0.3", features = ["consoleapi", "debugapi", "processenv"] }
+
+[features]
+llvm = ['rustc_interface/llvm']
+max_level_info = ['rustc_log/max_level_info']
+rustc_use_parallel_compiler = ['rustc_data_structures/rustc_use_parallel_compiler', 'rustc_interface/rustc_use_parallel_compiler',
+    'rustc_middle/rustc_use_parallel_compiler']
diff --git a/compiler/rustc_driver/README.md b/compiler/rustc_driver_impl/README.md
index 6d7fba36f..6d7fba36f 100644
--- a/compiler/rustc_driver/README.md
+++ b/compiler/rustc_driver_impl/README.md
diff --git a/compiler/rustc_driver_impl/locales/en-US.ftl b/compiler/rustc_driver_impl/locales/en-US.ftl
new file mode 100644
index 000000000..f19b1ff64
--- /dev/null
+++ b/compiler/rustc_driver_impl/locales/en-US.ftl
@@ -0,0 +1,19 @@
+driver_impl_rlink_unable_to_read = failed to read rlink file: `{$err}`
+
+driver_impl_rlink_wrong_file_type = The input does not look like a .rlink file
+
+driver_impl_rlink_empty_version_number = The input does not contain version number
+
+driver_impl_rlink_encoding_version_mismatch = .rlink file was produced with encoding version `{$version_array}`, but the current version is `{$rlink_version}`
+
+driver_impl_rlink_rustc_version_mismatch = .rlink file was produced by rustc version `{$rustc_version}`, but the current version is `{$current_version}`
+
+driver_impl_rlink_no_a_file = rlink must be a file
+
+driver_impl_unpretty_dump_fail = pretty-print failed to write `{$path}` due to error `{$err}`
+
+driver_impl_ice = the compiler unexpectedly panicked. this is a bug.
+driver_impl_ice_bug_report = we would appreciate a bug report: {$bug_report_url}
+driver_impl_ice_version = rustc {$version} running on {$triple}
+driver_impl_ice_flags = compiler flags: {$flags}
+driver_impl_ice_exclude_cargo_defaults = some of the compiler flags provided by cargo are hidden
diff --git a/compiler/rustc_driver/src/args.rs b/compiler/rustc_driver_impl/src/args.rs
index 42c97cc6a..42c97cc6a 100644
--- a/compiler/rustc_driver/src/args.rs
+++ b/compiler/rustc_driver_impl/src/args.rs
diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs
new file mode 100644
index 000000000..464ddae47
--- /dev/null
+++ b/compiler/rustc_driver_impl/src/lib.rs
@@ -0,0 +1,1364 @@
+//! The Rust compiler.
+//!
+//! # Note
+//!
+//! This API is completely unstable and subject to change.
+
+#![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
+#![feature(is_terminal)]
+#![feature(once_cell)]
+#![feature(decl_macro)]
+#![recursion_limit = "256"]
+#![allow(rustc::potential_query_instability)]
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
+
+#[macro_use]
+extern crate tracing;
+
+pub extern crate rustc_plugin_impl as plugin;
+
+use rustc_ast as ast;
+use rustc_codegen_ssa::{traits::CodegenBackend, CodegenErrors, CodegenResults};
+use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry};
+use rustc_data_structures::sync::SeqCst;
+use rustc_errors::registry::{InvalidErrorCode, Registry};
+use rustc_errors::{
+    DiagnosticMessage, ErrorGuaranteed, PResult, SubdiagnosticMessage, TerminalUrl,
+};
+use rustc_feature::find_gated_cfg;
+use rustc_interface::util::{self, collect_crate_types, get_codegen_backend};
+use rustc_interface::{interface, Queries};
+use rustc_lint::LintStore;
+use rustc_macros::fluent_messages;
+use rustc_metadata::locator;
+use rustc_session::config::{nightly_options, CG_OPTIONS, Z_OPTIONS};
+use rustc_session::config::{ErrorOutputType, Input, OutputType, PrintRequest, TrimmedDefPaths};
+use rustc_session::cstore::MetadataLoader;
+use rustc_session::getopts;
+use rustc_session::lint::{Lint, LintId};
+use rustc_session::{config, Session};
+use rustc_session::{early_error, early_error_no_abort, early_warn};
+use rustc_span::source_map::{FileLoader, FileName};
+use rustc_span::symbol::sym;
+use rustc_target::json::ToJson;
+
+use std::cmp::max;
+use std::env;
+use std::ffi::OsString;
+use std::fs;
+use std::io::{self, IsTerminal, Read, Write};
+use std::panic::{self, catch_unwind};
+use std::path::PathBuf;
+use std::process::{self, Command, Stdio};
+use std::str;
+use std::sync::LazyLock;
+use std::time::Instant;
+
+pub mod args;
+pub mod pretty;
+mod session_diagnostics;
+
+use crate::session_diagnostics::{
+    RLinkEmptyVersionNumber, RLinkEncodingVersionMismatch, RLinkRustcVersionMismatch,
+    RLinkWrongFileType, RlinkNotAFile, RlinkUnableToRead,
+};
+
+fluent_messages! { "../locales/en-US.ftl" }
+
+pub static DEFAULT_LOCALE_RESOURCES: &[&str] = &[
+    // tidy-alphabetical-start
+    crate::DEFAULT_LOCALE_RESOURCE,
+    rustc_ast_lowering::DEFAULT_LOCALE_RESOURCE,
+    rustc_ast_passes::DEFAULT_LOCALE_RESOURCE,
+    rustc_attr::DEFAULT_LOCALE_RESOURCE,
+    rustc_borrowck::DEFAULT_LOCALE_RESOURCE,
+    rustc_builtin_macros::DEFAULT_LOCALE_RESOURCE,
+    rustc_codegen_ssa::DEFAULT_LOCALE_RESOURCE,
+    rustc_const_eval::DEFAULT_LOCALE_RESOURCE,
+    rustc_error_messages::DEFAULT_LOCALE_RESOURCE,
+    rustc_expand::DEFAULT_LOCALE_RESOURCE,
+    rustc_hir_analysis::DEFAULT_LOCALE_RESOURCE,
+    rustc_hir_typeck::DEFAULT_LOCALE_RESOURCE,
+    rustc_incremental::DEFAULT_LOCALE_RESOURCE,
+    rustc_infer::DEFAULT_LOCALE_RESOURCE,
+    rustc_interface::DEFAULT_LOCALE_RESOURCE,
+    rustc_lint::DEFAULT_LOCALE_RESOURCE,
+    rustc_metadata::DEFAULT_LOCALE_RESOURCE,
+    rustc_middle::DEFAULT_LOCALE_RESOURCE,
+    rustc_mir_build::DEFAULT_LOCALE_RESOURCE,
+    rustc_mir_dataflow::DEFAULT_LOCALE_RESOURCE,
+    rustc_monomorphize::DEFAULT_LOCALE_RESOURCE,
+    rustc_parse::DEFAULT_LOCALE_RESOURCE,
+    rustc_passes::DEFAULT_LOCALE_RESOURCE,
+    rustc_plugin_impl::DEFAULT_LOCALE_RESOURCE,
+    rustc_privacy::DEFAULT_LOCALE_RESOURCE,
+    rustc_query_system::DEFAULT_LOCALE_RESOURCE,
+    rustc_resolve::DEFAULT_LOCALE_RESOURCE,
+    rustc_session::DEFAULT_LOCALE_RESOURCE,
+    rustc_symbol_mangling::DEFAULT_LOCALE_RESOURCE,
+    rustc_trait_selection::DEFAULT_LOCALE_RESOURCE,
+    rustc_ty_utils::DEFAULT_LOCALE_RESOURCE,
+    // tidy-alphabetical-end
+];
+
+/// Exit status code used for successful compilation and help output.
+pub const EXIT_SUCCESS: i32 = 0;
+
+/// Exit status code used for compilation failures and invalid flags.
+pub const EXIT_FAILURE: i32 = 1;
+
+const BUG_REPORT_URL: &str = "https://github.com/rust-lang/rust/issues/new\
+    ?labels=C-bug%2C+I-ICE%2C+T-compiler&template=ice.md";
+
+const ICE_REPORT_COMPILER_FLAGS: &[&str] = &["-Z", "-C", "--crate-type"];
+
+const ICE_REPORT_COMPILER_FLAGS_EXCLUDE: &[&str] = &["metadata", "extra-filename"];
+
+const ICE_REPORT_COMPILER_FLAGS_STRIP_VALUE: &[&str] = &["incremental"];
+
+pub fn abort_on_err<T>(result: Result<T, ErrorGuaranteed>, sess: &Session) -> T {
+    match result {
+        Err(..) => {
+            sess.abort_if_errors();
+            panic!("error reported but abort_if_errors didn't abort???");
+        }
+        Ok(x) => x,
+    }
+}
+
+pub trait Callbacks {
+    /// Called before creating the compiler instance
+    fn config(&mut self, _config: &mut interface::Config) {}
+    /// Called after parsing. Return value instructs the compiler whether to
+    /// continue the compilation afterwards (defaults to `Compilation::Continue`)
+    fn after_parsing<'tcx>(
+        &mut self,
+        _compiler: &interface::Compiler,
+        _queries: &'tcx Queries<'tcx>,
+    ) -> Compilation {
+        Compilation::Continue
+    }
+    /// Called after expansion. Return value instructs the compiler whether to
+    /// continue the compilation afterwards (defaults to `Compilation::Continue`)
+    fn after_expansion<'tcx>(
+        &mut self,
+        _compiler: &interface::Compiler,
+        _queries: &'tcx Queries<'tcx>,
+    ) -> Compilation {
+        Compilation::Continue
+    }
+    /// Called after analysis. Return value instructs the compiler whether to
+    /// continue the compilation afterwards (defaults to `Compilation::Continue`)
+    fn after_analysis<'tcx>(
+        &mut self,
+        _compiler: &interface::Compiler,
+        _queries: &'tcx Queries<'tcx>,
+    ) -> Compilation {
+        Compilation::Continue
+    }
+}
+
+#[derive(Default)]
+pub struct TimePassesCallbacks {
+    time_passes: bool,
+}
+
+impl Callbacks for TimePassesCallbacks {
+    // JUSTIFICATION: the session doesn't exist at this point.
+    #[allow(rustc::bad_opt_access)]
+    fn config(&mut self, config: &mut interface::Config) {
+        // If a --print=... option has been given, we don't print the "total"
+        // time because it will mess up the --print output. See #64339.
+        //
+        self.time_passes = config.opts.prints.is_empty() && config.opts.unstable_opts.time_passes;
+        config.opts.trimmed_def_paths = TrimmedDefPaths::GoodPath;
+    }
+}
+
+pub fn diagnostics_registry() -> Registry {
+    Registry::new(rustc_error_codes::DIAGNOSTICS)
+}
+
+/// This is the primary entry point for rustc.
+pub struct RunCompiler<'a, 'b> {
+    at_args: &'a [String],
+    callbacks: &'b mut (dyn Callbacks + Send),
+    file_loader: Option<Box<dyn FileLoader + Send + Sync>>,
+    make_codegen_backend:
+        Option<Box<dyn FnOnce(&config::Options) -> Box<dyn CodegenBackend> + Send>>,
+}
+
+impl<'a, 'b> RunCompiler<'a, 'b> {
+    pub fn new(at_args: &'a [String], callbacks: &'b mut (dyn Callbacks + Send)) -> Self {
+        Self { at_args, callbacks, file_loader: None, make_codegen_backend: None }
+    }
+
+    /// Set a custom codegen backend.
+    ///
+    /// Has no uses within this repository, but is used by bjorn3 for "the
+    /// hotswapping branch of cg_clif" for "setting the codegen backend from a
+    /// custom driver where the custom codegen backend has arbitrary data."
+    /// (See #102759.)
+    pub fn set_make_codegen_backend(
+        &mut self,
+        make_codegen_backend: Option<
+            Box<dyn FnOnce(&config::Options) -> Box<dyn CodegenBackend> + Send>,
+        >,
+    ) -> &mut Self {
+        self.make_codegen_backend = make_codegen_backend;
+        self
+    }
+
+    /// Load files from sources other than the file system.
+    ///
+    /// Has no uses within this repository, but may be used in the future by
+    /// bjorn3 for "hooking rust-analyzer's VFS into rustc at some point for
+    /// running rustc without having to save". (See #102759.)
+    pub fn set_file_loader(
+        &mut self,
+        file_loader: Option<Box<dyn FileLoader + Send + Sync>>,
+    ) -> &mut Self {
+        self.file_loader = file_loader;
+        self
+    }
+
+    /// Parse args and run the compiler.
+    pub fn run(self) -> interface::Result<()> {
+        run_compiler(self.at_args, self.callbacks, self.file_loader, self.make_codegen_backend)
+    }
+}
+
+fn run_compiler(
+    at_args: &[String],
+    callbacks: &mut (dyn Callbacks + Send),
+    file_loader: Option<Box<dyn FileLoader + Send + Sync>>,
+    make_codegen_backend: Option<
+        Box<dyn FnOnce(&config::Options) -> Box<dyn CodegenBackend> + Send>,
+    >,
+) -> interface::Result<()> {
+    let args = args::arg_expand_all(at_args);
+
+    let Some(matches) = handle_options(&args) else { return Ok(()) };
+
+    let sopts = config::build_session_options(&matches);
+
+    if let Some(ref code) = matches.opt_str("explain") {
+        handle_explain(diagnostics_registry(), code, sopts.error_format);
+        return Ok(());
+    }
+
+    let cfg = interface::parse_cfgspecs(matches.opt_strs("cfg"));
+    let check_cfg = interface::parse_check_cfg(matches.opt_strs("check-cfg"));
+    let (odir, ofile) = make_output(&matches);
+    let mut config = interface::Config {
+        opts: sopts,
+        crate_cfg: cfg,
+        crate_check_cfg: check_cfg,
+        input: Input::File(PathBuf::new()),
+        output_file: ofile,
+        output_dir: odir,
+        file_loader,
+        locale_resources: DEFAULT_LOCALE_RESOURCES,
+        lint_caps: Default::default(),
+        parse_sess_created: None,
+        register_lints: None,
+        override_queries: None,
+        make_codegen_backend,
+        registry: diagnostics_registry(),
+    };
+
+    match make_input(config.opts.error_format, &matches.free) {
+        Err(reported) => return Err(reported),
+        Ok(Some(input)) => {
+            config.input = input;
+
+            callbacks.config(&mut config);
+        }
+        Ok(None) => match matches.free.len() {
+            0 => {
+                callbacks.config(&mut config);
+                interface::run_compiler(config, |compiler| {
+                    let sopts = &compiler.session().opts;
+                    if sopts.describe_lints {
+                        let mut lint_store =
+                            rustc_lint::new_lint_store(compiler.session().enable_internal_lints());
+                        let registered_lints =
+                            if let Some(register_lints) = compiler.register_lints() {
+                                register_lints(compiler.session(), &mut lint_store);
+                                true
+                            } else {
+                                false
+                            };
+                        describe_lints(compiler.session(), &lint_store, registered_lints);
+                        return;
+                    }
+                    let should_stop =
+                        print_crate_info(&***compiler.codegen_backend(), compiler.session(), false);
+
+                    if should_stop == Compilation::Stop {
+                        return;
+                    }
+                    early_error(sopts.error_format, "no input filename given")
+                });
+                return Ok(());
+            }
+            1 => panic!("make_input should have provided valid inputs"),
+            _ => early_error(
+                config.opts.error_format,
+                &format!(
+                    "multiple input filenames provided (first two filenames are `{}` and `{}`)",
+                    matches.free[0], matches.free[1],
+                ),
+            ),
+        },
+    };
+
+    interface::run_compiler(config, |compiler| {
+        let sess = compiler.session();
+        let should_stop = print_crate_info(&***compiler.codegen_backend(), sess, true)
+            .and_then(|| list_metadata(sess, &*compiler.codegen_backend().metadata_loader()))
+            .and_then(|| try_process_rlink(sess, compiler));
+
+        if should_stop == Compilation::Stop {
+            return sess.compile_status();
+        }
+
+        let linker = compiler.enter(|queries| {
+            let early_exit = || sess.compile_status().map(|_| None);
+            queries.parse()?;
+
+            if let Some(ppm) = &sess.opts.pretty {
+                if ppm.needs_ast_map() {
+                    queries.global_ctxt()?.enter(|tcx| {
+                        pretty::print_after_hir_lowering(tcx, *ppm);
+                        Ok(())
+                    })?;
+                } else {
+                    let krate = queries.parse()?.steal();
+                    pretty::print_after_parsing(sess, &krate, *ppm);
+                }
+                trace!("finished pretty-printing");
+                return early_exit();
+            }
+
+            if callbacks.after_parsing(compiler, queries) == Compilation::Stop {
+                return early_exit();
+            }
+
+            if sess.opts.unstable_opts.parse_only || sess.opts.unstable_opts.show_span.is_some() {
+                return early_exit();
+            }
+
+            {
+                let plugins = queries.register_plugins()?;
+                let (_, lint_store) = &*plugins.borrow();
+
+                // Lint plugins are registered; now we can process command line flags.
+                if sess.opts.describe_lints {
+                    describe_lints(sess, lint_store, true);
+                    return early_exit();
+                }
+            }
+
+            // Make sure name resolution and macro expansion is run.
+            queries.global_ctxt()?.enter(|tcx| tcx.resolver_for_lowering(()));
+
+            if callbacks.after_expansion(compiler, queries) == Compilation::Stop {
+                return early_exit();
+            }
+
+            // Make sure the `output_filenames` query is run for its side
+            // effects of writing the dep-info and reporting errors.
+            queries.global_ctxt()?.enter(|tcx| tcx.output_filenames(()));
+
+            if sess.opts.output_types.contains_key(&OutputType::DepInfo)
+                && sess.opts.output_types.len() == 1
+            {
+                return early_exit();
+            }
+
+            if sess.opts.unstable_opts.no_analysis {
+                return early_exit();
+            }
+
+            queries.global_ctxt()?.enter(|tcx| tcx.analysis(()))?;
+
+            if callbacks.after_analysis(compiler, queries) == Compilation::Stop {
+                return early_exit();
+            }
+
+            queries.ongoing_codegen()?;
+
+            if sess.opts.unstable_opts.print_type_sizes {
+                sess.code_stats.print_type_sizes();
+            }
+
+            let linker = queries.linker()?;
+            Ok(Some(linker))
+        })?;
+
+        if let Some(linker) = linker {
+            let _timer = sess.timer("link");
+            linker.link()?
+        }
+
+        if sess.opts.unstable_opts.perf_stats {
+            sess.print_perf_stats();
+        }
+
+        if sess.opts.unstable_opts.print_fuel.is_some() {
+            eprintln!(
+                "Fuel used by {}: {}",
+                sess.opts.unstable_opts.print_fuel.as_ref().unwrap(),
+                sess.print_fuel.load(SeqCst)
+            );
+        }
+
+        Ok(())
+    })
+}
+
+// Extract output directory and file from matches.
+fn make_output(matches: &getopts::Matches) -> (Option<PathBuf>, Option<PathBuf>) {
+    let odir = matches.opt_str("out-dir").map(|o| PathBuf::from(&o));
+    let ofile = matches.opt_str("o").map(|o| PathBuf::from(&o));
+    (odir, ofile)
+}
+
+// Extract input (string or file and optional path) from matches.
+fn make_input(
+    error_format: ErrorOutputType,
+    free_matches: &[String],
+) -> Result<Option<Input>, ErrorGuaranteed> {
+    if free_matches.len() == 1 {
+        let ifile = &free_matches[0];
+        if ifile == "-" {
+            let mut src = String::new();
+            if io::stdin().read_to_string(&mut src).is_err() {
+                // Immediately stop compilation if there was an issue reading
+                // the input (for example if the input stream is not UTF-8).
+                let reported = early_error_no_abort(
+                    error_format,
+                    "couldn't read from stdin, as it did not contain valid UTF-8",
+                );
+                return Err(reported);
+            }
+            if let Ok(path) = env::var("UNSTABLE_RUSTDOC_TEST_PATH") {
+                let line = env::var("UNSTABLE_RUSTDOC_TEST_LINE").expect(
+                    "when UNSTABLE_RUSTDOC_TEST_PATH is set \
+                                    UNSTABLE_RUSTDOC_TEST_LINE also needs to be set",
+                );
+                let line = isize::from_str_radix(&line, 10)
+                    .expect("UNSTABLE_RUSTDOC_TEST_LINE needs to be an number");
+                let file_name = FileName::doc_test_source_code(PathBuf::from(path), line);
+                Ok(Some(Input::Str { name: file_name, input: src }))
+            } else {
+                Ok(Some(Input::Str { name: FileName::anon_source_code(&src), input: src }))
+            }
+        } else {
+            Ok(Some(Input::File(PathBuf::from(ifile))))
+        }
+    } else {
+        Ok(None)
+    }
+}
+
+/// Whether to stop or continue compilation.
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+pub enum Compilation {
+    Stop,
+    Continue,
+}
+
+impl Compilation {
+    pub fn and_then<F: FnOnce() -> Compilation>(self, next: F) -> Compilation {
+        match self {
+            Compilation::Stop => Compilation::Stop,
+            Compilation::Continue => next(),
+        }
+    }
+}
+
+fn handle_explain(registry: Registry, code: &str, output: ErrorOutputType) {
+    let upper_cased_code = code.to_ascii_uppercase();
+    let normalised =
+        if upper_cased_code.starts_with('E') { upper_cased_code } else { format!("E{code:0>4}") };
+    match registry.try_find_description(&normalised) {
+        Ok(description) => {
+            let mut is_in_code_block = false;
+            let mut text = String::new();
+            // Slice off the leading newline and print.
+            for line in description.lines() {
+                let indent_level =
+                    line.find(|c: char| !c.is_whitespace()).unwrap_or_else(|| line.len());
+                let dedented_line = &line[indent_level..];
+                if dedented_line.starts_with("```") {
+                    is_in_code_block = !is_in_code_block;
+                    text.push_str(&line[..(indent_level + 3)]);
+                } else if is_in_code_block && dedented_line.starts_with("# ") {
+                    continue;
+                } else {
+                    text.push_str(line);
+                }
+                text.push('\n');
+            }
+            if io::stdout().is_terminal() {
+                show_content_with_pager(&text);
+            } else {
+                print!("{text}");
+            }
+        }
+        Err(InvalidErrorCode) => {
+            early_error(output, &format!("{code} is not a valid error code"));
+        }
+    }
+}
+
+fn show_content_with_pager(content: &str) {
+    let pager_name = env::var_os("PAGER").unwrap_or_else(|| {
+        if cfg!(windows) { OsString::from("more.com") } else { OsString::from("less") }
+    });
+
+    let mut fallback_to_println = false;
+
+    match Command::new(pager_name).stdin(Stdio::piped()).spawn() {
+        Ok(mut pager) => {
+            if let Some(pipe) = pager.stdin.as_mut() {
+                if pipe.write_all(content.as_bytes()).is_err() {
+                    fallback_to_println = true;
+                }
+            }
+
+            if pager.wait().is_err() {
+                fallback_to_println = true;
+            }
+        }
+        Err(_) => {
+            fallback_to_println = true;
+        }
+    }
+
+    // If pager fails for whatever reason, we should still print the content
+    // to standard output
+    if fallback_to_println {
+        print!("{content}");
+    }
+}
+
+pub fn try_process_rlink(sess: &Session, compiler: &interface::Compiler) -> Compilation {
+    if sess.opts.unstable_opts.link_only {
+        if let Input::File(file) = &sess.io.input {
+            // FIXME: #![crate_type] and #![crate_name] support not implemented yet
+            sess.init_crate_types(collect_crate_types(sess, &[]));
+            let outputs = compiler.build_output_filenames(sess, &[]);
+            let rlink_data = fs::read(file).unwrap_or_else(|err| {
+                sess.emit_fatal(RlinkUnableToRead { err });
+            });
+            let codegen_results = match CodegenResults::deserialize_rlink(rlink_data) {
+                Ok(codegen) => codegen,
+                Err(err) => {
+                    match err {
+                        CodegenErrors::WrongFileType => sess.emit_fatal(RLinkWrongFileType),
+                        CodegenErrors::EmptyVersionNumber => {
+                            sess.emit_fatal(RLinkEmptyVersionNumber)
+                        }
+                        CodegenErrors::EncodingVersionMismatch { version_array, rlink_version } => {
+                            sess.emit_fatal(RLinkEncodingVersionMismatch {
+                                version_array,
+                                rlink_version,
+                            })
+                        }
+                        CodegenErrors::RustcVersionMismatch { rustc_version, current_version } => {
+                            sess.emit_fatal(RLinkRustcVersionMismatch {
+                                rustc_version,
+                                current_version,
+                            })
+                        }
+                    };
+                }
+            };
+            let result = compiler.codegen_backend().link(sess, codegen_results, &outputs);
+            abort_on_err(result, sess);
+        } else {
+            sess.emit_fatal(RlinkNotAFile {})
+        }
+        Compilation::Stop
+    } else {
+        Compilation::Continue
+    }
+}
+
+pub fn list_metadata(sess: &Session, metadata_loader: &dyn MetadataLoader) -> Compilation {
+    if sess.opts.unstable_opts.ls {
+        match sess.io.input {
+            Input::File(ref ifile) => {
+                let path = &(*ifile);
+                let mut v = Vec::new();
+                locator::list_file_metadata(&sess.target, path, metadata_loader, &mut v).unwrap();
+                println!("{}", String::from_utf8(v).unwrap());
+            }
+            Input::Str { .. } => {
+                early_error(ErrorOutputType::default(), "cannot list metadata for stdin");
+            }
+        }
+        return Compilation::Stop;
+    }
+
+    Compilation::Continue
+}
+
+fn print_crate_info(
+    codegen_backend: &dyn CodegenBackend,
+    sess: &Session,
+    parse_attrs: bool,
+) -> Compilation {
+    use rustc_session::config::PrintRequest::*;
+    // NativeStaticLibs and LinkArgs are special - printed during linking
+    // (empty iterator returns true)
+    if sess.opts.prints.iter().all(|&p| p == NativeStaticLibs || p == LinkArgs) {
+        return Compilation::Continue;
+    }
+
+    let attrs = if parse_attrs {
+        let result = parse_crate_attrs(sess);
+        match result {
+            Ok(attrs) => Some(attrs),
+            Err(mut parse_error) => {
+                parse_error.emit();
+                return Compilation::Stop;
+            }
+        }
+    } else {
+        None
+    };
+    for req in &sess.opts.prints {
+        match *req {
+            TargetList => {
+                let mut targets = rustc_target::spec::TARGETS.to_vec();
+                targets.sort_unstable();
+                println!("{}", targets.join("\n"));
+            }
+            Sysroot => println!("{}", sess.sysroot.display()),
+            TargetLibdir => println!("{}", sess.target_tlib_path.dir.display()),
+            TargetSpec => {
+                println!("{}", serde_json::to_string_pretty(&sess.target.to_json()).unwrap());
+            }
+            FileNames | CrateName => {
+                let Some(attrs) = attrs.as_ref() else {
+                    // no crate attributes, print out an error and exit
+                    return Compilation::Continue;
+                };
+                let t_outputs = rustc_interface::util::build_output_filenames(attrs, sess);
+                let id = rustc_session::output::find_crate_name(sess, attrs);
+                if *req == PrintRequest::CrateName {
+                    println!("{id}");
+                    continue;
+                }
+                let crate_types = collect_crate_types(sess, attrs);
+                for &style in &crate_types {
+                    let fname =
+                        rustc_session::output::filename_for_input(sess, style, id, &t_outputs);
+                    println!("{}", fname.file_name().unwrap().to_string_lossy());
+                }
+            }
+            Cfg => {
+                let mut cfgs = sess
+                    .parse_sess
+                    .config
+                    .iter()
+                    .filter_map(|&(name, value)| {
+                        // Note that crt-static is a specially recognized cfg
+                        // directive that's printed out here as part of
+                        // rust-lang/rust#37406, but in general the
+                        // `target_feature` cfg is gated under
+                        // rust-lang/rust#29717. For now this is just
+                        // specifically allowing the crt-static cfg and that's
+                        // it, this is intended to get into Cargo and then go
+                        // through to build scripts.
+                        if (name != sym::target_feature || value != Some(sym::crt_dash_static))
+                            && !sess.is_nightly_build()
+                            && find_gated_cfg(|cfg_sym| cfg_sym == name).is_some()
+                        {
+                            return None;
+                        }
+
+                        if let Some(value) = value {
+                            Some(format!("{name}=\"{value}\""))
+                        } else {
+                            Some(name.to_string())
+                        }
+                    })
+                    .collect::<Vec<String>>();
+
+                cfgs.sort();
+                for cfg in cfgs {
+                    println!("{cfg}");
+                }
+            }
+            CallingConventions => {
+                let mut calling_conventions = rustc_target::spec::abi::all_names();
+                calling_conventions.sort_unstable();
+                println!("{}", calling_conventions.join("\n"));
+            }
+            RelocationModels
+            | CodeModels
+            | TlsModels
+            | TargetCPUs
+            | StackProtectorStrategies
+            | TargetFeatures => {
+                codegen_backend.print(*req, sess);
+            }
+            // Any output here interferes with Cargo's parsing of other printed output
+            NativeStaticLibs => {}
+            LinkArgs => {}
+            SplitDebuginfo => {
+                use rustc_target::spec::SplitDebuginfo::{Off, Packed, Unpacked};
+
+                for split in &[Off, Packed, Unpacked] {
+                    let stable = sess.target.options.supported_split_debuginfo.contains(split);
+                    let unstable_ok = sess.unstable_options();
+                    if stable || unstable_ok {
+                        println!("{split}");
+                    }
+                }
+            }
+        }
+    }
+    Compilation::Stop
+}
+
+/// Prints version information
+///
+/// NOTE: this is a macro to support drivers built at a different time than the main `rustc_driver` crate.
+pub macro version($binary: literal, $matches: expr) {
+    fn unw(x: Option<&str>) -> &str {
+        x.unwrap_or("unknown")
+    }
+    $crate::version_at_macro_invocation(
+        $binary,
+        $matches,
+        unw(option_env!("CFG_VERSION")),
+        unw(option_env!("CFG_VER_HASH")),
+        unw(option_env!("CFG_VER_DATE")),
+        unw(option_env!("CFG_RELEASE")),
+    )
+}
+
+#[doc(hidden)] // use the macro instead
+pub fn version_at_macro_invocation(
+    binary: &str,
+    matches: &getopts::Matches,
+    version: &str,
+    commit_hash: &str,
+    commit_date: &str,
+    release: &str,
+) {
+    let verbose = matches.opt_present("verbose");
+
+    println!("{binary} {version}");
+
+    if verbose {
+        println!("binary: {binary}");
+        println!("commit-hash: {commit_hash}");
+        println!("commit-date: {commit_date}");
+        println!("host: {}", config::host_triple());
+        println!("release: {release}");
+
+        let debug_flags = matches.opt_strs("Z");
+        let backend_name = debug_flags.iter().find_map(|x| x.strip_prefix("codegen-backend="));
+        get_codegen_backend(&None, backend_name).print_version();
+    }
+}
+
+fn usage(verbose: bool, include_unstable_options: bool, nightly_build: bool) {
+    let groups = if verbose { config::rustc_optgroups() } else { config::rustc_short_optgroups() };
+    let mut options = getopts::Options::new();
+    for option in groups.iter().filter(|x| include_unstable_options || x.is_stable()) {
+        (option.apply)(&mut options);
+    }
+    let message = "Usage: rustc [OPTIONS] INPUT";
+    let nightly_help = if nightly_build {
+        "\n    -Z help             Print unstable compiler options"
+    } else {
+        ""
+    };
+    let verbose_help = if verbose {
+        ""
+    } else {
+        "\n    --help -v           Print the full set of options rustc accepts"
+    };
+    let at_path = if verbose {
+        "    @path               Read newline separated options from `path`\n"
+    } else {
+        ""
+    };
+    println!(
+        "{options}{at_path}\nAdditional help:
+    -C help             Print codegen options
+    -W help             \
+              Print 'lint' options and default settings{nightly}{verbose}\n",
+        options = options.usage(message),
+        at_path = at_path,
+        nightly = nightly_help,
+        verbose = verbose_help
+    );
+}
+
+fn print_wall_help() {
+    println!(
+        "
+The flag `-Wall` does not exist in `rustc`. Most useful lints are enabled by
+default. Use `rustc -W help` to see all available lints. It's more common to put
+warning settings in the crate root using `#![warn(LINT_NAME)]` instead of using
+the command line flag directly.
+"
+    );
+}
+
+/// Write to stdout lint command options, together with a list of all available lints
+pub fn describe_lints(sess: &Session, lint_store: &LintStore, loaded_plugins: bool) {
+    println!(
+        "
+Available lint options:
+    -W <foo>           Warn about <foo>
+    -A <foo>           \
+              Allow <foo>
+    -D <foo>           Deny <foo>
+    -F <foo>           Forbid <foo> \
+              (deny <foo> and all attempts to override)
+
+"
+    );
+
+    fn sort_lints(sess: &Session, mut lints: Vec<&'static Lint>) -> Vec<&'static Lint> {
+        // The sort doesn't case-fold but it's doubtful we care.
+        lints.sort_by_cached_key(|x: &&Lint| (x.default_level(sess.edition()), x.name));
+        lints
+    }
+
+    fn sort_lint_groups(
+        lints: Vec<(&'static str, Vec<LintId>, bool)>,
+    ) -> Vec<(&'static str, Vec<LintId>)> {
+        let mut lints: Vec<_> = lints.into_iter().map(|(x, y, _)| (x, y)).collect();
+        lints.sort_by_key(|l| l.0);
+        lints
+    }
+
+    let (plugin, builtin): (Vec<_>, _) =
+        lint_store.get_lints().iter().cloned().partition(|&lint| lint.is_plugin);
+    let plugin = sort_lints(sess, plugin);
+    let builtin = sort_lints(sess, builtin);
+
+    let (plugin_groups, builtin_groups): (Vec<_>, _) =
+        lint_store.get_lint_groups().partition(|&(.., p)| p);
+    let plugin_groups = sort_lint_groups(plugin_groups);
+    let builtin_groups = sort_lint_groups(builtin_groups);
+
+    let max_name_len =
+        plugin.iter().chain(&builtin).map(|&s| s.name.chars().count()).max().unwrap_or(0);
+    let padded = |x: &str| {
+        let mut s = " ".repeat(max_name_len - x.chars().count());
+        s.push_str(x);
+        s
+    };
+
+    println!("Lint checks provided by rustc:\n");
+
+    let print_lints = |lints: Vec<&Lint>| {
+        println!("    {}  {:7.7}  {}", padded("name"), "default", "meaning");
+        println!("    {}  {:7.7}  {}", padded("----"), "-------", "-------");
+        for lint in lints {
+            let name = lint.name_lower().replace('_', "-");
+            println!(
+                "    {}  {:7.7}  {}",
+                padded(&name),
+                lint.default_level(sess.edition()).as_str(),
+                lint.desc
+            );
+        }
+        println!("\n");
+    };
+
+    print_lints(builtin);
+
+    let max_name_len = max(
+        "warnings".len(),
+        plugin_groups
+            .iter()
+            .chain(&builtin_groups)
+            .map(|&(s, _)| s.chars().count())
+            .max()
+            .unwrap_or(0),
+    );
+
+    let padded = |x: &str| {
+        let mut s = " ".repeat(max_name_len - x.chars().count());
+        s.push_str(x);
+        s
+    };
+
+    println!("Lint groups provided by rustc:\n");
+
+    let print_lint_groups = |lints: Vec<(&'static str, Vec<LintId>)>, all_warnings| {
+        println!("    {}  sub-lints", padded("name"));
+        println!("    {}  ---------", padded("----"));
+
+        if all_warnings {
+            println!("    {}  all lints that are set to issue warnings", padded("warnings"));
+        }
+
+        for (name, to) in lints {
+            let name = name.to_lowercase().replace('_', "-");
+            let desc = to
+                .into_iter()
+                .map(|x| x.to_string().replace('_', "-"))
+                .collect::<Vec<String>>()
+                .join(", ");
+            println!("    {}  {}", padded(&name), desc);
+        }
+        println!("\n");
+    };
+
+    print_lint_groups(builtin_groups, true);
+
+    match (loaded_plugins, plugin.len(), plugin_groups.len()) {
+        (false, 0, _) | (false, _, 0) => {
+            println!("Lint tools like Clippy can provide additional lints and lint groups.");
+        }
+        (false, ..) => panic!("didn't load lint plugins but got them anyway!"),
+        (true, 0, 0) => println!("This crate does not load any lint plugins or lint groups."),
+        (true, l, g) => {
+            if l > 0 {
+                println!("Lint checks provided by plugins loaded by this crate:\n");
+                print_lints(plugin);
+            }
+            if g > 0 {
+                println!("Lint groups provided by plugins loaded by this crate:\n");
+                print_lint_groups(plugin_groups, false);
+            }
+        }
+    }
+}
+
+fn describe_debug_flags() {
+    println!("\nAvailable options:\n");
+    print_flag_list("-Z", config::Z_OPTIONS);
+}
+
+fn describe_codegen_flags() {
+    println!("\nAvailable codegen options:\n");
+    print_flag_list("-C", config::CG_OPTIONS);
+}
+
+pub fn print_flag_list<T>(
+    cmdline_opt: &str,
+    flag_list: &[(&'static str, T, &'static str, &'static str)],
+) {
+    let max_len = flag_list.iter().map(|&(name, _, _, _)| name.chars().count()).max().unwrap_or(0);
+
+    for &(name, _, _, desc) in flag_list {
+        println!(
+            "    {} {:>width$}=val -- {}",
+            cmdline_opt,
+            name.replace('_', "-"),
+            desc,
+            width = max_len
+        );
+    }
+}
+
+/// Process command line options. Emits messages as appropriate. If compilation
+/// should continue, returns a getopts::Matches object parsed from args,
+/// otherwise returns `None`.
+///
+/// The compiler's handling of options is a little complicated as it ties into
+/// our stability story. The current intention of each compiler option is to
+/// have one of two modes:
+///
+/// 1. An option is stable and can be used everywhere.
+/// 2. An option is unstable, and can only be used on nightly.
+///
+/// Like unstable library and language features, however, unstable options have
+/// always required a form of "opt in" to indicate that you're using them. This
+/// provides the easy ability to scan a code base to check to see if anything
+/// unstable is being used. Currently, this "opt in" is the `-Z` "zed" flag.
+///
+/// All options behind `-Z` are considered unstable by default. Other top-level
+/// options can also be considered unstable, and they were unlocked through the
+/// `-Z unstable-options` flag. Note that `-Z` remains to be the root of
+/// instability in both cases, though.
+///
+/// So with all that in mind, the comments below have some more detail about the
+/// contortions done here to get things to work out correctly.
+pub fn handle_options(args: &[String]) -> Option<getopts::Matches> {
+    // Throw away the first argument, the name of the binary
+    let args = &args[1..];
+
+    if args.is_empty() {
+        // user did not write `-v` nor `-Z unstable-options`, so do not
+        // include that extra information.
+        let nightly_build =
+            rustc_feature::UnstableFeatures::from_environment(None).is_nightly_build();
+        usage(false, false, nightly_build);
+        return None;
+    }
+
+    // Parse with *all* options defined in the compiler, we don't worry about
+    // option stability here we just want to parse as much as possible.
+    let mut options = getopts::Options::new();
+    for option in config::rustc_optgroups() {
+        (option.apply)(&mut options);
+    }
+    let matches = options.parse(args).unwrap_or_else(|e| {
+        let msg = match e {
+            getopts::Fail::UnrecognizedOption(ref opt) => CG_OPTIONS
+                .iter()
+                .map(|&(name, ..)| ('C', name))
+                .chain(Z_OPTIONS.iter().map(|&(name, ..)| ('Z', name)))
+                .find(|&(_, name)| *opt == name.replace('_', "-"))
+                .map(|(flag, _)| format!("{e}. Did you mean `-{flag} {opt}`?")),
+            _ => None,
+        };
+        early_error(ErrorOutputType::default(), &msg.unwrap_or_else(|| e.to_string()));
+    });
+
+    // For all options we just parsed, we check a few aspects:
+    //
+    // * If the option is stable, we're all good
+    // * If the option wasn't passed, we're all good
+    // * If `-Z unstable-options` wasn't passed (and we're not a -Z option
+    //   ourselves), then we require the `-Z unstable-options` flag to unlock
+    //   this option that was passed.
+    // * If we're a nightly compiler, then unstable options are now unlocked, so
+    //   we're good to go.
+    // * Otherwise, if we're an unstable option then we generate an error
+    //   (unstable option being used on stable)
+    nightly_options::check_nightly_options(&matches, &config::rustc_optgroups());
+
+    if matches.opt_present("h") || matches.opt_present("help") {
+        // Only show unstable options in --help if we accept unstable options.
+        let unstable_enabled = nightly_options::is_unstable_enabled(&matches);
+        let nightly_build = nightly_options::match_is_nightly_build(&matches);
+        usage(matches.opt_present("verbose"), unstable_enabled, nightly_build);
+        return None;
+    }
+
+    // Handle the special case of -Wall.
+    let wall = matches.opt_strs("W");
+    if wall.iter().any(|x| *x == "all") {
+        print_wall_help();
+        rustc_errors::FatalError.raise();
+    }
+
+    // Don't handle -W help here, because we might first load plugins.
+    let debug_flags = matches.opt_strs("Z");
+    if debug_flags.iter().any(|x| *x == "help") {
+        describe_debug_flags();
+        return None;
+    }
+
+    let cg_flags = matches.opt_strs("C");
+
+    if cg_flags.iter().any(|x| *x == "help") {
+        describe_codegen_flags();
+        return None;
+    }
+
+    if cg_flags.iter().any(|x| *x == "no-stack-check") {
+        early_warn(
+            ErrorOutputType::default(),
+            "the --no-stack-check flag is deprecated and does nothing",
+        );
+    }
+
+    if cg_flags.iter().any(|x| *x == "passes=list") {
+        let backend_name = debug_flags.iter().find_map(|x| x.strip_prefix("codegen-backend="));
+        get_codegen_backend(&None, backend_name).print_passes();
+        return None;
+    }
+
+    if matches.opt_present("version") {
+        version!("rustc", &matches);
+        return None;
+    }
+
+    Some(matches)
+}
+
+fn parse_crate_attrs<'a>(sess: &'a Session) -> PResult<'a, ast::AttrVec> {
+    match &sess.io.input {
+        Input::File(ifile) => rustc_parse::parse_crate_attrs_from_file(ifile, &sess.parse_sess),
+        Input::Str { name, input } => rustc_parse::parse_crate_attrs_from_source_str(
+            name.clone(),
+            input.clone(),
+            &sess.parse_sess,
+        ),
+    }
+}
+
+/// Gets a list of extra command-line flags provided by the user, as strings.
+///
+/// This function is used during ICEs to show more information useful for
+/// debugging, since some ICEs only happens with non-default compiler flags
+/// (and the users don't always report them).
+fn extra_compiler_flags() -> Option<(Vec<String>, bool)> {
+    let mut args = env::args_os().map(|arg| arg.to_string_lossy().to_string()).peekable();
+
+    let mut result = Vec::new();
+    let mut excluded_cargo_defaults = false;
+    while let Some(arg) = args.next() {
+        if let Some(a) = ICE_REPORT_COMPILER_FLAGS.iter().find(|a| arg.starts_with(*a)) {
+            let content = if arg.len() == a.len() {
+                // A space-separated option, like `-C incremental=foo` or `--crate-type rlib`
+                match args.next() {
+                    Some(arg) => arg.to_string(),
+                    None => continue,
+                }
+            } else if arg.get(a.len()..a.len() + 1) == Some("=") {
+                // An equals option, like `--crate-type=rlib`
+                arg[a.len() + 1..].to_string()
+            } else {
+                // A non-space option, like `-Cincremental=foo`
+                arg[a.len()..].to_string()
+            };
+            let option = content.split_once('=').map(|s| s.0).unwrap_or(&content);
+            if ICE_REPORT_COMPILER_FLAGS_EXCLUDE.iter().any(|exc| option == *exc) {
+                excluded_cargo_defaults = true;
+            } else {
+                result.push(a.to_string());
+                match ICE_REPORT_COMPILER_FLAGS_STRIP_VALUE.iter().find(|s| option == **s) {
+                    Some(s) => result.push(format!("{s}=[REDACTED]")),
+                    None => result.push(content),
+                }
+            }
+        }
+    }
+
+    if !result.is_empty() { Some((result, excluded_cargo_defaults)) } else { None }
+}
+
+/// Runs a closure and catches unwinds triggered by fatal errors.
+///
+/// The compiler currently unwinds with a special sentinel value to abort
+/// compilation on fatal errors. This function catches that sentinel and turns
+/// the panic into a `Result` instead.
+pub fn catch_fatal_errors<F: FnOnce() -> R, R>(f: F) -> Result<R, ErrorGuaranteed> {
+    catch_unwind(panic::AssertUnwindSafe(f)).map_err(|value| {
+        if value.is::<rustc_errors::FatalErrorMarker>() {
+            ErrorGuaranteed::unchecked_claim_error_was_emitted()
+        } else {
+            panic::resume_unwind(value);
+        }
+    })
+}
+
+/// Variant of `catch_fatal_errors` for the `interface::Result` return type
+/// that also computes the exit code.
+pub fn catch_with_exit_code(f: impl FnOnce() -> interface::Result<()>) -> i32 {
+    let result = catch_fatal_errors(f).and_then(|result| result);
+    match result {
+        Ok(()) => EXIT_SUCCESS,
+        Err(_) => EXIT_FAILURE,
+    }
+}
+
+static DEFAULT_HOOK: LazyLock<Box<dyn Fn(&panic::PanicInfo<'_>) + Sync + Send + 'static>> =
+    LazyLock::new(|| {
+        let hook = panic::take_hook();
+        panic::set_hook(Box::new(|info| {
+            // If the error was caused by a broken pipe then this is not a bug.
+            // Write the error and return immediately. See #98700.
+            #[cfg(windows)]
+            if let Some(msg) = info.payload().downcast_ref::<String>() {
+                if msg.starts_with("failed printing to stdout: ") && msg.ends_with("(os error 232)")
+                {
+                    early_error_no_abort(ErrorOutputType::default(), &msg);
+                    return;
+                }
+            };
+
+            // Invoke the default handler, which prints the actual panic message and optionally a backtrace
+            // Don't do this for delayed bugs, which already emit their own more useful backtrace.
+            if !info.payload().is::<rustc_errors::DelayedBugPanic>() {
+                (*DEFAULT_HOOK)(info);
+
+                // Separate the output with an empty line
+                eprintln!();
+            }
+
+            // Print the ICE message
+            report_ice(info, BUG_REPORT_URL);
+        }));
+        hook
+    });
+
+/// Prints the ICE message, including query stack, but without backtrace.
+///
+/// The message will point the user at `bug_report_url` to report the ICE.
+///
+/// When `install_ice_hook` is called, this function will be called as the panic
+/// hook.
+pub fn report_ice(info: &panic::PanicInfo<'_>, bug_report_url: &str) {
+    let fallback_bundle =
+        rustc_errors::fallback_fluent_bundle(crate::DEFAULT_LOCALE_RESOURCES.to_vec(), false);
+    let emitter = Box::new(rustc_errors::emitter::EmitterWriter::stderr(
+        rustc_errors::ColorConfig::Auto,
+        None,
+        None,
+        fallback_bundle,
+        false,
+        false,
+        None,
+        false,
+        false,
+        TerminalUrl::No,
+    ));
+    let handler = rustc_errors::Handler::with_emitter(true, None, emitter);
+
+    // a .span_bug or .bug call has already printed what
+    // it wants to print.
+    if !info.payload().is::<rustc_errors::ExplicitBug>()
+        && !info.payload().is::<rustc_errors::DelayedBugPanic>()
+    {
+        handler.emit_err(session_diagnostics::Ice);
+    }
+
+    handler.emit_note(session_diagnostics::IceBugReport { bug_report_url });
+    handler.emit_note(session_diagnostics::IceVersion {
+        version: util::version_str!().unwrap_or("unknown_version"),
+        triple: config::host_triple(),
+    });
+
+    if let Some((flags, excluded_cargo_defaults)) = extra_compiler_flags() {
+        handler.emit_note(session_diagnostics::IceFlags { flags: flags.join(" ") });
+        if excluded_cargo_defaults {
+            handler.emit_note(session_diagnostics::IceExcludeCargoDefaults);
+        }
+    }
+
+    // If backtraces are enabled, also print the query stack
+    let backtrace = env::var_os("RUST_BACKTRACE").map_or(false, |x| &x != "0");
+
+    let num_frames = if backtrace { None } else { Some(2) };
+
+    interface::try_print_query_stack(&handler, num_frames);
+
+    #[cfg(windows)]
+    unsafe {
+        if env::var("RUSTC_BREAK_ON_ICE").is_ok() {
+            // Trigger a debugger if we crashed during bootstrap
+            winapi::um::debugapi::DebugBreak();
+        }
+    }
+}
+
+/// Installs a panic hook that will print the ICE message on unexpected panics.
+///
+/// A custom rustc driver can skip calling this to set up a custom ICE hook.
+pub fn install_ice_hook() {
+    // If the user has not explicitly overridden "RUST_BACKTRACE", then produce
+    // full backtraces. When a compiler ICE happens, we want to gather
+    // as much information as possible to present in the issue opened
+    // by the user. Compiler developers and other rustc users can
+    // opt in to less-verbose backtraces by manually setting "RUST_BACKTRACE"
+    // (e.g. `RUST_BACKTRACE=1`)
+    if std::env::var("RUST_BACKTRACE").is_err() {
+        std::env::set_var("RUST_BACKTRACE", "full");
+    }
+    LazyLock::force(&DEFAULT_HOOK);
+}
+
+/// This allows tools to enable rust logging without having to magically match rustc's
+/// tracing crate version.
+pub fn init_rustc_env_logger() {
+    init_env_logger("RUSTC_LOG");
+}
+
+/// This allows tools to enable rust logging without having to magically match rustc's
+/// tracing crate version. In contrast to `init_rustc_env_logger` it allows you to choose an env var
+/// other than `RUSTC_LOG`.
+pub fn init_env_logger(env: &str) {
+    if let Err(error) = rustc_log::init_env_logger(env) {
+        early_error(ErrorOutputType::default(), &error.to_string());
+    }
+}
+
+#[cfg(all(unix, any(target_env = "gnu", target_os = "macos")))]
+mod signal_handler {
+    extern "C" {
+        fn backtrace_symbols_fd(
+            buffer: *const *mut libc::c_void,
+            size: libc::c_int,
+            fd: libc::c_int,
+        );
+    }
+
+    extern "C" fn print_stack_trace(_: libc::c_int) {
+        const MAX_FRAMES: usize = 256;
+        static mut STACK_TRACE: [*mut libc::c_void; MAX_FRAMES] =
+            [std::ptr::null_mut(); MAX_FRAMES];
+        unsafe {
+            let depth = libc::backtrace(STACK_TRACE.as_mut_ptr(), MAX_FRAMES as i32);
+            if depth == 0 {
+                return;
+            }
+            backtrace_symbols_fd(STACK_TRACE.as_ptr(), depth, 2);
+        }
+    }
+
+    /// When an error signal (such as SIGABRT or SIGSEGV) is delivered to the
+    /// process, print a stack trace and then exit.
+    pub(super) fn install() {
+        unsafe {
+            const ALT_STACK_SIZE: usize = libc::MINSIGSTKSZ + 64 * 1024;
+            let mut alt_stack: libc::stack_t = std::mem::zeroed();
+            alt_stack.ss_sp =
+                std::alloc::alloc(std::alloc::Layout::from_size_align(ALT_STACK_SIZE, 1).unwrap())
+                    as *mut libc::c_void;
+            alt_stack.ss_size = ALT_STACK_SIZE;
+            libc::sigaltstack(&alt_stack, std::ptr::null_mut());
+
+            let mut sa: libc::sigaction = std::mem::zeroed();
+            sa.sa_sigaction = print_stack_trace as libc::sighandler_t;
+            sa.sa_flags = libc::SA_NODEFER | libc::SA_RESETHAND | libc::SA_ONSTACK;
+            libc::sigemptyset(&mut sa.sa_mask);
+            libc::sigaction(libc::SIGSEGV, &sa, std::ptr::null_mut());
+        }
+    }
+}
+
+#[cfg(not(all(unix, any(target_env = "gnu", target_os = "macos"))))]
+mod signal_handler {
+    pub(super) fn install() {}
+}
+
+pub fn main() -> ! {
+    let start_time = Instant::now();
+    let start_rss = get_resident_set_size();
+    init_rustc_env_logger();
+    signal_handler::install();
+    let mut callbacks = TimePassesCallbacks::default();
+    install_ice_hook();
+    let exit_code = catch_with_exit_code(|| {
+        let args = env::args_os()
+            .enumerate()
+            .map(|(i, arg)| {
+                arg.into_string().unwrap_or_else(|arg| {
+                    early_error(
+                        ErrorOutputType::default(),
+                        &format!("argument {i} is not valid Unicode: {arg:?}"),
+                    )
+                })
+            })
+            .collect::<Vec<_>>();
+        RunCompiler::new(&args, &mut callbacks).run()
+    });
+
+    if callbacks.time_passes {
+        let end_rss = get_resident_set_size();
+        print_time_passes_entry("total", start_time.elapsed(), start_rss, end_rss);
+    }
+
+    process::exit(exit_code)
+}
diff --git a/compiler/rustc_driver/src/pretty.rs b/compiler/rustc_driver_impl/src/pretty.rs
index ae3ac8625..446c6832c 100644
--- a/compiler/rustc_driver/src/pretty.rs
+++ b/compiler/rustc_driver_impl/src/pretty.rs
@@ -403,7 +403,7 @@ pub fn print_after_parsing(sess: &Session, krate: &ast::Crate, ppm: PpMode) {
     write_or_print(&out, sess);
 }
 
-pub fn print_after_hir_lowering<'tcx>(tcx: TyCtxt<'tcx>, krate: &ast::Crate, ppm: PpMode) {
+pub fn print_after_hir_lowering<'tcx>(tcx: TyCtxt<'tcx>, ppm: PpMode) {
     if ppm.needs_analysis() {
         abort_on_err(print_with_analysis(tcx, ppm), tcx.sess);
         return;
@@ -420,7 +420,7 @@ pub fn print_after_hir_lowering<'tcx>(tcx: TyCtxt<'tcx>, krate: &ast::Crate, ppm
                 let parse = &sess.parse_sess;
                 pprust::print_crate(
                     sess.source_map(),
-                    krate,
+                    &tcx.resolver_for_lowering(()).borrow().1,
                     src_name,
                     src,
                     annotation.pp_ann(),
@@ -433,7 +433,7 @@ pub fn print_after_hir_lowering<'tcx>(tcx: TyCtxt<'tcx>, krate: &ast::Crate, ppm
 
         AstTree(PpAstTreeMode::Expanded) => {
             debug!("pretty-printing expanded AST");
-            format!("{krate:#?}")
+            format!("{:#?}", tcx.resolver_for_lowering(()).borrow().1)
         }
 
         Hir(s) => call_with_pp_support_hir(&s, tcx, move |annotation, hir_map| {
@@ -498,6 +498,21 @@ fn print_with_analysis(tcx: TyCtxt<'_>, ppm: PpMode) -> Result<(), ErrorGuarante
             out
         }
 
+        ThirFlat => {
+            let mut out = String::new();
+            abort_on_err(rustc_hir_analysis::check_crate(tcx), tcx.sess);
+            debug!("pretty printing THIR flat");
+            for did in tcx.hir().body_owners() {
+                let _ = writeln!(
+                    out,
+                    "{:?}:\n{}\n",
+                    did,
+                    tcx.thir_flat(ty::WithOptConstParam::unknown(did))
+                );
+            }
+            out
+        }
+
         _ => unreachable!(),
     };
 
diff --git a/compiler/rustc_driver_impl/src/session_diagnostics.rs b/compiler/rustc_driver_impl/src/session_diagnostics.rs
new file mode 100644
index 000000000..638b368f7
--- /dev/null
+++ b/compiler/rustc_driver_impl/src/session_diagnostics.rs
@@ -0,0 +1,67 @@
+use rustc_macros::Diagnostic;
+
+#[derive(Diagnostic)]
+#[diag(driver_impl_rlink_unable_to_read)]
+pub(crate) struct RlinkUnableToRead {
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(driver_impl_rlink_wrong_file_type)]
+pub(crate) struct RLinkWrongFileType;
+
+#[derive(Diagnostic)]
+#[diag(driver_impl_rlink_empty_version_number)]
+pub(crate) struct RLinkEmptyVersionNumber;
+
+#[derive(Diagnostic)]
+#[diag(driver_impl_rlink_encoding_version_mismatch)]
+pub(crate) struct RLinkEncodingVersionMismatch {
+    pub version_array: String,
+    pub rlink_version: u32,
+}
+
+#[derive(Diagnostic)]
+#[diag(driver_impl_rlink_rustc_version_mismatch)]
+pub(crate) struct RLinkRustcVersionMismatch<'a> {
+    pub rustc_version: String,
+    pub current_version: &'a str,
+}
+
+#[derive(Diagnostic)]
+#[diag(driver_impl_rlink_no_a_file)]
+pub(crate) struct RlinkNotAFile;
+
+#[derive(Diagnostic)]
+#[diag(driver_impl_unpretty_dump_fail)]
+pub(crate) struct UnprettyDumpFail {
+    pub path: String,
+    pub err: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(driver_impl_ice)]
+pub(crate) struct Ice;
+
+#[derive(Diagnostic)]
+#[diag(driver_impl_ice_bug_report)]
+pub(crate) struct IceBugReport<'a> {
+    pub bug_report_url: &'a str,
+}
+
+#[derive(Diagnostic)]
+#[diag(driver_impl_ice_version)]
+pub(crate) struct IceVersion<'a> {
+    pub version: &'a str,
+    pub triple: &'a str,
+}
+
+#[derive(Diagnostic)]
+#[diag(driver_impl_ice_flags)]
+pub(crate) struct IceFlags {
+    pub flags: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(driver_impl_ice_exclude_cargo_defaults)]
+pub(crate) struct IceExcludeCargoDefaults;
diff --git a/compiler/rustc_error_codes/src/error_codes.rs b/compiler/rustc_error_codes/src/error_codes.rs
index 9d5f4ad75..df857be85 100644
--- a/compiler/rustc_error_codes/src/error_codes.rs
+++ b/compiler/rustc_error_codes/src/error_codes.rs
@@ -5,7 +5,7 @@
 // /!\ IMPORTANT /!\
 //
 // Error messages' format must follow the RFC 1567 available here:
-// https://github.com/rust-lang/rfcs/pull/1567
+// https://rust-lang.github.io/rfcs/1567-long-error-codes-explanation-normalization.html
 
 register_diagnostics! {
 E0001: include_str!("./error_codes/E0001.md"),
@@ -253,6 +253,7 @@ E0466: include_str!("./error_codes/E0466.md"),
 E0468: include_str!("./error_codes/E0468.md"),
 E0469: include_str!("./error_codes/E0469.md"),
 E0472: include_str!("./error_codes/E0472.md"),
+E0476: include_str!("./error_codes/E0476.md"),
 E0477: include_str!("./error_codes/E0477.md"),
 E0478: include_str!("./error_codes/E0478.md"),
 E0482: include_str!("./error_codes/E0482.md"),
@@ -286,6 +287,7 @@ E0519: include_str!("./error_codes/E0519.md"),
 E0520: include_str!("./error_codes/E0520.md"),
 E0521: include_str!("./error_codes/E0521.md"),
 E0522: include_str!("./error_codes/E0522.md"),
+E0523: include_str!("./error_codes/E0523.md"),
 E0524: include_str!("./error_codes/E0524.md"),
 E0525: include_str!("./error_codes/E0525.md"),
 E0527: include_str!("./error_codes/E0527.md"),
@@ -506,10 +508,14 @@ E0785: include_str!("./error_codes/E0785.md"),
 E0786: include_str!("./error_codes/E0786.md"),
 E0787: include_str!("./error_codes/E0787.md"),
 E0788: include_str!("./error_codes/E0788.md"),
+E0789: include_str!("./error_codes/E0789.md"),
 E0790: include_str!("./error_codes/E0790.md"),
 E0791: include_str!("./error_codes/E0791.md"),
 E0792: include_str!("./error_codes/E0792.md"),
-;
+E0793: include_str!("./error_codes/E0793.md"),
+}
+
+// Undocumented removed error codes. Note that many removed error codes are documented.
 //  E0006, // merged with E0005
 //  E0008, // cannot bind by-move into a pattern guard
 //  E0019, // merged into E0015
@@ -566,7 +572,7 @@ E0792: include_str!("./error_codes/E0792.md"),
 //  E0246, // invalid recursive type
 //  E0247,
 //  E0248, // value used as a type, now reported earlier during resolution
-           // as E0412
+//         // as E0412
 //  E0249,
 //  E0257,
 //  E0258,
@@ -608,7 +614,6 @@ E0792: include_str!("./error_codes/E0792.md"),
 //  E0473, // dereference of reference outside its lifetime
 //  E0474, // captured variable `..` does not outlive the enclosing closure
 //  E0475, // index of slice outside its lifetime
-    E0476, // lifetime of the source pointer does not outlive lifetime bound...
 //  E0479, // the type `..` (provided as the value of a type parameter) is...
 //  E0480, // lifetime of method receiver does not outlive the method call
 //  E0481, // lifetime of function argument does not outlive the function call
@@ -620,7 +625,6 @@ E0792: include_str!("./error_codes/E0792.md"),
 //  E0488, // lifetime of variable does not enclose its declaration
 //  E0489, // type/lifetime parameter not in scope here
 //  E0490, // removed: unreachable
-    E0523, // two dependencies have same (crate-name, disambiguator) but different SVH
 //  E0526, // shuffle indices are not constant
 //  E0540, // multiple rustc_deprecated attributes
 //  E0548, // replaced with a generic attribute input check
@@ -629,14 +633,14 @@ E0792: include_str!("./error_codes/E0792.md"),
 //  E0558, // replaced with a generic attribute input check
 //  E0563, // cannot determine a type for this `impl Trait` removed in 6383de15
 //  E0564, // only named lifetimes are allowed in `impl Trait`,
-           // but `{}` was found in the type `{}`
+//         // but `{}` was found in the type `{}`
 //  E0598, // lifetime of {} is too short to guarantee its contents can be...
 //  E0611, // merged into E0616
 //  E0612, // merged into E0609
 //  E0613, // Removed (merged with E0609)
 //  E0629, // missing 'feature' (rustc_const_unstable)
 //  E0630, // rustc_const_unstable attribute must be paired with stable/unstable
-           // attribute
+//         // attribute
 //  E0645, // trait aliases not finished
 //  E0694, // an unknown tool name found in scoped attributes
 //  E0702, // replaced with a generic attribute input check
@@ -645,5 +649,3 @@ E0792: include_str!("./error_codes/E0792.md"),
 //  E0721, // `await` keyword
 //  E0723, // unstable feature in `const` context
 //  E0738, // Removed; errored on `#[track_caller] fn`s in `extern "Rust" { ... }`.
-    E0789, // rustc_allowed_through_unstable_modules without stability attribute
-}
diff --git a/compiler/rustc_error_codes/src/error_codes/E0464.md b/compiler/rustc_error_codes/src/error_codes/E0464.md
index 9108d856c..209cbb00d 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0464.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0464.md
@@ -1,6 +1,21 @@
 The compiler found multiple library files with the requested crate name.
 
+```compile_fail
+// aux-build:crateresolve-1.rs
+// aux-build:crateresolve-2.rs
+// aux-build:crateresolve-3.rs
+
+extern crate crateresolve;
+//~^ ERROR multiple candidates for `rlib` dependency `crateresolve` found
+
+fn main() {}
+```
+
 This error can occur in several different cases -- for example, when using
 `extern crate` or passing `--extern` options without crate paths. It can also be
 caused by caching issues with the build directory, in which case `cargo clean`
 may help.
+
+In the above example, there are three different library files, all of which
+define the same crate name. Without providing a full path, there is no way for
+the compiler to know which crate it should use.
diff --git a/compiler/rustc_error_codes/src/error_codes/E0476.md b/compiler/rustc_error_codes/src/error_codes/E0476.md
new file mode 100644
index 000000000..fc141ba77
--- /dev/null
+++ b/compiler/rustc_error_codes/src/error_codes/E0476.md
@@ -0,0 +1,21 @@
+The coerced type does not outlive the value being coerced to.
+
+Example of erroneous code:
+
+```compile_fail,E0476
+#![feature(coerce_unsized)]
+#![feature(unsize)]
+
+use std::marker::Unsize;
+use std::ops::CoerceUnsized;
+
+// error: lifetime of the source pointer does not outlive lifetime bound of the
+//        object type
+impl<'a, 'b, T, S> CoerceUnsized<&'a T> for &'b S where S: Unsize<T> {}
+```
+
+During a coercion, the "source pointer" (the coerced type) did not outlive the
+"object type" (value being coerced to). In the above example, `'b` is not a
+subtype of `'a`. This error can currently only be encountered with the unstable
+`CoerceUnsized` trait which allows custom coercions of unsized types behind a
+smart pointer to be implemented.
diff --git a/compiler/rustc_error_codes/src/error_codes/E0523.md b/compiler/rustc_error_codes/src/error_codes/E0523.md
new file mode 100644
index 000000000..0ddf70386
--- /dev/null
+++ b/compiler/rustc_error_codes/src/error_codes/E0523.md
@@ -0,0 +1,25 @@
+#### Note: this error code is no longer emitted by the compiler.
+
+The compiler found multiple library files with the requested crate name.
+
+```compile_fail
+// aux-build:crateresolve-1.rs
+// aux-build:crateresolve-2.rs
+// aux-build:crateresolve-3.rs
+
+extern crate crateresolve;
+//~^ ERROR multiple candidates for `rlib` dependency `crateresolve` found
+
+fn main() {}
+```
+
+This error can occur in several different cases -- for example, when using
+`extern crate` or passing `--extern` options without crate paths. It can also be
+caused by caching issues with the build directory, in which case `cargo clean`
+may help.
+
+In the above example, there are three different library files, all of which
+define the same crate name. Without providing a full path, there is no way for
+the compiler to know which crate it should use.
+
+*Note that E0523 has been merged into E0464.*
diff --git a/compiler/rustc_error_codes/src/error_codes/E0587.md b/compiler/rustc_error_codes/src/error_codes/E0587.md
index ee9031dc3..d7998af85 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0587.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0587.md
@@ -11,6 +11,6 @@ You cannot use `packed` and `align` hints on a same type. If you want to pack a
 type to a given size, you should provide a size to packed:
 
 ```
-#[repr(packed)] // ok!
+#[repr(packed(8))] // ok!
 struct Umbrella(i32);
 ```
diff --git a/compiler/rustc_error_codes/src/error_codes/E0789.md b/compiler/rustc_error_codes/src/error_codes/E0789.md
new file mode 100644
index 000000000..89b7cd422
--- /dev/null
+++ b/compiler/rustc_error_codes/src/error_codes/E0789.md
@@ -0,0 +1,30 @@
+#### This error code is internal to the compiler and will not be emitted with normal Rust code.
+
+The internal `rustc_allowed_through_unstable_modules` attribute must be used
+on an item with a `stable` attribute.
+
+Erroneous code example:
+
+```compile_fail,E0789
+// NOTE: both of these attributes are perma-unstable and should *never* be
+//       used outside of the compiler and standard library.
+#![feature(rustc_attrs)]
+#![feature(staged_api)]
+
+#![unstable(feature = "foo_module", reason = "...", issue = "123")]
+
+#[rustc_allowed_through_unstable_modules]
+// #[stable(feature = "foo", since = "1.0")]
+struct Foo;
+// ^^^ error: `rustc_allowed_through_unstable_modules` attribute must be
+//            paired with a `stable` attribute
+```
+
+Typically when an item is marked with a `stable` attribute, the modules that
+enclose the item must also be marked with `stable` attributes, otherwise the
+item becomes *de facto* unstable. `#[rustc_allowed_through_unstable_modules]`
+is a workaround which allows an item to "escape" its unstable parent modules.
+This error occurs when an item is marked with
+`#[rustc_allowed_through_unstable_modules]` but no supplementary `stable`
+attribute exists. See [#99288](https://github.com/rust-lang/rust/pull/99288)
+for an example of `#[rustc_allowed_through_unstable_modules]` in use.
diff --git a/compiler/rustc_error_codes/src/error_codes/E0793.md b/compiler/rustc_error_codes/src/error_codes/E0793.md
new file mode 100644
index 000000000..b2e51e24e
--- /dev/null
+++ b/compiler/rustc_error_codes/src/error_codes/E0793.md
@@ -0,0 +1,64 @@
+An unaligned references to a field of a [packed] struct got created.
+
+Erroneous code example:
+
+```compile_fail,E0793
+#[repr(packed)]
+pub struct Foo {
+    field1: u64,
+    field2: u8,
+}
+
+unsafe {
+    let foo = Foo { field1: 0, field2: 0 };
+    // Accessing the field directly is fine.
+    let val = foo.field1;
+    // A reference to a packed field causes a error.
+    let val = &foo.field1; // ERROR
+    // An implicit `&` is added in format strings, causing the same error.
+    println!("{}", foo.field1); // ERROR
+}
+```
+
+Creating a reference to an insufficiently aligned packed field is
+[undefined behavior] and therefore disallowed. Using an `unsafe` block does not
+change anything about this. Instead, the code should do a copy of the data in
+the packed field or use raw pointers and unaligned accesses.
+
+```
+#[repr(packed)]
+pub struct Foo {
+    field1: u64,
+    field2: u8,
+}
+
+unsafe {
+    let foo = Foo { field1: 0, field2: 0 };
+
+    // Instead of a reference, we can create a raw pointer...
+    let ptr = std::ptr::addr_of!(foo.field1);
+    // ... and then (crucially!) access it in an explicitly unaligned way.
+    let val = unsafe { ptr.read_unaligned() };
+    // This would *NOT* be correct:
+    // let val = unsafe { *ptr }; // Undefined Behavior due to unaligned load!
+
+    // For formatting, we can create a copy to avoid the direct reference.
+    let copy = foo.field1;
+    println!("{}", copy);
+    // Creating a copy can be written in a single line with curly braces.
+    // (This is equivalent to the two lines above.)
+    println!("{}", { foo.field1 });
+}
+```
+
+### Additional information
+
+Note that this error is specifically about *references* to packed fields.
+Direct by-value access of those fields is fine, since then the compiler has
+enough information to generate the correct kind of access.
+
+See [issue #82523] for more information.
+
+[packed]: https://doc.rust-lang.org/reference/type-layout.html#the-alignment-modifiers
+[undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html
+[issue #82523]: https://github.com/rust-lang/rust/issues/82523
diff --git a/compiler/rustc_error_codes/src/lib.rs b/compiler/rustc_error_codes/src/lib.rs
index bd424dd9d..d6b120e4d 100644
--- a/compiler/rustc_error_codes/src/lib.rs
+++ b/compiler/rustc_error_codes/src/lib.rs
@@ -5,10 +5,9 @@
 //! the goal being to make their maintenance easier.
 
 macro_rules! register_diagnostics {
-    ($($ecode:ident: $message:expr,)* ; $($code:ident,)*) => (
-        pub static DIAGNOSTICS: &[(&str, Option<&str>)] = &[
-            $( (stringify!($ecode), Some($message)), )*
-            $( (stringify!($code), None), )*
+    ($($ecode:ident: $message:expr,)*) => (
+        pub static DIAGNOSTICS: &[(&str, &str)] = &[
+            $( (stringify!($ecode), $message), )*
         ];
     )
 }
diff --git a/compiler/rustc_error_messages/Cargo.toml b/compiler/rustc_error_messages/Cargo.toml
index 0c705d2ec..27783d60b 100644
--- a/compiler/rustc_error_messages/Cargo.toml
+++ b/compiler/rustc_error_messages/Cargo.toml
@@ -16,10 +16,9 @@ rustc_span = { path = "../rustc_span" }
 rustc_macros = { path = "../rustc_macros" }
 tracing = "0.1"
 unic-langid = { version = "0.9.0", features = ["macros"] }
-icu_list = "1.0.0"
-writeable = "0.5.0"
-icu_locid = "1.0.0"
-icu_provider_adapters = "1.0.0"
+icu_list = "1.1.0"
+icu_locid = "1.1.0"
+icu_provider_adapters = "1.1.0"
 
 [features]
 rustc_use_parallel_compiler = ['rustc_baked_icu_data/rustc_use_parallel_compiler']
diff --git a/compiler/rustc_error_messages/locales/en-US.ftl b/compiler/rustc_error_messages/locales/en-US.ftl
new file mode 100644
index 000000000..e62923744
--- /dev/null
+++ b/compiler/rustc_error_messages/locales/en-US.ftl
@@ -0,0 +1 @@
+# satisfy tidy lint by having a line in this file
diff --git a/compiler/rustc_error_messages/locales/en-US/ast_passes.ftl b/compiler/rustc_error_messages/locales/en-US/ast_passes.ftl
deleted file mode 100644
index 5f28839f1..000000000
--- a/compiler/rustc_error_messages/locales/en-US/ast_passes.ftl
+++ /dev/null
@@ -1,92 +0,0 @@
-ast_passes_forbidden_let =
-    `let` expressions are not supported here
-    .note = only supported directly in conditions of `if` and `while` expressions
-    .not_supported_or = `||` operators are not supported in let chain expressions
-    .not_supported_parentheses = `let`s wrapped in parentheses are not supported in a context with let chains
-
-ast_passes_forbidden_let_stable =
-    expected expression, found statement (`let`)
-    .note = variable declaration using `let` is a statement
-
-ast_passes_deprecated_where_clause_location =
-    where clause not allowed here
-
-ast_passes_forbidden_assoc_constraint =
-    associated type bounds are not allowed within structs, enums, or unions
-
-ast_passes_keyword_lifetime =
-    lifetimes cannot use keyword names
-
-ast_passes_invalid_label =
-    invalid label name `{$name}`
-
-ast_passes_invalid_visibility =
-    unnecessary visibility qualifier
-    .implied = `pub` not permitted here because it's implied
-    .individual_impl_items = place qualifiers on individual impl items instead
-    .individual_foreign_items = place qualifiers on individual foreign items instead
-
-ast_passes_trait_fn_const =
-    functions in traits cannot be declared const
-    .label = functions in traits cannot be const
-
-ast_passes_forbidden_lifetime_bound =
-    lifetime bounds cannot be used in this context
-
-ast_passes_forbidden_non_lifetime_param =
-    only lifetime parameters can be used in this context
-
-ast_passes_fn_param_too_many =
-    function can not have more than {$max_num_args} arguments
-
-ast_passes_fn_param_c_var_args_only =
-    C-variadic function must be declared with at least one named argument
-
-ast_passes_fn_param_c_var_args_not_last =
-    `...` must be the last argument of a C-variadic function
-
-ast_passes_fn_param_doc_comment =
-    documentation comments cannot be applied to function parameters
-    .label = doc comments are not allowed here
-
-ast_passes_fn_param_forbidden_attr =
-    allow, cfg, cfg_attr, deny, expect, forbid, and warn are the only allowed built-in attributes in function parameters
-
-ast_passes_fn_param_forbidden_self =
-    `self` parameter is only allowed in associated functions
-    .label = not semantically valid as function parameter
-    .note = associated functions are those in `impl` or `trait` definitions
-
-ast_passes_forbidden_default =
-    `default` is only allowed on items in trait impls
-    .label = `default` because of this
-
-ast_passes_assoc_const_without_body =
-    associated constant in `impl` without body
-    .suggestion = provide a definition for the constant
-
-ast_passes_assoc_fn_without_body =
-    associated function in `impl` without body
-    .suggestion = provide a definition for the function
-
-ast_passes_assoc_type_without_body =
-    associated type in `impl` without body
-    .suggestion = provide a definition for the type
-
-ast_passes_const_without_body =
-    free constant item without body
-    .suggestion = provide a definition for the constant
-
-ast_passes_static_without_body =
-    free static item without body
-    .suggestion = provide a definition for the static
-
-ast_passes_ty_alias_without_body =
-    free type alias without body
-    .suggestion = provide a definition for the type
-
-ast_passes_fn_without_body =
-    free function without a body
-    .suggestion = provide a definition for the function
-
-ast_passes_extern_block_suggestion = if you meant to declare an externally defined function, use an `extern` block
diff --git a/compiler/rustc_error_messages/locales/en-US/codegen_llvm.ftl b/compiler/rustc_error_messages/locales/en-US/codegen_llvm.ftl
deleted file mode 100644
index 860212b05..000000000
--- a/compiler/rustc_error_messages/locales/en-US/codegen_llvm.ftl
+++ /dev/null
@@ -1,46 +0,0 @@
-codegen_llvm_unknown_ctarget_feature =
-    unknown feature specified for `-Ctarget-feature`: `{$feature}`
-    .note = it is still passed through to the codegen backend
-    .possible_feature = you might have meant: `{$rust_feature}`
-    .consider_filing_feature_request = consider filing a feature request
-
-codegen_llvm_unknown_ctarget_feature_prefix =
-    unknown feature specified for `-Ctarget-feature`: `{$feature}`
-    .note = features must begin with a `+` to enable or `-` to disable it
-
-codegen_llvm_error_creating_import_library =
-    Error creating import library for {$lib_name}: {$error}
-
-codegen_llvm_instrument_coverage_requires_llvm_12 =
-    rustc option `-C instrument-coverage` requires LLVM 12 or higher.
-
-codegen_llvm_symbol_already_defined =
-    symbol `{$symbol_name}` is already defined
-
-codegen_llvm_invalid_minimum_alignment =
-    invalid minimum global alignment: {$err}
-
-codegen_llvm_sanitizer_memtag_requires_mte =
-    `-Zsanitizer=memtag` requires `-Ctarget-feature=+mte`
-
-codegen_llvm_error_writing_def_file =
-    Error writing .DEF file: {$error}
-
-codegen_llvm_error_calling_dlltool =
-    Error calling dlltool: {$error}
-
-codegen_llvm_dlltool_fail_import_library =
-    Dlltool could not create import library: {$stdout}\n{$stderr}
-
-codegen_llvm_target_feature_disable_or_enable =
-    the target features {$features} must all be either enabled or disabled together
-
-codegen_llvm_missing_features =
-    add the missing features in a `target_feature` attribute
-
-codegen_llvm_dynamic_linking_with_lto =
-    cannot prefer dynamic linking when performing LTO
-    .note = only 'staticlib', 'bin', and 'cdylib' outputs are supported with LTO
-
-codegen_llvm_fail_parsing_target_machine_config_to_target_machine =
-    failed to parse target machine config to target machine: {$error}
diff --git a/compiler/rustc_error_messages/locales/en-US/compiletest.ftl b/compiler/rustc_error_messages/locales/en-US/compiletest.ftl
deleted file mode 100644
index 55061fbce..000000000
--- a/compiler/rustc_error_messages/locales/en-US/compiletest.ftl
+++ /dev/null
@@ -1,5 +0,0 @@
-compiletest_example = this is an example message used in testing
-    .note = with a note
-    .help = with a help
-    .suggestion = with a suggestion
-    .label = with a label
diff --git a/compiler/rustc_error_messages/locales/en-US/driver.ftl b/compiler/rustc_error_messages/locales/en-US/driver.ftl
deleted file mode 100644
index 8ad198c86..000000000
--- a/compiler/rustc_error_messages/locales/en-US/driver.ftl
+++ /dev/null
@@ -1,13 +0,0 @@
-driver_rlink_unable_to_read = failed to read rlink file: `{$err}`
-
-driver_rlink_wrong_file_type = The input does not look like a .rlink file
-
-driver_rlink_empty_version_number = The input does not contain version number
-
-driver_rlink_encoding_version_mismatch = .rlink file was produced with encoding version `{$version_array}`, but the current version is `{$rlink_version}`
-
-driver_rlink_rustc_version_mismatch = .rlink file was produced by rustc version `{$rustc_version}`, but the current version is `{$current_version}`
-
-driver_rlink_no_a_file = rlink must be a file
-
-driver_unpretty_dump_fail = pretty-print failed to write `{$path}` due to error `{$err}`
diff --git a/compiler/rustc_error_messages/locales/en-US/errors.ftl b/compiler/rustc_error_messages/locales/en-US/errors.ftl
deleted file mode 100644
index 429bdd277..000000000
--- a/compiler/rustc_error_messages/locales/en-US/errors.ftl
+++ /dev/null
@@ -1,13 +0,0 @@
-errors_target_invalid_address_space = invalid address space `{$addr_space}` for `{$cause}` in "data-layout": {$err}
-
-errors_target_invalid_bits = invalid {$kind} `{$bit}` for `{$cause}` in "data-layout": {$err}
-
-errors_target_missing_alignment = missing alignment for `{$cause}` in "data-layout"
-
-errors_target_invalid_alignment = invalid alignment for `{$cause}` in "data-layout": {$err}
-
-errors_target_inconsistent_architecture = inconsistent target specification: "data-layout" claims architecture is {$dl}-endian, while "target-endian" is `{$target}`
-
-errors_target_inconsistent_pointer_width = inconsistent target specification: "data-layout" claims pointers are {$pointer_size}-bit, while "target-pointer-width" is `{$target}`
-
-errors_target_invalid_bits_size = {$err}
diff --git a/compiler/rustc_error_messages/locales/en-US/save_analysis.ftl b/compiler/rustc_error_messages/locales/en-US/save_analysis.ftl
deleted file mode 100644
index 36c2ff468..000000000
--- a/compiler/rustc_error_messages/locales/en-US/save_analysis.ftl
+++ /dev/null
@@ -1 +0,0 @@
-save_analysis_could_not_open = Could not open `{$file_name}`: `{$err}`
diff --git a/compiler/rustc_error_messages/src/lib.rs b/compiler/rustc_error_messages/src/lib.rs
index 37a51980a..010e5f060 100644
--- a/compiler/rustc_error_messages/src/lib.rs
+++ b/compiler/rustc_error_messages/src/lib.rs
@@ -34,47 +34,7 @@ use intl_memoizer::IntlLangMemoizer;
 pub use fluent_bundle::{self, types::FluentType, FluentArgs, FluentError, FluentValue};
 pub use unic_langid::{langid, LanguageIdentifier};
 
-// Generates `DEFAULT_LOCALE_RESOURCES` static and `fluent_generated` module.
-fluent_messages! {
-    // tidy-alphabetical-start
-    ast_lowering => "../locales/en-US/ast_lowering.ftl",
-    ast_passes => "../locales/en-US/ast_passes.ftl",
-    attr => "../locales/en-US/attr.ftl",
-    borrowck => "../locales/en-US/borrowck.ftl",
-    builtin_macros => "../locales/en-US/builtin_macros.ftl",
-    codegen_gcc => "../locales/en-US/codegen_gcc.ftl",
-    codegen_llvm => "../locales/en-US/codegen_llvm.ftl",
-    codegen_ssa => "../locales/en-US/codegen_ssa.ftl",
-    compiletest => "../locales/en-US/compiletest.ftl",
-    const_eval => "../locales/en-US/const_eval.ftl",
-    driver => "../locales/en-US/driver.ftl",
-    errors => "../locales/en-US/errors.ftl",
-    expand => "../locales/en-US/expand.ftl",
-    hir_analysis => "../locales/en-US/hir_analysis.ftl",
-    hir_typeck => "../locales/en-US/hir_typeck.ftl",
-    infer => "../locales/en-US/infer.ftl",
-    interface => "../locales/en-US/interface.ftl",
-    lint => "../locales/en-US/lint.ftl",
-    metadata => "../locales/en-US/metadata.ftl",
-    middle => "../locales/en-US/middle.ftl",
-    mir_build => "../locales/en-US/mir_build.ftl",
-    mir_dataflow => "../locales/en-US/mir_dataflow.ftl",
-    monomorphize => "../locales/en-US/monomorphize.ftl",
-    parse => "../locales/en-US/parse.ftl",
-    passes => "../locales/en-US/passes.ftl",
-    plugin_impl => "../locales/en-US/plugin_impl.ftl",
-    privacy => "../locales/en-US/privacy.ftl",
-    query_system => "../locales/en-US/query_system.ftl",
-    resolve => "../locales/en-US/resolve.ftl",
-    save_analysis => "../locales/en-US/save_analysis.ftl",
-    session => "../locales/en-US/session.ftl",
-    symbol_mangling => "../locales/en-US/symbol_mangling.ftl",
-    trait_selection => "../locales/en-US/trait_selection.ftl",
-    ty_utils => "../locales/en-US/ty_utils.ftl",
-    // tidy-alphabetical-end
-}
-
-pub use fluent_generated::{self as fluent, DEFAULT_LOCALE_RESOURCES};
+fluent_messages! { "../locales/en-US.ftl" }
 
 pub type FluentBundle = fluent_bundle::bundle::FluentBundle<FluentResource, IntlLangMemoizer>;
 
@@ -175,13 +135,19 @@ pub fn fluent_bundle(
 
     let fallback_locale = langid!("en-US");
     let requested_fallback_locale = requested_locale.as_ref() == Some(&fallback_locale);
-
+    trace!(?requested_fallback_locale);
+    if requested_fallback_locale && additional_ftl_path.is_none() {
+        return Ok(None);
+    }
     // If there is only `-Z additional-ftl-path`, assume locale is "en-US", otherwise use user
     // provided locale.
     let locale = requested_locale.clone().unwrap_or(fallback_locale);
     trace!(?locale);
     let mut bundle = new_bundle(vec![locale]);
 
+    // Add convenience functions available to ftl authors.
+    register_functions(&mut bundle);
+
     // Fluent diagnostics can insert directionality isolation markers around interpolated variables
     // indicating that there may be a shift from right-to-left to left-to-right text (or
     // vice-versa). These are disabled because they are sometimes visible in the error output, but
@@ -190,7 +156,7 @@ pub fn fluent_bundle(
     bundle.set_use_isolating(with_directionality_markers);
 
     // If the user requests the default locale then don't try to load anything.
-    if !requested_fallback_locale && let Some(requested_locale) = requested_locale {
+    if let Some(requested_locale) = requested_locale {
         let mut found_resources = false;
         for sysroot in user_provided_sysroot.iter_mut().chain(sysroot_candidates.iter_mut()) {
             sysroot.push("share");
@@ -244,6 +210,15 @@ pub fn fluent_bundle(
     Ok(Some(bundle))
 }
 
+fn register_functions(bundle: &mut FluentBundle) {
+    bundle
+        .add_function("STREQ", |positional, _named| match positional {
+            [FluentValue::String(a), FluentValue::String(b)] => format!("{}", (a == b)).into(),
+            _ => FluentValue::Error,
+        })
+        .expect("Failed to add a function to the bundle.");
+}
+
 /// Type alias for the result of `fallback_fluent_bundle` - a reference-counted pointer to a lazily
 /// evaluated fluent bundle.
 pub type LazyFallbackBundle = Lrc<Lazy<FluentBundle, impl FnOnce() -> FluentBundle>>;
@@ -251,11 +226,14 @@ pub type LazyFallbackBundle = Lrc<Lazy<FluentBundle, impl FnOnce() -> FluentBund
 /// Return the default `FluentBundle` with standard "en-US" diagnostic messages.
 #[instrument(level = "trace")]
 pub fn fallback_fluent_bundle(
-    resources: &'static [&'static str],
+    resources: Vec<&'static str>,
     with_directionality_markers: bool,
 ) -> LazyFallbackBundle {
     Lrc::new(Lazy::new(move || {
         let mut fallback_bundle = new_bundle(vec![langid!("en-US")]);
+
+        register_functions(&mut fallback_bundle);
+
         // See comment in `fluent_bundle`.
         fallback_bundle.set_use_isolating(with_directionality_markers);
 
diff --git a/compiler/rustc_errors/locales/en-US.ftl b/compiler/rustc_errors/locales/en-US.ftl
new file mode 100644
index 000000000..dde1d6c0a
--- /dev/null
+++ b/compiler/rustc_errors/locales/en-US.ftl
@@ -0,0 +1,19 @@
+errors_target_invalid_address_space =
+    invalid address space `{$addr_space}` for `{$cause}` in "data-layout": {$err}
+
+errors_target_invalid_bits =
+    invalid {$kind} `{$bit}` for `{$cause}` in "data-layout": {$err}
+
+errors_target_missing_alignment =
+    missing alignment for `{$cause}` in "data-layout"
+
+errors_target_invalid_alignment =
+    invalid alignment for `{$cause}` in "data-layout": {$err}
+
+errors_target_inconsistent_architecture =
+    inconsistent target specification: "data-layout" claims architecture is {$dl}-endian, while "target-endian" is `{$target}`
+
+errors_target_inconsistent_pointer_width =
+    inconsistent target specification: "data-layout" claims pointers are {$pointer_size}-bit, while "target-pointer-width" is `{$target}`
+
+errors_target_invalid_bits_size = {$err}
diff --git a/compiler/rustc_errors/src/diagnostic.rs b/compiler/rustc_errors/src/diagnostic.rs
index 51b2ff6a0..9ed8ab674 100644
--- a/compiler/rustc_errors/src/diagnostic.rs
+++ b/compiler/rustc_errors/src/diagnostic.rs
@@ -7,7 +7,6 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_error_messages::fluent_value_from_str_list_sep_by_and;
 use rustc_error_messages::FluentValue;
 use rustc_lint_defs::{Applicability, LintExpectationId};
-use rustc_span::edition::LATEST_STABLE_EDITION;
 use rustc_span::symbol::Symbol;
 use rustc_span::{Span, DUMMY_SP};
 use std::borrow::Cow;
@@ -555,18 +554,6 @@ impl Diagnostic {
         self
     }
 
-    /// Help the user upgrade to the latest edition.
-    /// This is factored out to make sure it does the right thing with `Cargo.toml`.
-    pub fn help_use_latest_edition(&mut self) -> &mut Self {
-        if std::env::var_os("CARGO").is_some() {
-            self.help(&format!("set `edition = \"{}\"` in `Cargo.toml`", LATEST_STABLE_EDITION));
-        } else {
-            self.help(&format!("pass `--edition {}` to `rustc`", LATEST_STABLE_EDITION));
-        }
-        self.note("for more on editions, read https://doc.rust-lang.org/edition-guide");
-        self
-    }
-
     /// Disallow attaching suggestions this diagnostic.
     /// Any suggestions attached e.g. with the `span_suggestion_*` methods
     /// (before and after the call to `disable_suggestions`) will be ignored.
@@ -629,19 +616,27 @@ impl Diagnostic {
         applicability: Applicability,
         style: SuggestionStyle,
     ) -> &mut Self {
-        assert!(!suggestion.is_empty());
-        debug_assert!(
-            !(suggestion.iter().any(|(sp, text)| sp.is_empty() && text.is_empty())),
-            "Span must not be empty and have no suggestion"
+        let mut parts = suggestion
+            .into_iter()
+            .map(|(span, snippet)| SubstitutionPart { snippet, span })
+            .collect::<Vec<_>>();
+
+        parts.sort_unstable_by_key(|part| part.span);
+
+        assert!(!parts.is_empty());
+        debug_assert_eq!(
+            parts.iter().find(|part| part.span.is_empty() && part.snippet.is_empty()),
+            None,
+            "Span must not be empty and have no suggestion",
+        );
+        debug_assert_eq!(
+            parts.array_windows().find(|[a, b]| a.span.overlaps(b.span)),
+            None,
+            "suggestion must not have overlapping parts",
         );
 
         self.push_suggestion(CodeSuggestion {
-            substitutions: vec![Substitution {
-                parts: suggestion
-                    .into_iter()
-                    .map(|(span, snippet)| SubstitutionPart { snippet, span })
-                    .collect(),
-            }],
+            substitutions: vec![Substitution { parts }],
             msg: self.subdiagnostic_message_to_diagnostic_message(msg),
             style,
             applicability,
@@ -802,25 +797,34 @@ impl Diagnostic {
         suggestions: impl IntoIterator<Item = Vec<(Span, String)>>,
         applicability: Applicability,
     ) -> &mut Self {
-        let suggestions: Vec<_> = suggestions.into_iter().collect();
-        debug_assert!(
-            !(suggestions
-                .iter()
-                .flatten()
-                .any(|(sp, suggestion)| sp.is_empty() && suggestion.is_empty())),
-            "Span must not be empty and have no suggestion"
-        );
+        let substitutions = suggestions
+            .into_iter()
+            .map(|sugg| {
+                let mut parts = sugg
+                    .into_iter()
+                    .map(|(span, snippet)| SubstitutionPart { snippet, span })
+                    .collect::<Vec<_>>();
+
+                parts.sort_unstable_by_key(|part| part.span);
+
+                assert!(!parts.is_empty());
+                debug_assert_eq!(
+                    parts.iter().find(|part| part.span.is_empty() && part.snippet.is_empty()),
+                    None,
+                    "Span must not be empty and have no suggestion",
+                );
+                debug_assert_eq!(
+                    parts.array_windows().find(|[a, b]| a.span.overlaps(b.span)),
+                    None,
+                    "suggestion must not have overlapping parts",
+                );
+
+                Substitution { parts }
+            })
+            .collect();
 
         self.push_suggestion(CodeSuggestion {
-            substitutions: suggestions
-                .into_iter()
-                .map(|sugg| Substitution {
-                    parts: sugg
-                        .into_iter()
-                        .map(|(span, snippet)| SubstitutionPart { snippet, span })
-                        .collect(),
-                })
-                .collect(),
+            substitutions,
             msg: self.subdiagnostic_message_to_diagnostic_message(msg),
             style: SuggestionStyle::ShowCode,
             applicability,
@@ -1034,6 +1038,7 @@ impl Diagnostic {
     ) -> (
         &Level,
         &[(DiagnosticMessage, Style)],
+        Vec<(&Cow<'static, str>, &DiagnosticArgValue<'static>)>,
         &Option<DiagnosticId>,
         &MultiSpan,
         &Result<Vec<CodeSuggestion>, SuggestionsDisabled>,
@@ -1042,6 +1047,7 @@ impl Diagnostic {
         (
             &self.level,
             &self.message,
+            self.args().collect(),
             &self.code,
             &self.span,
             &self.suggestions,
diff --git a/compiler/rustc_errors/src/diagnostic_builder.rs b/compiler/rustc_errors/src/diagnostic_builder.rs
index cbfee582d..3064d2bed 100644
--- a/compiler/rustc_errors/src/diagnostic_builder.rs
+++ b/compiler/rustc_errors/src/diagnostic_builder.rs
@@ -408,6 +408,59 @@ impl EmissionGuarantee for ! {
     }
 }
 
+impl<'a> DiagnosticBuilder<'a, rustc_span::fatal_error::FatalError> {
+    /// Convenience function for internal use, clients should use one of the
+    /// `struct_*` methods on [`Handler`].
+    #[track_caller]
+    pub(crate) fn new_almost_fatal(
+        handler: &'a Handler,
+        message: impl Into<DiagnosticMessage>,
+    ) -> Self {
+        let diagnostic = Diagnostic::new_with_code(Level::Fatal, None, message);
+        Self::new_diagnostic_almost_fatal(handler, diagnostic)
+    }
+
+    /// Creates a new `DiagnosticBuilder` with an already constructed
+    /// diagnostic.
+    pub(crate) fn new_diagnostic_almost_fatal(
+        handler: &'a Handler,
+        diagnostic: Diagnostic,
+    ) -> Self {
+        debug!("Created new diagnostic");
+        Self {
+            inner: DiagnosticBuilderInner {
+                state: DiagnosticBuilderState::Emittable(handler),
+                diagnostic: Box::new(diagnostic),
+            },
+            _marker: PhantomData,
+        }
+    }
+}
+
+impl EmissionGuarantee for rustc_span::fatal_error::FatalError {
+    fn diagnostic_builder_emit_producing_guarantee(db: &mut DiagnosticBuilder<'_, Self>) -> Self {
+        match db.inner.state {
+            // First `.emit()` call, the `&Handler` is still available.
+            DiagnosticBuilderState::Emittable(handler) => {
+                db.inner.state = DiagnosticBuilderState::AlreadyEmittedOrDuringCancellation;
+
+                handler.emit_diagnostic(&mut db.inner.diagnostic);
+            }
+            // `.emit()` was previously called, disallowed from repeating it.
+            DiagnosticBuilderState::AlreadyEmittedOrDuringCancellation => {}
+        }
+        // Then fatally error..
+        rustc_span::fatal_error::FatalError
+    }
+
+    fn make_diagnostic_builder(
+        handler: &Handler,
+        msg: impl Into<DiagnosticMessage>,
+    ) -> DiagnosticBuilder<'_, Self> {
+        DiagnosticBuilder::new_almost_fatal(handler, msg)
+    }
+}
+
 /// In general, the `DiagnosticBuilder` uses deref to allow access to
 /// the fields and methods of the embedded `diagnostic` in a
 /// transparent way. *However,* many of the methods are intended to
@@ -616,7 +669,6 @@ impl<'a, G: EmissionGuarantee> DiagnosticBuilder<'a, G> {
         sp: impl Into<MultiSpan>,
         msg: impl Into<SubdiagnosticMessage>,
     ) -> &mut Self);
-    forward!(pub fn help_use_latest_edition(&mut self,) -> &mut Self);
     forward!(pub fn set_is_lint(&mut self,) -> &mut Self);
 
     forward!(pub fn disable_suggestions(&mut self,) -> &mut Self);
diff --git a/compiler/rustc_errors/src/diagnostic_impls.rs b/compiler/rustc_errors/src/diagnostic_impls.rs
index dad5e98aa..e82bad67b 100644
--- a/compiler/rustc_errors/src/diagnostic_impls.rs
+++ b/compiler/rustc_errors/src/diagnostic_impls.rs
@@ -1,6 +1,5 @@
-use crate::{
-    fluent, DiagnosticArgValue, DiagnosticBuilder, Handler, IntoDiagnostic, IntoDiagnosticArg,
-};
+use crate::fluent_generated as fluent;
+use crate::{DiagnosticArgValue, DiagnosticBuilder, Handler, IntoDiagnostic, IntoDiagnosticArg};
 use rustc_ast as ast;
 use rustc_ast_pretty::pprust;
 use rustc_hir as hir;
@@ -36,6 +35,12 @@ impl<'a, T: fmt::Display> From<&'a T> for DiagnosticArgFromDisplay<'a> {
     }
 }
 
+impl<'a, T: Clone + IntoDiagnosticArg> IntoDiagnosticArg for &'a T {
+    fn into_diagnostic_arg(self) -> DiagnosticArgValue<'static> {
+        self.clone().into_diagnostic_arg()
+    }
+}
+
 macro_rules! into_diagnostic_arg_using_display {
     ($( $ty:ty ),+ $(,)?) => {
         $(
@@ -49,6 +54,7 @@ macro_rules! into_diagnostic_arg_using_display {
 }
 
 into_diagnostic_arg_using_display!(
+    ast::ParamKindOrd,
     i8,
     u8,
     i16,
@@ -153,12 +159,6 @@ impl IntoDiagnosticArg for ast::Path {
     }
 }
 
-impl IntoDiagnosticArg for &ast::Path {
-    fn into_diagnostic_arg(self) -> DiagnosticArgValue<'static> {
-        DiagnosticArgValue::Str(Cow::Owned(pprust::path_to_string(self)))
-    }
-}
-
 impl IntoDiagnosticArg for ast::token::Token {
     fn into_diagnostic_arg(self) -> DiagnosticArgValue<'static> {
         DiagnosticArgValue::Str(pprust::token_to_string(&self))
@@ -177,6 +177,26 @@ impl IntoDiagnosticArg for type_ir::FloatTy {
     }
 }
 
+impl IntoDiagnosticArg for std::ffi::CString {
+    fn into_diagnostic_arg(self) -> DiagnosticArgValue<'static> {
+        DiagnosticArgValue::Str(Cow::Owned(self.to_string_lossy().into_owned()))
+    }
+}
+
+impl IntoDiagnosticArg for rustc_data_structures::small_c_str::SmallCStr {
+    fn into_diagnostic_arg(self) -> DiagnosticArgValue<'static> {
+        DiagnosticArgValue::Str(Cow::Owned(self.to_string_lossy().into_owned()))
+    }
+}
+
+impl IntoDiagnosticArg for ast::Visibility {
+    fn into_diagnostic_arg(self) -> DiagnosticArgValue<'static> {
+        let s = pprust::vis_to_string(&self);
+        let s = s.trim_end().to_string();
+        DiagnosticArgValue::Str(Cow::Owned(s))
+    }
+}
+
 impl IntoDiagnosticArg for Level {
     fn into_diagnostic_arg(self) -> DiagnosticArgValue<'static> {
         DiagnosticArgValue::Str(Cow::Borrowed(self.to_cmd_flag()))
diff --git a/compiler/rustc_errors/src/emitter.rs b/compiler/rustc_errors/src/emitter.rs
index 628e19999..211bbf4f5 100644
--- a/compiler/rustc_errors/src/emitter.rs
+++ b/compiler/rustc_errors/src/emitter.rs
@@ -18,7 +18,7 @@ use crate::translation::{to_fluent_args, Translate};
 use crate::{
     diagnostic::DiagnosticLocation, CodeSuggestion, Diagnostic, DiagnosticId, DiagnosticMessage,
     FluentBundle, Handler, LazyFallbackBundle, Level, MultiSpan, SubDiagnostic,
-    SubstitutionHighlight, SuggestionStyle,
+    SubstitutionHighlight, SuggestionStyle, TerminalUrl,
 };
 use rustc_lint_defs::pluralize;
 
@@ -66,6 +66,7 @@ impl HumanReadableErrorType {
         diagnostic_width: Option<usize>,
         macro_backtrace: bool,
         track_diagnostics: bool,
+        terminal_url: TerminalUrl,
     ) -> EmitterWriter {
         let (short, color_config) = self.unzip();
         let color = color_config.suggests_using_colors();
@@ -80,6 +81,7 @@ impl HumanReadableErrorType {
             diagnostic_width,
             macro_backtrace,
             track_diagnostics,
+            terminal_url,
         )
     }
 }
@@ -652,6 +654,7 @@ pub struct EmitterWriter {
 
     macro_backtrace: bool,
     track_diagnostics: bool,
+    terminal_url: TerminalUrl,
 }
 
 #[derive(Debug)]
@@ -672,6 +675,7 @@ impl EmitterWriter {
         diagnostic_width: Option<usize>,
         macro_backtrace: bool,
         track_diagnostics: bool,
+        terminal_url: TerminalUrl,
     ) -> EmitterWriter {
         let dst = Destination::from_stderr(color_config);
         EmitterWriter {
@@ -685,6 +689,7 @@ impl EmitterWriter {
             diagnostic_width,
             macro_backtrace,
             track_diagnostics,
+            terminal_url,
         }
     }
 
@@ -699,6 +704,7 @@ impl EmitterWriter {
         diagnostic_width: Option<usize>,
         macro_backtrace: bool,
         track_diagnostics: bool,
+        terminal_url: TerminalUrl,
     ) -> EmitterWriter {
         EmitterWriter {
             dst: Raw(dst, colored),
@@ -711,6 +717,7 @@ impl EmitterWriter {
             diagnostic_width,
             macro_backtrace,
             track_diagnostics,
+            terminal_url,
         }
     }
 
@@ -1378,7 +1385,13 @@ impl EmitterWriter {
             // only render error codes, not lint codes
             if let Some(DiagnosticId::Error(ref code)) = *code {
                 buffer.append(0, "[", Style::Level(*level));
-                buffer.append(0, code, Style::Level(*level));
+                let code = if let TerminalUrl::Yes = self.terminal_url {
+                    let path = "https://doc.rust-lang.org/error_codes";
+                    format!("\x1b]8;;{path}/{code}.html\x07{code}\x1b]8;;\x07")
+                } else {
+                    code.clone()
+                };
+                buffer.append(0, &code, Style::Level(*level));
                 buffer.append(0, "]", Style::Level(*level));
                 label_width += 2 + code.len();
             }
@@ -1755,7 +1768,7 @@ impl EmitterWriter {
 
         // Render the replacements for each suggestion
         let suggestions = suggestion.splice_lines(sm);
-        debug!("emit_suggestion_default: suggestions={:?}", suggestions);
+        debug!(?suggestions);
 
         if suggestions.is_empty() {
             // Suggestions coming from macros can have malformed spans. This is a heavy handed
@@ -1784,6 +1797,7 @@ impl EmitterWriter {
         for (complete, parts, highlights, only_capitalization) in
             suggestions.iter().take(MAX_SUGGESTIONS)
         {
+            debug!(?complete, ?parts, ?highlights);
             notice_capitalization |= only_capitalization;
 
             let has_deletion = parts.iter().any(|p| p.is_deletion(sm));
@@ -1796,17 +1810,17 @@ impl EmitterWriter {
                 // telling users to make a change but not clarifying *where*.
                 let loc = sm.lookup_char_pos(parts[0].span.lo());
                 if loc.file.name != sm.span_to_filename(span) && loc.file.name.is_real() {
-                    buffer.puts(row_num - 1, 0, "--> ", Style::LineNumber);
-                    buffer.append(
-                        row_num - 1,
-                        &format!(
-                            "{}:{}:{}",
-                            sm.filename_for_diagnostics(&loc.file.name),
-                            sm.doctest_offset_line(&loc.file.name, loc.line),
-                            loc.col.0 + 1,
-                        ),
-                        Style::LineAndColumn,
-                    );
+                    let arrow = "--> ";
+                    buffer.puts(row_num - 1, 0, arrow, Style::LineNumber);
+                    let filename = sm.filename_for_diagnostics(&loc.file.name);
+                    let offset = sm.doctest_offset_line(&loc.file.name, loc.line);
+                    let message = format!("{}:{}:{}", filename, offset, loc.col.0 + 1);
+                    if row_num == 2 {
+                        let col = usize::max(max_line_num_len + 1, arrow.len());
+                        buffer.puts(1, col, &message, Style::LineAndColumn);
+                    } else {
+                        buffer.append(row_num - 1, &message, Style::LineAndColumn);
+                    }
                     for _ in 0..max_line_num_len {
                         buffer.prepend(row_num - 1, " ", Style::NoStyle);
                     }
@@ -1882,9 +1896,8 @@ impl EmitterWriter {
                             &mut buffer,
                             &mut row_num,
                             &Vec::new(),
-                            p,
+                            p + line_start,
                             l,
-                            line_start,
                             show_code_change,
                             max_line_num_len,
                             &file_lines,
@@ -1907,9 +1920,8 @@ impl EmitterWriter {
                                 &mut buffer,
                                 &mut row_num,
                                 &Vec::new(),
-                                p,
+                                p + line_start,
                                 l,
-                                line_start,
                                 show_code_change,
                                 max_line_num_len,
                                 &file_lines,
@@ -1925,9 +1937,8 @@ impl EmitterWriter {
                                 &mut buffer,
                                 &mut row_num,
                                 &Vec::new(),
-                                p,
+                                p + line_start,
                                 l,
-                                line_start,
                                 show_code_change,
                                 max_line_num_len,
                                 &file_lines,
@@ -1941,9 +1952,8 @@ impl EmitterWriter {
                     &mut buffer,
                     &mut row_num,
                     highlight_parts,
-                    line_pos,
+                    line_pos + line_start,
                     line,
-                    line_start,
                     show_code_change,
                     max_line_num_len,
                     &file_lines,
@@ -2113,30 +2123,38 @@ impl EmitterWriter {
                         }
                     }
                     for sugg in suggestions {
-                        if sugg.style == SuggestionStyle::CompletelyHidden {
-                            // do not display this suggestion, it is meant only for tools
-                        } else if sugg.style == SuggestionStyle::HideCodeAlways {
-                            if let Err(e) = self.emit_message_default(
-                                &MultiSpan::new(),
-                                &[(sugg.msg.to_owned(), Style::HeaderMsg)],
-                                args,
-                                &None,
-                                &Level::Help,
-                                max_line_num_len,
-                                true,
-                                None,
-                            ) {
-                                panic!("failed to emit error: {}", e);
+                        match sugg.style {
+                            SuggestionStyle::CompletelyHidden => {
+                                // do not display this suggestion, it is meant only for tools
                             }
-                        } else if let Err(e) = self.emit_suggestion_default(
-                            span,
-                            sugg,
-                            args,
-                            &Level::Help,
-                            max_line_num_len,
-                        ) {
-                            panic!("failed to emit error: {}", e);
-                        };
+                            SuggestionStyle::HideCodeAlways => {
+                                if let Err(e) = self.emit_message_default(
+                                    &MultiSpan::new(),
+                                    &[(sugg.msg.to_owned(), Style::HeaderMsg)],
+                                    args,
+                                    &None,
+                                    &Level::Help,
+                                    max_line_num_len,
+                                    true,
+                                    None,
+                                ) {
+                                    panic!("failed to emit error: {}", e);
+                                }
+                            }
+                            SuggestionStyle::HideCodeInline
+                            | SuggestionStyle::ShowCode
+                            | SuggestionStyle::ShowAlways => {
+                                if let Err(e) = self.emit_suggestion_default(
+                                    span,
+                                    sugg,
+                                    args,
+                                    &Level::Help,
+                                    max_line_num_len,
+                                ) {
+                                    panic!("failed to emit error: {}", e);
+                                }
+                            }
+                        }
                     }
                 }
             }
@@ -2159,40 +2177,63 @@ impl EmitterWriter {
         buffer: &mut StyledBuffer,
         row_num: &mut usize,
         highlight_parts: &Vec<SubstitutionHighlight>,
-        line_pos: usize,
-        line: &str,
-        line_start: usize,
+        line_num: usize,
+        line_to_add: &str,
         show_code_change: DisplaySuggestion,
         max_line_num_len: usize,
         file_lines: &FileLines,
         is_multiline: bool,
     ) {
-        // Print the span column to avoid confusion
-        buffer.puts(*row_num, 0, &self.maybe_anonymized(line_start + line_pos), Style::LineNumber);
         if let DisplaySuggestion::Diff = show_code_change {
-            // Add the line number for both addition and removal to drive the point home.
-            //
-            // N - fn foo<A: T>(bar: A) {
-            // N + fn foo(bar: impl T) {
-            buffer.puts(
-                *row_num - 1,
-                0,
-                &self.maybe_anonymized(line_start + line_pos),
-                Style::LineNumber,
-            );
-            buffer.puts(*row_num - 1, max_line_num_len + 1, "- ", Style::Removal);
-            buffer.puts(
-                *row_num - 1,
-                max_line_num_len + 3,
-                &normalize_whitespace(
-                    &file_lines.file.get_line(file_lines.lines[line_pos].line_index).unwrap(),
-                ),
-                Style::NoStyle,
-            );
-            buffer.puts(*row_num, max_line_num_len + 1, "+ ", Style::Addition);
+            // We need to print more than one line if the span we need to remove is multiline.
+            // For more info: https://github.com/rust-lang/rust/issues/92741
+            let lines_to_remove = file_lines.lines.iter().take(file_lines.lines.len() - 1);
+            for (index, line_to_remove) in lines_to_remove.enumerate() {
+                buffer.puts(
+                    *row_num - 1,
+                    0,
+                    &self.maybe_anonymized(line_num + index),
+                    Style::LineNumber,
+                );
+                buffer.puts(*row_num - 1, max_line_num_len + 1, "- ", Style::Removal);
+                let line = normalize_whitespace(
+                    &file_lines.file.get_line(line_to_remove.line_index).unwrap(),
+                );
+                buffer.puts(*row_num - 1, max_line_num_len + 3, &line, Style::NoStyle);
+                *row_num += 1;
+            }
+            // If the last line is exactly equal to the line we need to add, we can skip both of them.
+            // This allows us to avoid output like the following:
+            // 2 - &
+            // 2 + if true { true } else { false }
+            // 3 - if true { true } else { false }
+            // If those lines aren't equal, we print their diff
+            let last_line_index = file_lines.lines[file_lines.lines.len() - 1].line_index;
+            let last_line = &file_lines.file.get_line(last_line_index).unwrap();
+            if last_line != line_to_add {
+                buffer.puts(
+                    *row_num - 1,
+                    0,
+                    &self.maybe_anonymized(line_num + file_lines.lines.len() - 1),
+                    Style::LineNumber,
+                );
+                buffer.puts(*row_num - 1, max_line_num_len + 1, "- ", Style::Removal);
+                buffer.puts(
+                    *row_num - 1,
+                    max_line_num_len + 3,
+                    &normalize_whitespace(last_line),
+                    Style::NoStyle,
+                );
+                buffer.puts(*row_num, 0, &self.maybe_anonymized(line_num), Style::LineNumber);
+                buffer.puts(*row_num, max_line_num_len + 1, "+ ", Style::Addition);
+                buffer.append(*row_num, &normalize_whitespace(line_to_add), Style::NoStyle);
+            } else {
+                *row_num -= 2;
+            }
         } else if is_multiline {
+            buffer.puts(*row_num, 0, &self.maybe_anonymized(line_num), Style::LineNumber);
             match &highlight_parts[..] {
-                [SubstitutionHighlight { start: 0, end }] if *end == line.len() => {
+                [SubstitutionHighlight { start: 0, end }] if *end == line_to_add.len() => {
                     buffer.puts(*row_num, max_line_num_len + 1, "+ ", Style::Addition);
                 }
                 [] => {
@@ -2202,17 +2243,17 @@ impl EmitterWriter {
                     buffer.puts(*row_num, max_line_num_len + 1, "~ ", Style::Addition);
                 }
             }
+            buffer.append(*row_num, &normalize_whitespace(line_to_add), Style::NoStyle);
         } else {
+            buffer.puts(*row_num, 0, &self.maybe_anonymized(line_num), Style::LineNumber);
             draw_col_separator(buffer, *row_num, max_line_num_len + 1);
+            buffer.append(*row_num, &normalize_whitespace(line_to_add), Style::NoStyle);
         }
 
-        // print the suggestion
-        buffer.append(*row_num, &normalize_whitespace(line), Style::NoStyle);
-
         // Colorize addition/replacements with green.
         for &SubstitutionHighlight { start, end } in highlight_parts {
             // Account for tabs when highlighting (#87972).
-            let tabs: usize = line
+            let tabs: usize = line_to_add
                 .chars()
                 .take(start)
                 .map(|ch| match ch {
diff --git a/compiler/rustc_errors/src/json.rs b/compiler/rustc_errors/src/json.rs
index dc38b8725..f32d6b96b 100644
--- a/compiler/rustc_errors/src/json.rs
+++ b/compiler/rustc_errors/src/json.rs
@@ -17,6 +17,7 @@ use crate::translation::{to_fluent_args, Translate};
 use crate::DiagnosticId;
 use crate::{
     CodeSuggestion, FluentBundle, LazyFallbackBundle, MultiSpan, SpanLabel, SubDiagnostic,
+    TerminalUrl,
 };
 use rustc_lint_defs::Applicability;
 
@@ -47,6 +48,7 @@ pub struct JsonEmitter {
     diagnostic_width: Option<usize>,
     macro_backtrace: bool,
     track_diagnostics: bool,
+    terminal_url: TerminalUrl,
 }
 
 impl JsonEmitter {
@@ -60,6 +62,7 @@ impl JsonEmitter {
         diagnostic_width: Option<usize>,
         macro_backtrace: bool,
         track_diagnostics: bool,
+        terminal_url: TerminalUrl,
     ) -> JsonEmitter {
         JsonEmitter {
             dst: Box::new(io::BufWriter::new(io::stderr())),
@@ -73,6 +76,7 @@ impl JsonEmitter {
             diagnostic_width,
             macro_backtrace,
             track_diagnostics,
+            terminal_url,
         }
     }
 
@@ -84,6 +88,7 @@ impl JsonEmitter {
         diagnostic_width: Option<usize>,
         macro_backtrace: bool,
         track_diagnostics: bool,
+        terminal_url: TerminalUrl,
     ) -> JsonEmitter {
         let file_path_mapping = FilePathMapping::empty();
         JsonEmitter::stderr(
@@ -96,6 +101,7 @@ impl JsonEmitter {
             diagnostic_width,
             macro_backtrace,
             track_diagnostics,
+            terminal_url,
         )
     }
 
@@ -110,6 +116,7 @@ impl JsonEmitter {
         diagnostic_width: Option<usize>,
         macro_backtrace: bool,
         track_diagnostics: bool,
+        terminal_url: TerminalUrl,
     ) -> JsonEmitter {
         JsonEmitter {
             dst,
@@ -123,6 +130,7 @@ impl JsonEmitter {
             diagnostic_width,
             macro_backtrace,
             track_diagnostics,
+            terminal_url,
         }
     }
 
@@ -360,6 +368,7 @@ impl Diagnostic {
                 je.diagnostic_width,
                 je.macro_backtrace,
                 je.track_diagnostics,
+                je.terminal_url,
             )
             .ui_testing(je.ui_testing)
             .emit_diagnostic(diag);
@@ -571,7 +580,7 @@ impl DiagnosticCode {
             let je_result =
                 je.registry.as_ref().map(|registry| registry.try_find_description(&s)).unwrap();
 
-            DiagnosticCode { code: s, explanation: je_result.unwrap_or(None) }
+            DiagnosticCode { code: s, explanation: je_result.ok() }
         })
     }
 }
diff --git a/compiler/rustc_errors/src/json/tests.rs b/compiler/rustc_errors/src/json/tests.rs
index f13146897..671dc449e 100644
--- a/compiler/rustc_errors/src/json/tests.rs
+++ b/compiler/rustc_errors/src/json/tests.rs
@@ -4,7 +4,7 @@ use crate::json::JsonEmitter;
 use rustc_span::source_map::{FilePathMapping, SourceMap};
 
 use crate::emitter::{ColorConfig, HumanReadableErrorType};
-use crate::Handler;
+use crate::{Handler, TerminalUrl};
 use rustc_span::{BytePos, Span};
 
 use std::str;
@@ -46,7 +46,7 @@ fn test_positions(code: &str, span: (u32, u32), expected_output: SpanTestData) {
         let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
         sm.new_source_file(Path::new("test.rs").to_owned().into(), code.to_owned());
         let fallback_bundle =
-            crate::fallback_fluent_bundle(rustc_error_messages::DEFAULT_LOCALE_RESOURCES, false);
+            crate::fallback_fluent_bundle(vec![crate::DEFAULT_LOCALE_RESOURCE], false);
 
         let output = Arc::new(Mutex::new(Vec::new()));
         let je = JsonEmitter::new(
@@ -60,6 +60,7 @@ fn test_positions(code: &str, span: (u32, u32), expected_output: SpanTestData) {
             None,
             false,
             false,
+            TerminalUrl::No,
         );
 
         let span = Span::with_root_ctxt(BytePos(span.0), BytePos(span.1));
diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs
index 535812fb0..09bf28ed4 100644
--- a/compiler/rustc_errors/src/lib.rs
+++ b/compiler/rustc_errors/src/lib.rs
@@ -3,6 +3,7 @@
 //! This module contains the code for creating and emitting diagnostics.
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
+#![feature(array_windows)]
 #![feature(drain_filter)]
 #![feature(if_let_guard)]
 #![feature(is_terminal)]
@@ -35,13 +36,13 @@ use rustc_data_structures::stable_hasher::StableHasher;
 use rustc_data_structures::sync::{self, Lock, Lrc};
 use rustc_data_structures::AtomicRef;
 pub use rustc_error_messages::{
-    fallback_fluent_bundle, fluent, fluent_bundle, DelayDm, DiagnosticMessage, FluentBundle,
+    fallback_fluent_bundle, fluent_bundle, DelayDm, DiagnosticMessage, FluentBundle,
     LanguageIdentifier, LazyFallbackBundle, MultiSpan, SpanLabel, SubdiagnosticMessage,
-    DEFAULT_LOCALE_RESOURCES,
 };
 pub use rustc_lint_defs::{pluralize, Applicability};
+use rustc_macros::fluent_messages;
 use rustc_span::source_map::SourceMap;
-use rustc_span::HashStableContext;
+pub use rustc_span::ErrorGuaranteed;
 use rustc_span::{Loc, Span};
 
 use std::borrow::Cow;
@@ -75,6 +76,8 @@ pub use snippet::Style;
 pub type PErr<'a> = DiagnosticBuilder<'a, ErrorGuaranteed>;
 pub type PResult<'a, T> = Result<T, PErr<'a>>;
 
+fluent_messages! { "../locales/en-US.ftl" }
+
 // `PResult` is used a lot. Make sure it doesn't unintentionally get bigger.
 // (See also the comment on `DiagnosticBuilderInner`'s `diagnostic` field.)
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
@@ -572,6 +575,7 @@ impl Handler {
             None,
             flags.macro_backtrace,
             flags.track_diagnostics,
+            TerminalUrl::No,
         ));
         Self::with_emitter_and_flags(emitter, flags)
     }
@@ -616,22 +620,24 @@ impl Handler {
         }
     }
 
-    /// Translate `message` eagerly with `args`.
+    /// Translate `message` eagerly with `args` to `SubdiagnosticMessage::Eager`.
     pub fn eagerly_translate<'a>(
         &self,
         message: DiagnosticMessage,
         args: impl Iterator<Item = DiagnosticArg<'a, 'static>>,
     ) -> SubdiagnosticMessage {
+        SubdiagnosticMessage::Eager(self.eagerly_translate_to_string(message, args))
+    }
+
+    /// Translate `message` eagerly with `args` to `String`.
+    pub fn eagerly_translate_to_string<'a>(
+        &self,
+        message: DiagnosticMessage,
+        args: impl Iterator<Item = DiagnosticArg<'a, 'static>>,
+    ) -> String {
         let inner = self.inner.borrow();
         let args = crate::translation::to_fluent_args(args);
-        SubdiagnosticMessage::Eager(
-            inner
-                .emitter
-                .translate_message(&message, &args)
-                .map_err(Report::new)
-                .unwrap()
-                .to_string(),
-        )
+        inner.emitter.translate_message(&message, &args).map_err(Report::new).unwrap().to_string()
     }
 
     // This is here to not allow mutation of flags;
@@ -1009,6 +1015,7 @@ impl Handler {
     }
 
     #[track_caller]
+    #[rustc_lint_diagnostics]
     pub fn span_note_without_error(
         &self,
         span: impl Into<MultiSpan>,
@@ -1018,6 +1025,7 @@ impl Handler {
     }
 
     #[track_caller]
+    #[rustc_lint_diagnostics]
     pub fn span_note_diag(
         &self,
         span: Span,
@@ -1029,19 +1037,23 @@ impl Handler {
     }
 
     // NOTE: intentionally doesn't raise an error so rustc_codegen_ssa only reports fatal errors in the main thread
+    #[rustc_lint_diagnostics]
     pub fn fatal(&self, msg: impl Into<DiagnosticMessage>) -> FatalError {
         self.inner.borrow_mut().fatal(msg)
     }
 
+    #[rustc_lint_diagnostics]
     pub fn err(&self, msg: impl Into<DiagnosticMessage>) -> ErrorGuaranteed {
         self.inner.borrow_mut().err(msg)
     }
 
+    #[rustc_lint_diagnostics]
     pub fn warn(&self, msg: impl Into<DiagnosticMessage>) {
         let mut db = DiagnosticBuilder::new(self, Warning(None), msg);
         db.emit();
     }
 
+    #[rustc_lint_diagnostics]
     pub fn note_without_error(&self, msg: impl Into<DiagnosticMessage>) {
         DiagnosticBuilder::new(self, Note, msg).emit();
     }
@@ -1056,28 +1068,26 @@ impl Handler {
     }
 
     pub fn has_errors(&self) -> Option<ErrorGuaranteed> {
-        if self.inner.borrow().has_errors() { Some(ErrorGuaranteed(())) } else { None }
+        self.inner.borrow().has_errors().then(ErrorGuaranteed::unchecked_claim_error_was_emitted)
     }
+
     pub fn has_errors_or_lint_errors(&self) -> Option<ErrorGuaranteed> {
-        if self.inner.borrow().has_errors_or_lint_errors() {
-            Some(ErrorGuaranteed::unchecked_claim_error_was_emitted())
-        } else {
-            None
-        }
+        self.inner
+            .borrow()
+            .has_errors_or_lint_errors()
+            .then(ErrorGuaranteed::unchecked_claim_error_was_emitted)
     }
     pub fn has_errors_or_delayed_span_bugs(&self) -> Option<ErrorGuaranteed> {
-        if self.inner.borrow().has_errors_or_delayed_span_bugs() {
-            Some(ErrorGuaranteed::unchecked_claim_error_was_emitted())
-        } else {
-            None
-        }
+        self.inner
+            .borrow()
+            .has_errors_or_delayed_span_bugs()
+            .then(ErrorGuaranteed::unchecked_claim_error_was_emitted)
     }
     pub fn is_compilation_going_to_fail(&self) -> Option<ErrorGuaranteed> {
-        if self.inner.borrow().is_compilation_going_to_fail() {
-            Some(ErrorGuaranteed::unchecked_claim_error_was_emitted())
-        } else {
-            None
-        }
+        self.inner
+            .borrow()
+            .is_compilation_going_to_fail()
+            .then(ErrorGuaranteed::unchecked_claim_error_was_emitted)
     }
 
     pub fn print_error_count(&self, registry: &Registry) {
@@ -1131,6 +1141,20 @@ impl Handler {
         self.create_warning(warning).emit()
     }
 
+    pub fn create_almost_fatal<'a>(
+        &'a self,
+        fatal: impl IntoDiagnostic<'a, FatalError>,
+    ) -> DiagnosticBuilder<'a, FatalError> {
+        fatal.into_diagnostic(self)
+    }
+
+    pub fn emit_almost_fatal<'a>(
+        &'a self,
+        fatal: impl IntoDiagnostic<'a, FatalError>,
+    ) -> FatalError {
+        self.create_almost_fatal(fatal).emit()
+    }
+
     pub fn create_fatal<'a>(
         &'a self,
         fatal: impl IntoDiagnostic<'a, !>,
@@ -1156,6 +1180,17 @@ impl Handler {
         self.create_bug(bug).emit()
     }
 
+    pub fn emit_note<'a>(&'a self, note: impl IntoDiagnostic<'a, Noted>) -> Noted {
+        self.create_note(note).emit()
+    }
+
+    pub fn create_note<'a>(
+        &'a self,
+        note: impl IntoDiagnostic<'a, Noted>,
+    ) -> DiagnosticBuilder<'a, Noted> {
+        note.into_diagnostic(self)
+    }
+
     fn emit_diag_at_span(
         &self,
         mut diag: Diagnostic,
@@ -1440,9 +1475,7 @@ impl HandlerInner {
                 .emitted_diagnostic_codes
                 .iter()
                 .filter_map(|x| match &x {
-                    DiagnosticId::Error(s)
-                        if registry.try_find_description(s).map_or(false, |o| o.is_some()) =>
-                    {
+                    DiagnosticId::Error(s) if registry.try_find_description(s).is_ok() => {
                         Some(s.clone())
                     }
                     _ => None,
@@ -1803,16 +1836,9 @@ pub fn add_elided_lifetime_in_path_suggestion(
     );
 }
 
-/// Useful type to use with `Result<>` indicate that an error has already
-/// been reported to the user, so no need to continue checking.
-#[derive(Clone, Copy, Debug, Encodable, Decodable, Hash, PartialEq, Eq, PartialOrd, Ord)]
-#[derive(HashStable_Generic)]
-pub struct ErrorGuaranteed(());
-
-impl ErrorGuaranteed {
-    /// To be used only if you really know what you are doing... ideally, we would find a way to
-    /// eliminate all calls to this method.
-    pub fn unchecked_claim_error_was_emitted() -> Self {
-        ErrorGuaranteed(())
-    }
+#[derive(Clone, Copy, PartialEq, Hash, Debug)]
+pub enum TerminalUrl {
+    No,
+    Yes,
+    Auto,
 }
diff --git a/compiler/rustc_errors/src/registry.rs b/compiler/rustc_errors/src/registry.rs
index da764d993..f26d8e7eb 100644
--- a/compiler/rustc_errors/src/registry.rs
+++ b/compiler/rustc_errors/src/registry.rs
@@ -5,21 +5,17 @@ pub struct InvalidErrorCode;
 
 #[derive(Clone)]
 pub struct Registry {
-    long_descriptions: FxHashMap<&'static str, Option<&'static str>>,
+    long_descriptions: FxHashMap<&'static str, &'static str>,
 }
 
 impl Registry {
-    pub fn new(long_descriptions: &[(&'static str, Option<&'static str>)]) -> Registry {
+    pub fn new(long_descriptions: &[(&'static str, &'static str)]) -> Registry {
         Registry { long_descriptions: long_descriptions.iter().copied().collect() }
     }
 
     /// Returns `InvalidErrorCode` if the code requested does not exist in the
-    /// registry. Otherwise, returns an `Option` where `None` means the error
-    /// code is valid but has no extended information.
-    pub fn try_find_description(
-        &self,
-        code: &str,
-    ) -> Result<Option<&'static str>, InvalidErrorCode> {
+    /// registry.
+    pub fn try_find_description(&self, code: &str) -> Result<&'static str, InvalidErrorCode> {
         self.long_descriptions.get(code).copied().ok_or(InvalidErrorCode)
     }
 }
diff --git a/compiler/rustc_errors/src/styled_buffer.rs b/compiler/rustc_errors/src/styled_buffer.rs
index 9abdb5fc9..9aa14e1f2 100644
--- a/compiler/rustc_errors/src/styled_buffer.rs
+++ b/compiler/rustc_errors/src/styled_buffer.rs
@@ -142,7 +142,7 @@ impl StyledBuffer {
     pub fn set_style(&mut self, line: usize, col: usize, style: Style, overwrite: bool) {
         if let Some(ref mut line) = self.lines.get_mut(line) {
             if let Some(StyledChar { style: s, .. }) = line.get_mut(col) {
-                if overwrite || *s == Style::NoStyle || *s == Style::Quotation {
+                if overwrite || matches!(s, Style::NoStyle | Style::Quotation) {
                     *s = style;
                 }
             }
diff --git a/compiler/rustc_errors/src/translation.rs b/compiler/rustc_errors/src/translation.rs
index addfc9726..ed35eb1b6 100644
--- a/compiler/rustc_errors/src/translation.rs
+++ b/compiler/rustc_errors/src/translation.rs
@@ -1,9 +1,10 @@
-use crate::error::TranslateError;
+use crate::error::{TranslateError, TranslateErrorKind};
 use crate::snippet::Style;
 use crate::{DiagnosticArg, DiagnosticMessage, FluentBundle};
 use rustc_data_structures::sync::Lrc;
 use rustc_error_messages::FluentArgs;
 use std::borrow::Cow;
+use std::env;
 use std::error::Report;
 
 /// Convert diagnostic arguments (a rustc internal type that exists to implement
@@ -94,12 +95,29 @@ pub trait Translate {
                 // The primary bundle was present and translation succeeded
                 Some(Ok(t)) => t,
 
-                // Always yeet out for errors on debug
-                Some(Err(primary)) if cfg!(debug_assertions) => do yeet primary,
-
                 // If `translate_with_bundle` returns `Err` with the primary bundle, this is likely
-                // just that the primary bundle doesn't contain the message being translated or
-                // something else went wrong) so proceed to the fallback bundle.
+                // just that the primary bundle doesn't contain the message being translated, so
+                // proceed to the fallback bundle.
+                Some(Err(
+                    primary @ TranslateError::One {
+                        kind: TranslateErrorKind::MessageMissing, ..
+                    },
+                )) => translate_with_bundle(self.fallback_fluent_bundle())
+                    .map_err(|fallback| primary.and(fallback))?,
+
+                // Always yeet out for errors on debug (unless
+                // `RUSTC_TRANSLATION_NO_DEBUG_ASSERT` is set in the environment - this allows
+                // local runs of the test suites, of builds with debug assertions, to test the
+                // behaviour in a normal build).
+                Some(Err(primary))
+                    if cfg!(debug_assertions)
+                        && env::var("RUSTC_TRANSLATION_NO_DEBUG_ASSERT").is_err() =>
+                {
+                    do yeet primary
+                }
+
+                // ..otherwise, for end users, an error about this wouldn't be useful or actionable, so
+                // just hide it and try with the fallback bundle.
                 Some(Err(primary)) => translate_with_bundle(self.fallback_fluent_bundle())
                     .map_err(|fallback| primary.and(fallback))?,
 
diff --git a/compiler/rustc_expand/Cargo.toml b/compiler/rustc_expand/Cargo.toml
index 192f54171..c971714e0 100644
--- a/compiler/rustc_expand/Cargo.toml
+++ b/compiler/rustc_expand/Cargo.toml
@@ -24,5 +24,5 @@ rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-thin-vec = "0.2.8"
+thin-vec = "0.2.12"
 tracing = "0.1"
diff --git a/compiler/rustc_error_messages/locales/en-US/expand.ftl b/compiler/rustc_expand/locales/en-US.ftl
index df0e8ae5d..cfae781bd 100644
--- a/compiler/rustc_error_messages/locales/en-US/expand.ftl
+++ b/compiler/rustc_expand/locales/en-US.ftl
@@ -127,3 +127,12 @@ expand_module_file_not_found =
 expand_module_multiple_candidates =
     file for module `{$name}` found at both "{$default_path}" and "{$secondary_path}"
     .help = delete or rename one of them to remove the ambiguity
+
+expand_trace_macro = trace_macro
+
+expand_proc_macro_panicked =
+    proc macro panicked
+    .help = message: {$message}
+
+expand_proc_macro_derive_tokens =
+    proc-macro derive produced unparseable tokens
diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs
index 951d59246..22bc90f5c 100644
--- a/compiler/rustc_expand/src/base.rs
+++ b/compiler/rustc_expand/src/base.rs
@@ -1,15 +1,11 @@
 #![deny(rustc::untranslatable_diagnostic)]
 
-use crate::errors::{
-    ArgumentNotAttributes, AttrNoArguments, AttributeMetaItem, AttributeSingleWord,
-    AttributesWrongForm, CannotBeNameOfMacro, ExpectedCommaInList, HelperAttributeNameInvalid,
-    MacroBodyStability, MacroConstStability, NotAMetaItem, OnlyOneArgument, OnlyOneWord,
-    ResolveRelativePath, TakesNoArguments,
-};
+use crate::errors;
 use crate::expand::{self, AstFragment, Invocation};
 use crate::module::DirOwnership;
 
 use rustc_ast::attr::MarkedAttrs;
+use rustc_ast::mut_visit::DummyAstNode;
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Nonterminal};
 use rustc_ast::tokenstream::TokenStream;
@@ -33,10 +29,11 @@ use rustc_span::source_map::SourceMap;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{BytePos, FileName, Span, DUMMY_SP};
 use smallvec::{smallvec, SmallVec};
-
+use std::default::Default;
 use std::iter;
 use std::path::{Path, PathBuf};
 use std::rc::Rc;
+use thin_vec::ThinVec;
 
 pub(crate) use rustc_span::hygiene::MacroKind;
 
@@ -558,7 +555,7 @@ impl DummyResult {
     pub fn raw_expr(sp: Span, is_error: bool) -> P<ast::Expr> {
         P(ast::Expr {
             id: ast::DUMMY_NODE_ID,
-            kind: if is_error { ast::ExprKind::Err } else { ast::ExprKind::Tup(Vec::new()) },
+            kind: if is_error { ast::ExprKind::Err } else { ast::ExprKind::Tup(ThinVec::new()) },
             span: sp,
             attrs: ast::AttrVec::new(),
             tokens: None,
@@ -574,7 +571,7 @@ impl DummyResult {
     pub fn raw_ty(sp: Span, is_error: bool) -> P<ast::Ty> {
         P(ast::Ty {
             id: ast::DUMMY_NODE_ID,
-            kind: if is_error { ast::TyKind::Err } else { ast::TyKind::Tup(Vec::new()) },
+            kind: if is_error { ast::TyKind::Err } else { ast::TyKind::Tup(ThinVec::new()) },
             span: sp,
             tokens: None,
         })
@@ -645,6 +642,10 @@ impl MacResult for DummyResult {
     fn make_variants(self: Box<DummyResult>) -> Option<SmallVec<[ast::Variant; 1]>> {
         Some(SmallVec::new())
     }
+
+    fn make_crate(self: Box<DummyResult>) -> Option<ast::Crate> {
+        Some(DummyAstNode::dummy())
+    }
 }
 
 /// A syntax extension kind.
@@ -796,13 +797,13 @@ impl SyntaxExtension {
             .unwrap_or_else(|| (None, helper_attrs));
         let (stability, const_stability, body_stability) = attr::find_stability(&sess, attrs, span);
         if let Some((_, sp)) = const_stability {
-            sess.emit_err(MacroConstStability {
+            sess.emit_err(errors::MacroConstStability {
                 span: sp,
                 head_span: sess.source_map().guess_head_span(span),
             });
         }
         if let Some((_, sp)) = body_stability {
-            sess.emit_err(MacroBodyStability {
+            sess.emit_err(errors::MacroBodyStability {
                 span: sp,
                 head_span: sess.source_map().guess_head_span(span),
             });
@@ -1142,8 +1143,8 @@ impl<'a> ExtCtxt<'a> {
         self.sess.parse_sess.span_diagnostic.span_bug(sp, msg);
     }
     pub fn trace_macros_diag(&mut self) {
-        for (sp, notes) in self.expansions.iter() {
-            let mut db = self.sess.parse_sess.span_diagnostic.span_note_diag(*sp, "trace_macro");
+        for (span, notes) in self.expansions.iter() {
+            let mut db = self.sess.parse_sess.create_note(errors::TraceMacro { span: *span });
             for note in notes {
                 db.note(note);
             }
@@ -1197,7 +1198,7 @@ pub fn resolve_path(
                 .expect("attempting to resolve a file path in an external file"),
             FileName::DocTest(path, _) => path,
             other => {
-                return Err(ResolveRelativePath {
+                return Err(errors::ResolveRelativePath {
                     span,
                     path: parse_sess.source_map().filename_for_diagnostics(&other).to_string(),
                 }
@@ -1279,7 +1280,7 @@ pub fn expr_to_string(
 /// done as rarely as possible).
 pub fn check_zero_tts(cx: &ExtCtxt<'_>, span: Span, tts: TokenStream, name: &str) {
     if !tts.is_empty() {
-        cx.emit_err(TakesNoArguments { span, name });
+        cx.emit_err(errors::TakesNoArguments { span, name });
     }
 }
 
@@ -1307,14 +1308,14 @@ pub fn get_single_str_from_tts(
 ) -> Option<Symbol> {
     let mut p = cx.new_parser_from_tts(tts);
     if p.token == token::Eof {
-        cx.emit_err(OnlyOneArgument { span, name });
+        cx.emit_err(errors::OnlyOneArgument { span, name });
         return None;
     }
     let ret = parse_expr(&mut p)?;
     let _ = p.eat(&token::Comma);
 
     if p.token != token::Eof {
-        cx.emit_err(OnlyOneArgument { span, name });
+        cx.emit_err(errors::OnlyOneArgument { span, name });
     }
     expr_to_string(cx, ret, "argument must be a string literal").map(|(s, _)| s)
 }
@@ -1336,7 +1337,7 @@ pub fn get_exprs_from_tts(cx: &mut ExtCtxt<'_>, tts: TokenStream) -> Option<Vec<
             continue;
         }
         if p.token != token::Eof {
-            cx.emit_err(ExpectedCommaInList { span: p.token.span });
+            cx.emit_err(errors::ExpectedCommaInList { span: p.token.span });
             return None;
         }
     }
@@ -1353,51 +1354,58 @@ pub fn parse_macro_name_and_helper_attrs(
     // `#[proc_macro_derive(Foo, attributes(A, ..))]`
     let list = attr.meta_item_list()?;
     if list.len() != 1 && list.len() != 2 {
-        diag.emit_err(AttrNoArguments { span: attr.span });
+        diag.emit_err(errors::AttrNoArguments { span: attr.span });
         return None;
     }
     let Some(trait_attr) = list[0].meta_item() else {
-        diag.emit_err(NotAMetaItem {span: list[0].span()});
+        diag.emit_err(errors::NotAMetaItem {span: list[0].span()});
         return None;
     };
     let trait_ident = match trait_attr.ident() {
         Some(trait_ident) if trait_attr.is_word() => trait_ident,
         _ => {
-            diag.emit_err(OnlyOneWord { span: trait_attr.span });
+            diag.emit_err(errors::OnlyOneWord { span: trait_attr.span });
             return None;
         }
     };
 
     if !trait_ident.name.can_be_raw() {
-        diag.emit_err(CannotBeNameOfMacro { span: trait_attr.span, trait_ident, macro_type });
+        diag.emit_err(errors::CannotBeNameOfMacro {
+            span: trait_attr.span,
+            trait_ident,
+            macro_type,
+        });
     }
 
     let attributes_attr = list.get(1);
     let proc_attrs: Vec<_> = if let Some(attr) = attributes_attr {
         if !attr.has_name(sym::attributes) {
-            diag.emit_err(ArgumentNotAttributes { span: attr.span() });
+            diag.emit_err(errors::ArgumentNotAttributes { span: attr.span() });
         }
         attr.meta_item_list()
             .unwrap_or_else(|| {
-                diag.emit_err(AttributesWrongForm { span: attr.span() });
+                diag.emit_err(errors::AttributesWrongForm { span: attr.span() });
                 &[]
             })
             .iter()
             .filter_map(|attr| {
                 let Some(attr) = attr.meta_item() else {
-                    diag.emit_err(AttributeMetaItem { span: attr.span() });
+                    diag.emit_err(errors::AttributeMetaItem { span: attr.span() });
                     return None;
                 };
 
                 let ident = match attr.ident() {
                     Some(ident) if attr.is_word() => ident,
                     _ => {
-                        diag.emit_err(AttributeSingleWord { span: attr.span });
+                        diag.emit_err(errors::AttributeSingleWord { span: attr.span });
                         return None;
                     }
                 };
                 if !ident.name.can_be_raw() {
-                    diag.emit_err(HelperAttributeNameInvalid { span: attr.span, name: ident });
+                    diag.emit_err(errors::HelperAttributeNameInvalid {
+                        span: attr.span,
+                        name: ident,
+                    });
                 }
 
                 Some(ident.name)
diff --git a/compiler/rustc_expand/src/build.rs b/compiler/rustc_expand/src/build.rs
index 9b16e79d4..8a78c3296 100644
--- a/compiler/rustc_expand/src/build.rs
+++ b/compiler/rustc_expand/src/build.rs
@@ -5,7 +5,7 @@ use rustc_ast::{attr, token, util::literal};
 use rustc_span::source_map::Spanned;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::Span;
-use thin_vec::ThinVec;
+use thin_vec::{thin_vec, ThinVec};
 
 impl<'a> ExtCtxt<'a> {
     pub fn path(&self, span: Span, strs: Vec<Ident>) -> ast::Path {
@@ -125,16 +125,20 @@ impl<'a> ExtCtxt<'a> {
 
     pub fn poly_trait_ref(&self, span: Span, path: ast::Path) -> ast::PolyTraitRef {
         ast::PolyTraitRef {
-            bound_generic_params: Vec::new(),
+            bound_generic_params: ThinVec::new(),
             trait_ref: self.trait_ref(path),
             span,
         }
     }
 
-    pub fn trait_bound(&self, path: ast::Path) -> ast::GenericBound {
+    pub fn trait_bound(&self, path: ast::Path, is_const: bool) -> ast::GenericBound {
         ast::GenericBound::Trait(
             self.poly_trait_ref(path.span, path),
-            ast::TraitBoundModifier::None,
+            if is_const {
+                ast::TraitBoundModifier::MaybeConst
+            } else {
+                ast::TraitBoundModifier::None
+            },
         )
     }
 
@@ -217,14 +221,14 @@ impl<'a> ExtCtxt<'a> {
     pub fn block_expr(&self, expr: P<ast::Expr>) -> P<ast::Block> {
         self.block(
             expr.span,
-            vec![ast::Stmt {
+            thin_vec![ast::Stmt {
                 id: ast::DUMMY_NODE_ID,
                 span: expr.span,
                 kind: ast::StmtKind::Expr(expr),
             }],
         )
     }
-    pub fn block(&self, span: Span, stmts: Vec<ast::Stmt>) -> P<ast::Block> {
+    pub fn block(&self, span: Span, stmts: ThinVec<ast::Stmt>) -> P<ast::Block> {
         P(ast::Block {
             stmts,
             id: ast::DUMMY_NODE_ID,
@@ -272,22 +276,31 @@ impl<'a> ExtCtxt<'a> {
         self.expr(sp, ast::ExprKind::AddrOf(ast::BorrowKind::Ref, ast::Mutability::Not, e))
     }
 
+    pub fn expr_paren(&self, sp: Span, e: P<ast::Expr>) -> P<ast::Expr> {
+        self.expr(sp, ast::ExprKind::Paren(e))
+    }
+
     pub fn expr_call(
         &self,
         span: Span,
         expr: P<ast::Expr>,
-        args: Vec<P<ast::Expr>>,
+        args: ThinVec<P<ast::Expr>>,
     ) -> P<ast::Expr> {
         self.expr(span, ast::ExprKind::Call(expr, args))
     }
-    pub fn expr_call_ident(&self, span: Span, id: Ident, args: Vec<P<ast::Expr>>) -> P<ast::Expr> {
+    pub fn expr_call_ident(
+        &self,
+        span: Span,
+        id: Ident,
+        args: ThinVec<P<ast::Expr>>,
+    ) -> P<ast::Expr> {
         self.expr(span, ast::ExprKind::Call(self.expr_ident(span, id), args))
     }
     pub fn expr_call_global(
         &self,
         sp: Span,
         fn_path: Vec<Ident>,
-        args: Vec<P<ast::Expr>>,
+        args: ThinVec<P<ast::Expr>>,
     ) -> P<ast::Expr> {
         let pathexpr = self.expr_path(self.path_global(sp, fn_path));
         self.expr_call(sp, pathexpr, args)
@@ -310,7 +323,7 @@ impl<'a> ExtCtxt<'a> {
         &self,
         span: Span,
         path: ast::Path,
-        fields: Vec<ast::ExprField>,
+        fields: ThinVec<ast::ExprField>,
     ) -> P<ast::Expr> {
         self.expr(
             span,
@@ -326,7 +339,7 @@ impl<'a> ExtCtxt<'a> {
         &self,
         span: Span,
         id: Ident,
-        fields: Vec<ast::ExprField>,
+        fields: ThinVec<ast::ExprField>,
     ) -> P<ast::Expr> {
         self.expr_struct(span, self.path_ident(span, id), fields)
     }
@@ -364,12 +377,12 @@ impl<'a> ExtCtxt<'a> {
     }
 
     /// `[expr1, expr2, ...]`
-    pub fn expr_array(&self, sp: Span, exprs: Vec<P<ast::Expr>>) -> P<ast::Expr> {
+    pub fn expr_array(&self, sp: Span, exprs: ThinVec<P<ast::Expr>>) -> P<ast::Expr> {
         self.expr(sp, ast::ExprKind::Array(exprs))
     }
 
     /// `&[expr1, expr2, ...]`
-    pub fn expr_array_ref(&self, sp: Span, exprs: Vec<P<ast::Expr>>) -> P<ast::Expr> {
+    pub fn expr_array_ref(&self, sp: Span, exprs: ThinVec<P<ast::Expr>>) -> P<ast::Expr> {
         self.expr_addr_of(sp, self.expr_array(sp, exprs))
     }
 
@@ -379,14 +392,14 @@ impl<'a> ExtCtxt<'a> {
 
     pub fn expr_some(&self, sp: Span, expr: P<ast::Expr>) -> P<ast::Expr> {
         let some = self.std_path(&[sym::option, sym::Option, sym::Some]);
-        self.expr_call_global(sp, some, vec![expr])
+        self.expr_call_global(sp, some, thin_vec![expr])
     }
 
     pub fn expr_none(&self, sp: Span) -> P<ast::Expr> {
         let none = self.std_path(&[sym::option, sym::Option, sym::None]);
         self.expr_path(self.path_global(sp, none))
     }
-    pub fn expr_tuple(&self, sp: Span, exprs: Vec<P<ast::Expr>>) -> P<ast::Expr> {
+    pub fn expr_tuple(&self, sp: Span, exprs: ThinVec<P<ast::Expr>>) -> P<ast::Expr> {
         self.expr(sp, ast::ExprKind::Tup(exprs))
     }
 
@@ -394,7 +407,7 @@ impl<'a> ExtCtxt<'a> {
         self.expr_call_global(
             span,
             [sym::std, sym::rt, sym::begin_panic].iter().map(|s| Ident::new(*s, span)).collect(),
-            vec![self.expr_str(span, msg)],
+            thin_vec![self.expr_str(span, msg)],
         )
     }
 
@@ -404,7 +417,7 @@ impl<'a> ExtCtxt<'a> {
 
     pub fn expr_ok(&self, sp: Span, expr: P<ast::Expr>) -> P<ast::Expr> {
         let ok = self.std_path(&[sym::result, sym::Result, sym::Ok]);
-        self.expr_call_global(sp, ok, vec![expr])
+        self.expr_call_global(sp, ok, thin_vec![expr])
     }
 
     pub fn expr_try(&self, sp: Span, head: P<ast::Expr>) -> P<ast::Expr> {
@@ -418,12 +431,12 @@ impl<'a> ExtCtxt<'a> {
         let binding_expr = self.expr_ident(sp, binding_variable);
 
         // `Ok(__try_var)` pattern
-        let ok_pat = self.pat_tuple_struct(sp, ok_path, vec![binding_pat.clone()]);
+        let ok_pat = self.pat_tuple_struct(sp, ok_path, thin_vec![binding_pat.clone()]);
 
         // `Err(__try_var)` (pattern and expression respectively)
-        let err_pat = self.pat_tuple_struct(sp, err_path.clone(), vec![binding_pat]);
+        let err_pat = self.pat_tuple_struct(sp, err_path.clone(), thin_vec![binding_pat]);
         let err_inner_expr =
-            self.expr_call(sp, self.expr_path(err_path), vec![binding_expr.clone()]);
+            self.expr_call(sp, self.expr_path(err_path), thin_vec![binding_expr.clone()]);
         // `return Err(__try_var)`
         let err_expr = self.expr(sp, ast::ExprKind::Ret(Some(err_inner_expr)));
 
@@ -433,7 +446,7 @@ impl<'a> ExtCtxt<'a> {
         let err_arm = self.arm(sp, err_pat, err_expr);
 
         // `match head { Ok() => ..., Err() => ... }`
-        self.expr_match(sp, head, vec![ok_arm, err_arm])
+        self.expr_match(sp, head, thin_vec![ok_arm, err_arm])
     }
 
     pub fn pat(&self, span: Span, kind: PatKind) -> P<ast::Pat> {
@@ -465,7 +478,7 @@ impl<'a> ExtCtxt<'a> {
         &self,
         span: Span,
         path: ast::Path,
-        subpats: Vec<P<ast::Pat>>,
+        subpats: ThinVec<P<ast::Pat>>,
     ) -> P<ast::Pat> {
         self.pat(span, PatKind::TupleStruct(None, path, subpats))
     }
@@ -473,18 +486,18 @@ impl<'a> ExtCtxt<'a> {
         &self,
         span: Span,
         path: ast::Path,
-        field_pats: Vec<ast::PatField>,
+        field_pats: ThinVec<ast::PatField>,
     ) -> P<ast::Pat> {
         self.pat(span, PatKind::Struct(None, path, field_pats, false))
     }
-    pub fn pat_tuple(&self, span: Span, pats: Vec<P<ast::Pat>>) -> P<ast::Pat> {
+    pub fn pat_tuple(&self, span: Span, pats: ThinVec<P<ast::Pat>>) -> P<ast::Pat> {
         self.pat(span, PatKind::Tuple(pats))
     }
 
     pub fn pat_some(&self, span: Span, pat: P<ast::Pat>) -> P<ast::Pat> {
         let some = self.std_path(&[sym::option, sym::Option, sym::Some]);
         let path = self.path_global(span, some);
-        self.pat_tuple_struct(span, path, vec![pat])
+        self.pat_tuple_struct(span, path, thin_vec![pat])
     }
 
     pub fn arm(&self, span: Span, pat: P<ast::Pat>, expr: P<ast::Expr>) -> ast::Arm {
@@ -503,7 +516,7 @@ impl<'a> ExtCtxt<'a> {
         self.arm(span, self.pat_wild(span), self.expr_unreachable(span))
     }
 
-    pub fn expr_match(&self, span: Span, arg: P<ast::Expr>, arms: Vec<ast::Arm>) -> P<Expr> {
+    pub fn expr_match(&self, span: Span, arg: P<ast::Expr>, arms: ThinVec<ast::Arm>) -> P<Expr> {
         self.expr(span, ast::ExprKind::Match(arg, arms))
     }
 
@@ -554,7 +567,12 @@ impl<'a> ExtCtxt<'a> {
         self.lambda(span, vec![ident], body)
     }
 
-    pub fn lambda_stmts_1(&self, span: Span, stmts: Vec<ast::Stmt>, ident: Ident) -> P<ast::Expr> {
+    pub fn lambda_stmts_1(
+        &self,
+        span: Span,
+        stmts: ThinVec<ast::Stmt>,
+        ident: Ident,
+    ) -> P<ast::Expr> {
         self.lambda1(span, self.expr_block(self.block(span, stmts)), ident)
     }
 
@@ -571,7 +589,7 @@ impl<'a> ExtCtxt<'a> {
     }
 
     // `self` is unused but keep it as method for the convenience use.
-    pub fn fn_decl(&self, inputs: Vec<ast::Param>, output: ast::FnRetTy) -> P<ast::FnDecl> {
+    pub fn fn_decl(&self, inputs: ThinVec<ast::Param>, output: ast::FnRetTy) -> P<ast::FnDecl> {
         P(ast::FnDecl { inputs, output })
     }
 
diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs
index 1fcbdfd9b..01500c2c7 100644
--- a/compiler/rustc_expand/src/config.rs
+++ b/compiler/rustc_expand/src/config.rs
@@ -24,6 +24,7 @@ use rustc_session::Session;
 use rustc_span::edition::{Edition, ALL_EDITIONS};
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::{Span, DUMMY_SP};
+use thin_vec::ThinVec;
 
 /// A folder that strips out items that do not belong in the current configuration.
 pub struct StripUnconfigured<'a> {
@@ -206,7 +207,7 @@ pub fn features(
         None => {
             // The entire crate is unconfigured.
             krate.attrs = ast::AttrVec::new();
-            krate.items = Vec::new();
+            krate.items = ThinVec::new();
             Features::default()
         }
         Some(attrs) => {
@@ -238,12 +239,10 @@ macro_rules! configure {
 impl<'a> StripUnconfigured<'a> {
     pub fn configure<T: HasAttrs + HasTokens>(&self, mut node: T) -> Option<T> {
         self.process_cfg_attrs(&mut node);
-        if self.in_cfg(node.attrs()) {
+        self.in_cfg(node.attrs()).then(|| {
             self.try_configure_tokens(&mut node);
-            Some(node)
-        } else {
-            None
-        }
+            node
+        })
     }
 
     fn try_configure_tokens<T: HasTokens>(&self, node: &mut T) {
@@ -257,7 +256,7 @@ impl<'a> StripUnconfigured<'a> {
 
     fn configure_krate_attrs(&self, mut attrs: ast::AttrVec) -> Option<ast::AttrVec> {
         attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr));
-        if self.in_cfg(&attrs) { Some(attrs) } else { None }
+        self.in_cfg(&attrs).then_some(attrs)
     }
 
     /// Performs cfg-expansion on `stream`, producing a new `AttrTokenStream`.
diff --git a/compiler/rustc_expand/src/errors.rs b/compiler/rustc_expand/src/errors.rs
index afe5169d3..e5102a952 100644
--- a/compiler/rustc_expand/src/errors.rs
+++ b/compiler/rustc_expand/src/errors.rs
@@ -65,7 +65,7 @@ pub(crate) struct MacroConstStability {
     #[primary_span]
     #[label]
     pub span: Span,
-    #[label(label2)]
+    #[label(expand_label2)]
     pub head_span: Span,
 }
 
@@ -75,7 +75,7 @@ pub(crate) struct MacroBodyStability {
     #[primary_span]
     #[label]
     pub span: Span,
-    #[label(label2)]
+    #[label(expand_label2)]
     pub head_span: Span,
 }
 
@@ -188,7 +188,7 @@ pub(crate) struct FeatureRemoved<'a> {
 }
 
 #[derive(Subdiagnostic)]
-#[note(reason)]
+#[note(expand_reason)]
 pub(crate) struct FeatureRemovedReason<'a> {
     pub reason: &'a str,
 }
@@ -223,12 +223,12 @@ pub(crate) struct MalformedFeatureAttribute {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum MalformedFeatureAttributeHelp {
-    #[label(expected)]
+    #[label(expand_expected)]
     Label {
         #[primary_span]
         span: Span,
     },
-    #[suggestion(expected, code = "{suggestion}", applicability = "maybe-incorrect")]
+    #[suggestion(expand_expected, code = "{suggestion}", applicability = "maybe-incorrect")]
     Suggestion {
         #[primary_span]
         span: Span,
@@ -306,7 +306,7 @@ pub(crate) struct IncompleteParse<'a> {
     pub kind_name: &'a str,
 
     #[suggestion(
-        suggestion_add_semi,
+        expand_suggestion_add_semi,
         style = "verbose",
         code = ";",
         applicability = "maybe-incorrect"
@@ -340,7 +340,7 @@ pub(crate) struct ModuleInBlock {
 }
 
 #[derive(Subdiagnostic)]
-#[note(note)]
+#[note(expand_note)]
 pub(crate) struct ModuleInBlockName {
     #[primary_span]
     pub span: Span,
@@ -368,3 +368,32 @@ pub(crate) struct ModuleMultipleCandidates {
     pub default_path: String,
     pub secondary_path: String,
 }
+
+#[derive(Diagnostic)]
+#[diag(expand_trace_macro)]
+pub struct TraceMacro {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(expand_proc_macro_panicked)]
+pub(crate) struct ProcMacroPanicked {
+    #[primary_span]
+    pub span: Span,
+    #[subdiagnostic]
+    pub message: Option<ProcMacroPanickedHelp>,
+}
+
+#[derive(Subdiagnostic)]
+#[help(expand_help)]
+pub(crate) struct ProcMacroPanickedHelp {
+    pub message: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(expand_proc_macro_derive_tokens)]
+pub struct ProcMacroDeriveTokens {
+    #[primary_span]
+    pub span: Span,
+}
diff --git a/compiler/rustc_expand/src/lib.rs b/compiler/rustc_expand/src/lib.rs
index 897268566..634e206e5 100644
--- a/compiler/rustc_expand/src/lib.rs
+++ b/compiler/rustc_expand/src/lib.rs
@@ -20,6 +20,9 @@ extern crate tracing;
 
 extern crate proc_macro as pm;
 
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
+use rustc_macros::fluent_messages;
+
 mod placeholders;
 mod proc_macro_server;
 
@@ -60,3 +63,5 @@ mod tokenstream {
 mod mut_visit {
     mod tests;
 }
+
+fluent_messages! { "../locales/en-US.ftl" }
diff --git a/compiler/rustc_expand/src/mbe/macro_parser.rs b/compiler/rustc_expand/src/mbe/macro_parser.rs
index 2e199541b..283e68a68 100644
--- a/compiler/rustc_expand/src/mbe/macro_parser.rs
+++ b/compiler/rustc_expand/src/mbe/macro_parser.rs
@@ -503,7 +503,7 @@ impl TtParser {
                         mp.push_match(metavar_idx, seq_depth, MatchedSeq(vec![]));
                     }
 
-                    if op == KleeneOp::ZeroOrMore || op == KleeneOp::ZeroOrOne {
+                    if matches!(op, KleeneOp::ZeroOrMore | KleeneOp::ZeroOrOne) {
                         // Try zero matches of this sequence, by skipping over it.
                         self.cur_mps.push(MatcherPos {
                             idx: idx_first_after,
diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs
index 4ebd75f01..5679cdcbb 100644
--- a/compiler/rustc_expand/src/mbe/macro_rules.rs
+++ b/compiler/rustc_expand/src/mbe/macro_rules.rs
@@ -10,13 +10,12 @@ use crate::mbe::transcribe::transcribe;
 
 use rustc_ast as ast;
 use rustc_ast::token::{self, Delimiter, NonterminalKind, Token, TokenKind, TokenKind::*};
-use rustc_ast::tokenstream::{DelimSpan, TokenStream};
+use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree};
 use rustc_ast::{NodeId, DUMMY_NODE_ID};
 use rustc_ast_pretty::pprust;
 use rustc_attr::{self as attr, TransparencyError};
 use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
 use rustc_errors::{Applicability, ErrorGuaranteed};
-use rustc_feature::Features;
 use rustc_lint_defs::builtin::{
     RUST_2021_INCOMPATIBLE_OR_PATTERNS, SEMICOLON_IN_EXPRESSIONS_FROM_MACROS,
 };
@@ -212,7 +211,6 @@ fn expand_macro<'cx>(
             };
             let arm_span = rhses[i].span();
 
-            let rhs_spans = rhs.tts.iter().map(|t| t.span()).collect::<Vec<_>>();
             // rhs has holes ( `$id` and `$(...)` that need filled)
             let mut tts = match transcribe(cx, &named_matches, &rhs, rhs_span, transparency) {
                 Ok(tts) => tts,
@@ -224,12 +222,25 @@ fn expand_macro<'cx>(
 
             // Replace all the tokens for the corresponding positions in the macro, to maintain
             // proper positions in error reporting, while maintaining the macro_backtrace.
-            if rhs_spans.len() == tts.len() {
+            if tts.len() == rhs.tts.len() {
                 tts = tts.map_enumerated(|i, tt| {
                     let mut tt = tt.clone();
-                    let mut sp = rhs_spans[i];
-                    sp = sp.with_ctxt(tt.span().ctxt());
-                    tt.set_span(sp);
+                    let rhs_tt = &rhs.tts[i];
+                    let ctxt = tt.span().ctxt();
+                    match (&mut tt, rhs_tt) {
+                        // preserve the delim spans if able
+                        (
+                            TokenTree::Delimited(target_sp, ..),
+                            mbe::TokenTree::Delimited(source_sp, ..),
+                        ) => {
+                            target_sp.open = source_sp.open.with_ctxt(ctxt);
+                            target_sp.close = source_sp.close.with_ctxt(ctxt);
+                        }
+                        _ => {
+                            let sp = rhs_tt.span().with_ctxt(ctxt);
+                            tt.set_span(sp);
+                        }
+                    }
                     tt
                 });
             }
@@ -367,7 +378,6 @@ pub(super) fn try_match_macro<'matcher, T: Tracker<'matcher>>(
 /// Converts a macro item into a syntax extension.
 pub fn compile_declarative_macro(
     sess: &Session,
-    features: &Features,
     def: &ast::Item,
     edition: Edition,
 ) -> (SyntaxExtension, Vec<(usize, Span)>) {
@@ -496,7 +506,7 @@ pub fn compile_declarative_macro(
                         true,
                         &sess.parse_sess,
                         def.id,
-                        features,
+                        sess.features_untracked(),
                         edition,
                     )
                     .pop()
@@ -520,7 +530,7 @@ pub fn compile_declarative_macro(
                         false,
                         &sess.parse_sess,
                         def.id,
-                        features,
+                        sess.features_untracked(),
                         edition,
                     )
                     .pop()
diff --git a/compiler/rustc_expand/src/mbe/metavar_expr.rs b/compiler/rustc_expand/src/mbe/metavar_expr.rs
index 99fe47454..de34df011 100644
--- a/compiler/rustc_expand/src/mbe/metavar_expr.rs
+++ b/compiler/rustc_expand/src/mbe/metavar_expr.rs
@@ -1,5 +1,5 @@
 use rustc_ast::token::{self, Delimiter};
-use rustc_ast::tokenstream::{CursorRef, TokenStream, TokenTree};
+use rustc_ast::tokenstream::{RefTokenTreeCursor, TokenStream, TokenTree};
 use rustc_ast::{LitIntType, LitKind};
 use rustc_ast_pretty::pprust;
 use rustc_errors::{Applicability, PResult};
@@ -72,7 +72,7 @@ impl MetaVarExpr {
 
 // Checks if there are any remaining tokens. For example, `${ignore(ident ... a b c ...)}`
 fn check_trailing_token<'sess>(
-    iter: &mut CursorRef<'_>,
+    iter: &mut RefTokenTreeCursor<'_>,
     sess: &'sess ParseSess,
 ) -> PResult<'sess, ()> {
     if let Some(tt) = iter.next() {
@@ -88,7 +88,7 @@ fn check_trailing_token<'sess>(
 
 /// Parse a meta-variable `count` expression: `count(ident[, depth])`
 fn parse_count<'sess>(
-    iter: &mut CursorRef<'_>,
+    iter: &mut RefTokenTreeCursor<'_>,
     sess: &'sess ParseSess,
     span: Span,
 ) -> PResult<'sess, MetaVarExpr> {
@@ -99,7 +99,7 @@ fn parse_count<'sess>(
 
 /// Parses the depth used by index(depth) and length(depth).
 fn parse_depth<'sess>(
-    iter: &mut CursorRef<'_>,
+    iter: &mut RefTokenTreeCursor<'_>,
     sess: &'sess ParseSess,
     span: Span,
 ) -> PResult<'sess, usize> {
@@ -126,7 +126,7 @@ fn parse_depth<'sess>(
 
 /// Parses an generic ident
 fn parse_ident<'sess>(
-    iter: &mut CursorRef<'_>,
+    iter: &mut RefTokenTreeCursor<'_>,
     sess: &'sess ParseSess,
     span: Span,
 ) -> PResult<'sess, Ident> {
@@ -152,7 +152,7 @@ fn parse_ident<'sess>(
 
 /// Tries to move the iterator forward returning `true` if there is a comma. If not, then the
 /// iterator is not modified and the result is `false`.
-fn try_eat_comma(iter: &mut CursorRef<'_>) -> bool {
+fn try_eat_comma(iter: &mut RefTokenTreeCursor<'_>) -> bool {
     if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. }, _)) = iter.look_ahead(0) {
         let _ = iter.next();
         return true;
diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs
index b79835be7..47a8b4bc4 100644
--- a/compiler/rustc_expand/src/mbe/transcribe.rs
+++ b/compiler/rustc_expand/src/mbe/transcribe.rs
@@ -282,7 +282,7 @@ pub(super) fn transcribe<'a>(
             }
 
             // There should be no meta-var declarations in the invocation of a macro.
-            mbe::TokenTree::MetaVarDecl(..) => panic!("unexpected `TokenTree::MetaVarDecl"),
+            mbe::TokenTree::MetaVarDecl(..) => panic!("unexpected `TokenTree::MetaVarDecl`"),
         }
     }
 }
diff --git a/compiler/rustc_expand/src/module.rs b/compiler/rustc_expand/src/module.rs
index 07f47a9c3..3779af19e 100644
--- a/compiler/rustc_expand/src/module.rs
+++ b/compiler/rustc_expand/src/module.rs
@@ -12,8 +12,8 @@ use rustc_session::Session;
 use rustc_span::symbol::{sym, Ident};
 use rustc_span::Span;
 use std::iter::once;
-
 use std::path::{self, Path, PathBuf};
+use thin_vec::ThinVec;
 
 #[derive(Copy, Clone)]
 pub enum DirOwnership {
@@ -31,7 +31,7 @@ pub struct ModulePathSuccess {
 }
 
 pub(crate) struct ParsedExternalMod {
-    pub items: Vec<P<Item>>,
+    pub items: ThinVec<P<Item>>,
     pub spans: ModSpans,
     pub file_path: PathBuf,
     pub dir_path: PathBuf,
diff --git a/compiler/rustc_expand/src/parse/tests.rs b/compiler/rustc_expand/src/parse/tests.rs
index 0726d922c..8b37728b6 100644
--- a/compiler/rustc_expand/src/parse/tests.rs
+++ b/compiler/rustc_expand/src/parse/tests.rs
@@ -18,7 +18,10 @@ use rustc_span::{BytePos, FileName, Pos, Span};
 use std::path::PathBuf;
 
 fn sess() -> ParseSess {
-    ParseSess::new(FilePathMapping::empty())
+    ParseSess::new(
+        vec![crate::DEFAULT_LOCALE_RESOURCE, rustc_parse::DEFAULT_LOCALE_RESOURCE],
+        FilePathMapping::empty(),
+    )
 }
 
 /// Parses an item.
diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs
index e9a691920..ddba14417 100644
--- a/compiler/rustc_expand/src/proc_macro.rs
+++ b/compiler/rustc_expand/src/proc_macro.rs
@@ -1,4 +1,5 @@
 use crate::base::{self, *};
+use crate::errors;
 use crate::proc_macro_server;
 
 use rustc_ast as ast;
@@ -60,11 +61,12 @@ impl base::BangProcMacro for BangProcMacro {
         let strategy = exec_strategy(ecx);
         let server = proc_macro_server::Rustc::new(ecx);
         self.client.run(&strategy, server, input, proc_macro_backtrace).map_err(|e| {
-            let mut err = ecx.struct_span_err(span, "proc macro panicked");
-            if let Some(s) = e.as_str() {
-                err.help(&format!("message: {}", s));
-            }
-            err.emit()
+            ecx.sess.emit_err(errors::ProcMacroPanicked {
+                span,
+                message: e
+                    .as_str()
+                    .map(|message| errors::ProcMacroPanickedHelp { message: message.into() }),
+            })
         })
     }
 }
@@ -174,7 +176,7 @@ impl MultiItemModifier for DeriveProcMacro {
 
         // fail if there have been errors emitted
         if ecx.sess.parse_sess.span_diagnostic.err_count() > error_count_before {
-            ecx.struct_span_err(span, "proc-macro derive produced unparseable tokens").emit();
+            ecx.sess.emit_err(errors::ProcMacroDeriveTokens { span });
         }
 
         ExpandResult::Ready(items)
diff --git a/compiler/rustc_expand/src/tests.rs b/compiler/rustc_expand/src/tests.rs
index 8f3bea29f..480d95b77 100644
--- a/compiler/rustc_expand/src/tests.rs
+++ b/compiler/rustc_expand/src/tests.rs
@@ -8,7 +8,7 @@ use rustc_span::{BytePos, Span};
 
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::emitter::EmitterWriter;
-use rustc_errors::{Handler, MultiSpan, PResult};
+use rustc_errors::{Handler, MultiSpan, PResult, TerminalUrl};
 
 use std::io;
 use std::io::prelude::*;
@@ -34,18 +34,23 @@ where
 
 /// Maps a string to tts, using a made-up filename.
 pub(crate) fn string_to_stream(source_str: String) -> TokenStream {
-    let ps = ParseSess::new(FilePathMapping::empty());
+    let ps = ParseSess::new(
+        vec![crate::DEFAULT_LOCALE_RESOURCE, rustc_parse::DEFAULT_LOCALE_RESOURCE],
+        FilePathMapping::empty(),
+    );
     source_file_to_stream(
         &ps,
         ps.source_map().new_source_file(PathBuf::from("bogofile").into(), source_str),
         None,
     )
-    .0
 }
 
 /// Parses a string, returns a crate.
 pub(crate) fn string_to_crate(source_str: String) -> ast::Crate {
-    let ps = ParseSess::new(FilePathMapping::empty());
+    let ps = ParseSess::new(
+        vec![crate::DEFAULT_LOCALE_RESOURCE, rustc_parse::DEFAULT_LOCALE_RESOURCE],
+        FilePathMapping::empty(),
+    );
     with_error_checking_parse(source_str, &ps, |p| p.parse_crate_mod())
 }
 
@@ -127,8 +132,10 @@ fn test_harness(file_text: &str, span_labels: Vec<SpanLabel>, expected_output: &
     create_default_session_if_not_set_then(|_| {
         let output = Arc::new(Mutex::new(Vec::new()));
 
-        let fallback_bundle =
-            rustc_errors::fallback_fluent_bundle(rustc_errors::DEFAULT_LOCALE_RESOURCES, false);
+        let fallback_bundle = rustc_errors::fallback_fluent_bundle(
+            vec![crate::DEFAULT_LOCALE_RESOURCE, rustc_parse::DEFAULT_LOCALE_RESOURCE],
+            false,
+        );
         let source_map = Lrc::new(SourceMap::new(FilePathMapping::empty()));
         source_map.new_source_file(Path::new("test.rs").to_owned().into(), file_text.to_owned());
 
@@ -152,6 +159,7 @@ fn test_harness(file_text: &str, span_labels: Vec<SpanLabel>, expected_output: &
             None,
             false,
             false,
+            TerminalUrl::No,
         );
         let handler = Handler::with_emitter(true, None, Box::new(emitter));
         #[allow(rustc::untranslatable_diagnostic)]
diff --git a/compiler/rustc_feature/src/accepted.rs b/compiler/rustc_feature/src/accepted.rs
index e064e87a5..d304d38e1 100644
--- a/compiler/rustc_feature/src/accepted.rs
+++ b/compiler/rustc_feature/src/accepted.rs
@@ -90,6 +90,8 @@ declare_features! (
     (accepted, clone_closures, "1.26.0", Some(44490), None),
     /// Allows coercing non capturing closures to function pointers.
     (accepted, closure_to_fn_coercion, "1.19.0", Some(39817), None),
+    /// Allows using `cmpxchg16b` from `core::arch::x86_64`.
+    (accepted, cmpxchg16b_target_feature, "1.69.0", Some(44839), None),
     /// Allows usage of the `compile_error!` macro.
     (accepted, compile_error, "1.20.0", Some(40872), None),
     /// Allows `impl Trait` in function return types.
diff --git a/compiler/rustc_feature/src/active.rs b/compiler/rustc_feature/src/active.rs
index b5256043e..baea5968e 100644
--- a/compiler/rustc_feature/src/active.rs
+++ b/compiler/rustc_feature/src/active.rs
@@ -160,6 +160,8 @@ declare_features! (
     (active, intrinsics, "1.0.0", None, None),
     /// Allows using `#[lang = ".."]` attribute for linking items to special compiler logic.
     (active, lang_items, "1.0.0", None, None),
+    /// Allows the `multiple_supertrait_upcastable` lint.
+    (active, multiple_supertrait_upcastable, "1.69.0", None, None),
     /// Allows using `#[omit_gdb_pretty_printer_section]`.
     (active, omit_gdb_pretty_printer_section, "1.5.0", None, None),
     /// Allows using `#[prelude_import]` on glob `use` items.
@@ -213,6 +215,8 @@ declare_features! (
     (active, linkage, "1.0.0", Some(29603), None),
     /// Allows declaring with `#![needs_panic_runtime]` that a panic runtime is needed.
     (active, needs_panic_runtime, "1.10.0", Some(32837), None),
+    /// Allows using `+bundled,+whole-archive` native libs.
+    (active, packed_bundled_libs, "1.69.0", Some(108081), None),
     /// Allows using the `#![panic_runtime]` attribute.
     (active, panic_runtime, "1.10.0", Some(32837), None),
     /// Allows using `#[rustc_allow_const_fn_unstable]`.
@@ -252,7 +256,6 @@ declare_features! (
     (active, arm_target_feature, "1.27.0", Some(44839), None),
     (active, avx512_target_feature, "1.27.0", Some(44839), None),
     (active, bpf_target_feature, "1.54.0", Some(44839), None),
-    (active, cmpxchg16b_target_feature, "1.32.0", Some(44839), None),
     (active, ermsb_target_feature, "1.49.0", Some(44839), None),
     (active, hexagon_target_feature, "1.27.0", Some(44839), None),
     (active, mips_target_feature, "1.27.0", Some(44839), None),
@@ -313,8 +316,6 @@ declare_features! (
     (active, c_unwind, "1.52.0", Some(74990), None),
     /// Allows using C-variadics.
     (active, c_variadic, "1.34.0", Some(44930), None),
-    /// Allows capturing disjoint fields in a closure/generator (RFC 2229).
-    (incomplete, capture_disjoint_fields, "1.49.0", Some(53488), None),
     /// Allows the use of `#[cfg(sanitize = "option")]`; set when -Zsanitizer is used.
     (active, cfg_sanitize, "1.41.0", Some(39699), None),
     /// Allows `cfg(target_abi = "...")`.
@@ -469,6 +470,8 @@ declare_features! (
     (active, no_sanitize, "1.42.0", Some(39699), None),
     /// Allows using the `non_exhaustive_omitted_patterns` lint.
     (active, non_exhaustive_omitted_patterns_lint, "1.57.0", Some(89554), None),
+    /// Allows `for<T>` binders in where-clauses
+    (incomplete, non_lifetime_binders, "1.69.0", Some(108185), None),
     /// Allows making `dyn Trait` well-formed even if `Trait` is not object safe.
     /// In that case, `dyn Trait: Trait` does not hold. Moreover, coercions and
     /// casts in safe Rust to `dyn Trait` for such a `Trait` is also forbidden.
diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs
index af56a0b24..493a9cd89 100644
--- a/compiler/rustc_feature/src/builtin_attrs.rs
+++ b/compiler/rustc_feature/src/builtin_attrs.rs
@@ -414,7 +414,10 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
     ),
 
     // Linking:
-    gated!(naked, Normal, template!(Word), WarnFollowing, @only_local: true, naked_functions, experimental!(naked)),
+    gated!(
+        naked, Normal, template!(Word), WarnFollowing, @only_local: true,
+        naked_functions, experimental!(naked)
+    ),
 
     // Plugins:
     BuiltinAttribute {
@@ -441,7 +444,8 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
     ),
     // RFC #1268
     gated!(
-        marker, Normal, template!(Word), WarnFollowing, marker_trait_attr, experimental!(marker)
+        marker, Normal, template!(Word), WarnFollowing, @only_local: true,
+        marker_trait_attr, experimental!(marker)
     ),
     gated!(
         thread_local, Normal, template!(Word), WarnFollowing,
@@ -682,8 +686,7 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         "language items are subject to change",
     ),
     rustc_attr!(
-        rustc_pass_by_value, Normal,
-        template!(Word), ErrorFollowing,
+        rustc_pass_by_value, Normal, template!(Word), ErrorFollowing,
         "#[rustc_pass_by_value] is used to mark types that must be passed by value instead of reference."
     ),
     rustc_attr!(
@@ -691,6 +694,10 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         "#![rustc_coherence_is_core] allows inherent methods on builtin types, only intended to be used in `core`."
     ),
     rustc_attr!(
+        rustc_coinductive, AttributeType::Normal, template!(Word), WarnFollowing, @only_local: true,
+        "#![rustc_coinductive] changes a trait to be coinductive, allowing cycles in the trait solver."
+    ),
+    rustc_attr!(
         rustc_allow_incoherent_impl, AttributeType::Normal, template!(Word), ErrorFollowing, @only_local: true,
         "#[rustc_allow_incoherent_impl] has to be added to all impl items of an incoherent inherent impl."
     ),
diff --git a/compiler/rustc_feature/src/lib.rs b/compiler/rustc_feature/src/lib.rs
index 8e2a13a6c..93d167163 100644
--- a/compiler/rustc_feature/src/lib.rs
+++ b/compiler/rustc_feature/src/lib.rs
@@ -83,7 +83,8 @@ impl UnstableFeatures {
     /// Otherwise, only `RUSTC_BOOTSTRAP=1` will work.
     pub fn from_environment(krate: Option<&str>) -> Self {
         // `true` if this is a feature-staged build, i.e., on the beta or stable channel.
-        let disable_unstable_features = option_env!("CFG_DISABLE_UNSTABLE_FEATURES").is_some();
+        let disable_unstable_features =
+            option_env!("CFG_DISABLE_UNSTABLE_FEATURES").map(|s| s != "0").unwrap_or(false);
         // Returns whether `krate` should be counted as unstable
         let is_unstable_crate = |var: &str| {
             krate.map_or(false, |name| var.split(',').any(|new_krate| new_krate == name))
diff --git a/compiler/rustc_feature/src/removed.rs b/compiler/rustc_feature/src/removed.rs
index 79a12801d..04d4f6cb1 100644
--- a/compiler/rustc_feature/src/removed.rs
+++ b/compiler/rustc_feature/src/removed.rs
@@ -52,6 +52,8 @@ declare_features! (
     (removed, allow_fail, "1.19.0", Some(46488), None, Some("removed due to no clear use cases")),
     (removed, await_macro, "1.38.0", Some(50547), None,
      Some("subsumed by `.await` syntax")),
+    /// Allows capturing disjoint fields in a closure/generator (RFC 2229).
+    (removed, capture_disjoint_fields, "1.49.0", Some(53488), None, Some("stabilized in Rust 2021")),
     /// Allows comparing raw pointers during const eval.
     (removed, const_compare_raw_pointers, "1.46.0", Some(53020), None,
      Some("cannot be allowed in const eval in any meaningful way")),
diff --git a/compiler/rustc_hir/src/def.rs b/compiler/rustc_hir/src/def.rs
index cca5ead0f..8c58129c8 100644
--- a/compiler/rustc_hir/src/def.rs
+++ b/compiler/rustc_hir/src/def.rs
@@ -2,6 +2,8 @@ use crate::hir;
 
 use rustc_ast as ast;
 use rustc_ast::NodeId;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::stable_hasher::ToStableHashKey;
 use rustc_macros::HashStable_Generic;
 use rustc_span::def_id::{DefId, LocalDefId};
 use rustc_span::hygiene::MacroKind;
@@ -114,12 +116,19 @@ pub enum DefKind {
     LifetimeParam,
     /// A use of `global_asm!`.
     GlobalAsm,
-    Impl,
+    Impl {
+        of_trait: bool,
+    },
     Closure,
     Generator,
 }
 
 impl DefKind {
+    /// Get an English description for the item's kind.
+    ///
+    /// If you have access to `TyCtxt`, use `TyCtxt::def_descr` or
+    /// `TyCtxt::def_kind_descr` instead, because they give better
+    /// information for generators and associated functions.
     pub fn descr(self, def_id: DefId) -> &'static str {
         match self {
             DefKind::Fn => "function",
@@ -153,7 +162,7 @@ impl DefKind {
             DefKind::AnonConst => "constant expression",
             DefKind::InlineConst => "inline constant",
             DefKind::Field => "field",
-            DefKind::Impl => "implementation",
+            DefKind::Impl { .. } => "implementation",
             DefKind::Closure => "closure",
             DefKind::Generator => "generator",
             DefKind::ExternCrate => "extern crate",
@@ -162,6 +171,10 @@ impl DefKind {
     }
 
     /// Gets an English article for the definition.
+    ///
+    /// If you have access to `TyCtxt`, use `TyCtxt::def_descr_article` or
+    /// `TyCtxt::def_kind_descr_article` instead, because they give better
+    /// information for generators and associated functions.
     pub fn article(&self) -> &'static str {
         match *self {
             DefKind::AssocTy
@@ -169,7 +182,7 @@ impl DefKind {
             | DefKind::AssocFn
             | DefKind::Enum
             | DefKind::OpaqueTy
-            | DefKind::Impl
+            | DefKind::Impl { .. }
             | DefKind::Use
             | DefKind::InlineConst
             | DefKind::ExternCrate => "an",
@@ -214,7 +227,7 @@ impl DefKind {
             | DefKind::Use
             | DefKind::ForeignMod
             | DefKind::GlobalAsm
-            | DefKind::Impl
+            | DefKind::Impl { .. }
             | DefKind::ImplTraitPlaceholder => None,
         }
     }
@@ -253,7 +266,7 @@ impl DefKind {
             | DefKind::ForeignMod
             | DefKind::OpaqueTy
             | DefKind::ImplTraitPlaceholder
-            | DefKind::Impl
+            | DefKind::Impl { .. }
             | DefKind::Field
             | DefKind::TyParam
             | DefKind::ConstParam
@@ -472,7 +485,8 @@ impl PartialRes {
 
 /// Different kinds of symbols can coexist even if they share the same textual name.
 /// Therefore, they each have a separate universe (known as a "namespace").
-#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
+#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Encodable, Decodable)]
+#[derive(HashStable_Generic)]
 pub enum Namespace {
     /// The type namespace includes `struct`s, `enum`s, `union`s, `trait`s, and `mod`s
     /// (and, by extension, crates).
@@ -499,6 +513,15 @@ impl Namespace {
     }
 }
 
+impl<CTX: crate::HashStableContext> ToStableHashKey<CTX> for Namespace {
+    type KeyType = Namespace;
+
+    #[inline]
+    fn to_stable_hash_key(&self, _: &CTX) -> Namespace {
+        *self
+    }
+}
+
 /// Just a helper ‒ separate structure for each namespace.
 #[derive(Copy, Clone, Default, Debug)]
 pub struct PerNS<T> {
@@ -760,3 +783,5 @@ pub enum LifetimeRes {
     /// HACK: This is used to recover the NodeId of an elided lifetime.
     ElidedAnchor { start: NodeId, end: NodeId },
 }
+
+pub type DocLinkResMap = FxHashMap<(Symbol, Namespace), Option<Res<NodeId>>>;
diff --git a/compiler/rustc_hir/src/definitions.rs b/compiler/rustc_hir/src/definitions.rs
index 21cf214e4..8ceb17649 100644
--- a/compiler/rustc_hir/src/definitions.rs
+++ b/compiler/rustc_hir/src/definitions.rs
@@ -92,7 +92,7 @@ impl DefPathTable {
 /// The definition table containing node definitions.
 /// It holds the `DefPathTable` for `LocalDefId`s/`DefPath`s.
 /// It also stores mappings to convert `LocalDefId`s to/from `HirId`s.
-#[derive(Clone, Debug)]
+#[derive(Debug)]
 pub struct Definitions {
     table: DefPathTable,
     next_disambiguator: FxHashMap<(LocalDefId, DefPathData), u32>,
@@ -280,6 +280,8 @@ pub enum DefPathData {
     AnonConst,
     /// An `impl Trait` type node.
     ImplTrait,
+    /// `impl Trait` generated associated type node.
+    ImplTraitAssocTy,
 }
 
 impl Definitions {
@@ -403,7 +405,7 @@ impl DefPathData {
             TypeNs(name) | ValueNs(name) | MacroNs(name) | LifetimeNs(name) => Some(name),
 
             Impl | ForeignMod | CrateRoot | Use | GlobalAsm | ClosureExpr | Ctor | AnonConst
-            | ImplTrait => None,
+            | ImplTrait | ImplTraitAssocTy => None,
         }
     }
 
@@ -422,7 +424,7 @@ impl DefPathData {
             ClosureExpr => DefPathDataName::Anon { namespace: sym::closure },
             Ctor => DefPathDataName::Anon { namespace: sym::constructor },
             AnonConst => DefPathDataName::Anon { namespace: sym::constant },
-            ImplTrait => DefPathDataName::Anon { namespace: sym::opaque },
+            ImplTrait | ImplTraitAssocTy => DefPathDataName::Anon { namespace: sym::opaque },
         }
     }
 }
diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs
index d6566860f..19d3d41c9 100644
--- a/compiler/rustc_hir/src/hir.rs
+++ b/compiler/rustc_hir/src/hir.rs
@@ -369,10 +369,10 @@ impl<'hir> GenericArgs<'hir> {
 
     pub fn has_err(&self) -> bool {
         self.args.iter().any(|arg| match arg {
-            GenericArg::Type(ty) => matches!(ty.kind, TyKind::Err),
+            GenericArg::Type(ty) => matches!(ty.kind, TyKind::Err(_)),
             _ => false,
         }) || self.bindings.iter().any(|arg| match arg.kind {
-            TypeBindingKind::Equality { term: Term::Ty(ty) } => matches!(ty.kind, TyKind::Err),
+            TypeBindingKind::Equality { term: Term::Ty(ty) } => matches!(ty.kind, TyKind::Err(_)),
             _ => false,
         })
     }
@@ -498,6 +498,7 @@ pub struct GenericParam<'hir> {
     pub pure_wrt_drop: bool,
     pub kind: GenericParamKind<'hir>,
     pub colon_span: Option<Span>,
+    pub source: GenericParamSource,
 }
 
 impl<'hir> GenericParam<'hir> {
@@ -516,6 +517,20 @@ impl<'hir> GenericParam<'hir> {
     }
 }
 
+/// Records where the generic parameter originated from.
+///
+/// This can either be from an item's generics, in which case it's typically
+/// early-bound (but can be a late-bound lifetime in functions, for example),
+/// or from a `for<...>` binder, in which case it's late-bound (and notably,
+/// does not show up in the parent item's generics).
+#[derive(Debug, HashStable_Generic, PartialEq, Eq, Copy, Clone)]
+pub enum GenericParamSource {
+    // Early or late-bound parameters defined on an item
+    Generics,
+    // Late-bound parameters defined via a `for<...>`
+    Binder,
+}
+
 #[derive(Default)]
 pub struct GenericParamCount {
     pub lifetimes: usize,
@@ -574,14 +589,11 @@ impl<'hir> Generics<'hir> {
 
     /// If there are generic parameters, return where to introduce a new one.
     pub fn span_for_param_suggestion(&self) -> Option<Span> {
-        if self.params.iter().any(|p| self.span.contains(p.span)) {
+        self.params.iter().any(|p| self.span.contains(p.span)).then(|| {
             // `fn foo<A>(t: impl Trait)`
             //          ^ suggest `, T: Trait` here
-            let span = self.span.with_lo(self.span.hi() - BytePos(1)).shrink_to_lo();
-            Some(span)
-        } else {
-            None
-        }
+            self.span.with_lo(self.span.hi() - BytePos(1)).shrink_to_lo()
+        })
     }
 
     /// `Span` where further predicates would be suggested, accounting for trailing commas, like
@@ -639,7 +651,7 @@ impl<'hir> Generics<'hir> {
                 // We include bounds that come from a `#[derive(_)]` but point at the user's code,
                 // as we use this method to get a span appropriate for suggestions.
                 let bs = bound.span();
-                if bs.can_be_used_for_suggestions() { Some(bs.shrink_to_hi()) } else { None }
+                bs.can_be_used_for_suggestions().then(|| bs.shrink_to_hi())
             },
         )
     }
@@ -831,8 +843,6 @@ pub struct OwnerNodes<'tcx> {
     pub nodes: IndexVec<ItemLocalId, Option<ParentedNode<'tcx>>>,
     /// Content of local bodies.
     pub bodies: SortedMap<ItemLocalId, &'tcx Body<'tcx>>,
-    /// Non-owning definitions contained in this owner.
-    pub local_id_to_def_id: SortedMap<ItemLocalId, LocalDefId>,
 }
 
 impl<'tcx> OwnerNodes<'tcx> {
@@ -862,7 +872,6 @@ impl fmt::Debug for OwnerNodes<'_> {
                     .collect::<Vec<_>>(),
             )
             .field("bodies", &self.bodies)
-            .field("local_id_to_def_id", &self.local_id_to_def_id)
             .field("hash_without_bodies", &self.hash_without_bodies)
             .field("hash_including_bodies", &self.hash_including_bodies)
             .finish()
@@ -993,7 +1002,6 @@ pub struct Pat<'hir> {
 }
 
 impl<'hir> Pat<'hir> {
-    // FIXME(#19596) this is a workaround, but there should be a better way
     fn walk_short_(&self, it: &mut impl FnMut(&Pat<'hir>) -> bool) -> bool {
         if !it(self) {
             return false;
@@ -1021,7 +1029,6 @@ impl<'hir> Pat<'hir> {
         self.walk_short_(&mut it)
     }
 
-    // FIXME(#19596) this is a workaround, but there should be a better way
     fn walk_(&self, it: &mut impl FnMut(&Pat<'hir>) -> bool) {
         if !it(self) {
             return;
@@ -1696,7 +1703,7 @@ impl Expr<'_> {
             ExprKind::Struct(..) => ExprPrecedence::Struct,
             ExprKind::Repeat(..) => ExprPrecedence::Repeat,
             ExprKind::Yield(..) => ExprPrecedence::Yield,
-            ExprKind::Err => ExprPrecedence::Err,
+            ExprKind::Err(_) => ExprPrecedence::Err,
         }
     }
 
@@ -1762,7 +1769,7 @@ impl Expr<'_> {
             | ExprKind::Yield(..)
             | ExprKind::Cast(..)
             | ExprKind::DropTemps(..)
-            | ExprKind::Err => false,
+            | ExprKind::Err(_) => false,
         }
     }
 
@@ -1848,7 +1855,7 @@ impl Expr<'_> {
             | ExprKind::Binary(..)
             | ExprKind::Yield(..)
             | ExprKind::DropTemps(..)
-            | ExprKind::Err => true,
+            | ExprKind::Err(_) => true,
         }
     }
 
@@ -2021,7 +2028,7 @@ pub enum ExprKind<'hir> {
     Yield(&'hir Expr<'hir>, YieldSource),
 
     /// A placeholder for an expression that wasn't syntactically well formed in some way.
-    Err,
+    Err(rustc_span::ErrorGuaranteed),
 }
 
 /// Represents an optionally `Self`-qualified value/type path or associated extension.
@@ -2106,8 +2113,8 @@ pub enum LocalSource {
 }
 
 /// Hints at the original code for a `match _ { .. }`.
-#[derive(Copy, Clone, PartialEq, Eq, Encodable, Hash, Debug)]
-#[derive(HashStable_Generic)]
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
+#[derive(HashStable_Generic, Encodable, Decodable)]
 pub enum MatchSource {
     /// A `match _ { .. }`.
     Normal,
@@ -2117,6 +2124,8 @@ pub enum MatchSource {
     TryDesugar,
     /// A desugared `<expr>.await`.
     AwaitDesugar,
+    /// A desugared `format_args!()`.
+    FormatArgs,
 }
 
 impl MatchSource {
@@ -2128,6 +2137,7 @@ impl MatchSource {
             ForLoopDesugar => "for",
             TryDesugar => "?",
             AwaitDesugar => ".await",
+            FormatArgs => "format_args!()",
         }
     }
 }
@@ -2263,7 +2273,7 @@ pub struct TraitItem<'hir> {
     pub defaultness: Defaultness,
 }
 
-impl TraitItem<'_> {
+impl<'hir> TraitItem<'hir> {
     #[inline]
     pub fn hir_id(&self) -> HirId {
         // Items are always HIR owners.
@@ -2273,6 +2283,32 @@ impl TraitItem<'_> {
     pub fn trait_item_id(&self) -> TraitItemId {
         TraitItemId { owner_id: self.owner_id }
     }
+
+    /// Expect an [`TraitItemKind::Const`] or panic.
+    #[track_caller]
+    pub fn expect_const(&self) -> (&'hir Ty<'hir>, Option<BodyId>) {
+        let TraitItemKind::Const(ty, body) = self.kind else { self.expect_failed("a constant") };
+        (ty, body)
+    }
+
+    /// Expect an [`TraitItemKind::Fn`] or panic.
+    #[track_caller]
+    pub fn expect_fn(&self) -> (&FnSig<'hir>, &TraitFn<'hir>) {
+        let TraitItemKind::Fn(ty, trfn) = &self.kind else { self.expect_failed("a function") };
+        (ty, trfn)
+    }
+
+    /// Expect an [`TraitItemKind::Type`] or panic.
+    #[track_caller]
+    pub fn expect_type(&self) -> (GenericBounds<'hir>, Option<&'hir Ty<'hir>>) {
+        let TraitItemKind::Type(bounds, ty) = self.kind else { self.expect_failed("a type") };
+        (bounds, ty)
+    }
+
+    #[track_caller]
+    fn expect_failed(&self, expected: &'static str) -> ! {
+        panic!("expected {expected} item, found {self:?}")
+    }
 }
 
 /// Represents a trait method's body (or just argument names).
@@ -2325,7 +2361,7 @@ pub struct ImplItem<'hir> {
     pub vis_span: Span,
 }
 
-impl ImplItem<'_> {
+impl<'hir> ImplItem<'hir> {
     #[inline]
     pub fn hir_id(&self) -> HirId {
         // Items are always HIR owners.
@@ -2335,6 +2371,32 @@ impl ImplItem<'_> {
     pub fn impl_item_id(&self) -> ImplItemId {
         ImplItemId { owner_id: self.owner_id }
     }
+
+    /// Expect an [`ImplItemKind::Const`] or panic.
+    #[track_caller]
+    pub fn expect_const(&self) -> (&'hir Ty<'hir>, BodyId) {
+        let ImplItemKind::Const(ty, body) = self.kind else { self.expect_failed("a constant") };
+        (ty, body)
+    }
+
+    /// Expect an [`ImplItemKind::Fn`] or panic.
+    #[track_caller]
+    pub fn expect_fn(&self) -> (&FnSig<'hir>, BodyId) {
+        let ImplItemKind::Fn(ty, body) = &self.kind else { self.expect_failed("a function") };
+        (ty, *body)
+    }
+
+    /// Expect an [`ImplItemKind::Type`] or panic.
+    #[track_caller]
+    pub fn expect_type(&self) -> &'hir Ty<'hir> {
+        let ImplItemKind::Type(ty) = self.kind else { self.expect_failed("a type") };
+        ty
+    }
+
+    #[track_caller]
+    fn expect_failed(&self, expected: &'static str) -> ! {
+        panic!("expected {expected} item, found {self:?}")
+    }
 }
 
 /// Represents various kinds of content within an `impl`.
@@ -2629,7 +2691,7 @@ pub enum TyKind<'hir> {
     /// specified. This can appear anywhere in a type.
     Infer,
     /// Placeholder for a type that has failed to be defined.
-    Err,
+    Err(rustc_span::ErrorGuaranteed),
 }
 
 #[derive(Debug, HashStable_Generic)]
@@ -2995,7 +3057,7 @@ pub struct Item<'hir> {
     pub vis_span: Span,
 }
 
-impl Item<'_> {
+impl<'hir> Item<'hir> {
     #[inline]
     pub fn hir_id(&self) -> HirId {
         // Items are always HIR owners.
@@ -3005,6 +3067,132 @@ impl Item<'_> {
     pub fn item_id(&self) -> ItemId {
         ItemId { owner_id: self.owner_id }
     }
+
+    /// Expect an [`ItemKind::ExternCrate`] or panic.
+    #[track_caller]
+    pub fn expect_extern_crate(&self) -> Option<Symbol> {
+        let ItemKind::ExternCrate(s) = self.kind else { self.expect_failed("an extern crate") };
+        s
+    }
+
+    /// Expect an [`ItemKind::Use`] or panic.
+    #[track_caller]
+    pub fn expect_use(&self) -> (&'hir UsePath<'hir>, UseKind) {
+        let ItemKind::Use(p, uk) = self.kind else { self.expect_failed("a use") };
+        (p, uk)
+    }
+
+    /// Expect an [`ItemKind::Static`] or panic.
+    #[track_caller]
+    pub fn expect_static(&self) -> (&'hir Ty<'hir>, Mutability, BodyId) {
+        let ItemKind::Static(ty, mutbl, body) = self.kind else { self.expect_failed("a static") };
+        (ty, mutbl, body)
+    }
+    /// Expect an [`ItemKind::Const`] or panic.
+    #[track_caller]
+    pub fn expect_const(&self) -> (&'hir Ty<'hir>, BodyId) {
+        let ItemKind::Const(ty, body) = self.kind else { self.expect_failed("a constant") };
+        (ty, body)
+    }
+    /// Expect an [`ItemKind::Fn`] or panic.
+    #[track_caller]
+    pub fn expect_fn(&self) -> (&FnSig<'hir>, &'hir Generics<'hir>, BodyId) {
+        let ItemKind::Fn(sig, gen, body) = &self.kind else { self.expect_failed("a function") };
+        (sig, gen, *body)
+    }
+
+    /// Expect an [`ItemKind::Macro`] or panic.
+    #[track_caller]
+    pub fn expect_macro(&self) -> (&ast::MacroDef, MacroKind) {
+        let ItemKind::Macro(def, mk) = &self.kind else { self.expect_failed("a macro") };
+        (def, *mk)
+    }
+
+    /// Expect an [`ItemKind::Mod`] or panic.
+    #[track_caller]
+    pub fn expect_mod(&self) -> &'hir Mod<'hir> {
+        let ItemKind::Mod(m) = self.kind else { self.expect_failed("a module") };
+        m
+    }
+
+    /// Expect an [`ItemKind::ForeignMod`] or panic.
+    #[track_caller]
+    pub fn expect_foreign_mod(&self) -> (Abi, &'hir [ForeignItemRef]) {
+        let ItemKind::ForeignMod { abi, items } = self.kind else { self.expect_failed("a foreign module") };
+        (abi, items)
+    }
+
+    /// Expect an [`ItemKind::GlobalAsm`] or panic.
+    #[track_caller]
+    pub fn expect_global_asm(&self) -> &'hir InlineAsm<'hir> {
+        let ItemKind::GlobalAsm(asm) = self.kind else { self.expect_failed("a global asm") };
+        asm
+    }
+
+    /// Expect an [`ItemKind::TyAlias`] or panic.
+    #[track_caller]
+    pub fn expect_ty_alias(&self) -> (&'hir Ty<'hir>, &'hir Generics<'hir>) {
+        let ItemKind::TyAlias(ty, gen) = self.kind else { self.expect_failed("a type alias") };
+        (ty, gen)
+    }
+
+    /// An opaque `impl Trait` type alias, e.g., `type Foo = impl Bar;`.
+    /// Expect an [`ItemKind::OpaqueTy`] or panic.
+    #[track_caller]
+    pub fn expect_opaque_ty(&self) -> &OpaqueTy<'hir> {
+        let ItemKind::OpaqueTy(ty) = &self.kind else { self.expect_failed("an opaque type") };
+        ty
+    }
+
+    /// Expect an [`ItemKind::Enum`] or panic.
+    #[track_caller]
+    pub fn expect_enum(&self) -> (&EnumDef<'hir>, &'hir Generics<'hir>) {
+        let ItemKind::Enum(def, gen) = &self.kind else { self.expect_failed("an enum") };
+        (def, gen)
+    }
+
+    /// Expect an [`ItemKind::Struct`] or panic.
+    #[track_caller]
+    pub fn expect_struct(&self) -> (&VariantData<'hir>, &'hir Generics<'hir>) {
+        let ItemKind::Struct(data, gen) = &self.kind else { self.expect_failed("a struct") };
+        (data, gen)
+    }
+
+    /// A union definition, e.g., `union Foo<A, B> {x: A, y: B}`.
+    /// Expect an [`ItemKind::Union`] or panic.
+    #[track_caller]
+    pub fn expect_union(&self) -> (&VariantData<'hir>, &'hir Generics<'hir>) {
+        let ItemKind::Union(data, gen) = &self.kind else { self.expect_failed("a union") };
+        (data, gen)
+    }
+
+    /// Expect an [`ItemKind::Trait`] or panic.
+    #[track_caller]
+    pub fn expect_trait(
+        self,
+    ) -> (IsAuto, Unsafety, &'hir Generics<'hir>, GenericBounds<'hir>, &'hir [TraitItemRef]) {
+        let ItemKind::Trait(is_auto, unsafety, gen, bounds, items) = self.kind else { self.expect_failed("a trait") };
+        (is_auto, unsafety, gen, bounds, items)
+    }
+
+    /// Expect an [`ItemKind::TraitAlias`] or panic.
+    #[track_caller]
+    pub fn expect_trait_alias(&self) -> (&'hir Generics<'hir>, GenericBounds<'hir>) {
+        let ItemKind::TraitAlias(gen, bounds) = self.kind else { self.expect_failed("a trait alias") };
+        (gen, bounds)
+    }
+
+    /// Expect an [`ItemKind::Impl`] or panic.
+    #[track_caller]
+    pub fn expect_impl(&self) -> &'hir Impl<'hir> {
+        let ItemKind::Impl(imp) = self.kind else { self.expect_failed("an impl") };
+        imp
+    }
+
+    #[track_caller]
+    fn expect_failed(&self, expected: &'static str) -> ! {
+        panic!("expected {expected} item, found {self:?}")
+    }
 }
 
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
@@ -3282,7 +3470,7 @@ pub struct Upvar {
 // The TraitCandidate's import_ids is empty if the trait is defined in the same module, and
 // has length > 0 if the trait is found through an chain of imports, starting with the
 // import/use statement in the scope where the trait is used.
-#[derive(Encodable, Decodable, Clone, Debug, HashStable_Generic)]
+#[derive(Encodable, Decodable, Debug, HashStable_Generic)]
 pub struct TraitCandidate {
     pub def_id: DefId,
     pub import_ids: SmallVec<[LocalDefId; 1]>,
@@ -3524,6 +3712,13 @@ impl<'hir> Node<'hir> {
         }
     }
 
+    pub fn alias_ty(self) -> Option<&'hir Ty<'hir>> {
+        match self {
+            Node::Item(Item { kind: ItemKind::TyAlias(ty, ..), .. }) => Some(ty),
+            _ => None,
+        }
+    }
+
     pub fn body_id(&self) -> Option<BodyId> {
         match self {
             Node::TraitItem(TraitItem {
@@ -3590,6 +3785,185 @@ impl<'hir> Node<'hir> {
     pub fn tuple_fields(&self) -> Option<&'hir [FieldDef<'hir>]> {
         if let Node::Ctor(&VariantData::Tuple(fields, _, _)) = self { Some(fields) } else { None }
     }
+
+    /// Expect a [`Node::Param`] or panic.
+    #[track_caller]
+    pub fn expect_param(self) -> &'hir Param<'hir> {
+        let Node::Param(this) = self else { self.expect_failed("a parameter") };
+        this
+    }
+
+    /// Expect a [`Node::Item`] or panic.
+    #[track_caller]
+    pub fn expect_item(self) -> &'hir Item<'hir> {
+        let Node::Item(this) = self else { self.expect_failed("a item") };
+        this
+    }
+
+    /// Expect a [`Node::ForeignItem`] or panic.
+    #[track_caller]
+    pub fn expect_foreign_item(self) -> &'hir ForeignItem<'hir> {
+        let Node::ForeignItem(this) = self else { self.expect_failed("a foreign item") };
+        this
+    }
+
+    /// Expect a [`Node::TraitItem`] or panic.
+    #[track_caller]
+    pub fn expect_trait_item(self) -> &'hir TraitItem<'hir> {
+        let Node::TraitItem(this) = self else { self.expect_failed("a trait item") };
+        this
+    }
+
+    /// Expect a [`Node::ImplItem`] or panic.
+    #[track_caller]
+    pub fn expect_impl_item(self) -> &'hir ImplItem<'hir> {
+        let Node::ImplItem(this) = self else { self.expect_failed("an implementation item") };
+        this
+    }
+
+    /// Expect a [`Node::Variant`] or panic.
+    #[track_caller]
+    pub fn expect_variant(self) -> &'hir Variant<'hir> {
+        let Node::Variant(this) = self else { self.expect_failed("a variant") };
+        this
+    }
+
+    /// Expect a [`Node::Field`] or panic.
+    #[track_caller]
+    pub fn expect_field(self) -> &'hir FieldDef<'hir> {
+        let Node::Field(this) = self else { self.expect_failed("a field definition") };
+        this
+    }
+
+    /// Expect a [`Node::AnonConst`] or panic.
+    #[track_caller]
+    pub fn expect_anon_const(self) -> &'hir AnonConst {
+        let Node::AnonConst(this) = self else { self.expect_failed("an anonymous constant") };
+        this
+    }
+
+    /// Expect a [`Node::Expr`] or panic.
+    #[track_caller]
+    pub fn expect_expr(self) -> &'hir Expr<'hir> {
+        let Node::Expr(this) = self else { self.expect_failed("an expression") };
+        this
+    }
+    /// Expect a [`Node::ExprField`] or panic.
+    #[track_caller]
+    pub fn expect_expr_field(self) -> &'hir ExprField<'hir> {
+        let Node::ExprField(this) = self else { self.expect_failed("an expression field") };
+        this
+    }
+
+    /// Expect a [`Node::Stmt`] or panic.
+    #[track_caller]
+    pub fn expect_stmt(self) -> &'hir Stmt<'hir> {
+        let Node::Stmt(this) = self else { self.expect_failed("a statement") };
+        this
+    }
+
+    /// Expect a [`Node::PathSegment`] or panic.
+    #[track_caller]
+    pub fn expect_path_segment(self) -> &'hir PathSegment<'hir> {
+        let Node::PathSegment(this) = self else { self.expect_failed("a path segment") };
+        this
+    }
+
+    /// Expect a [`Node::Ty`] or panic.
+    #[track_caller]
+    pub fn expect_ty(self) -> &'hir Ty<'hir> {
+        let Node::Ty(this) = self else { self.expect_failed("a type") };
+        this
+    }
+
+    /// Expect a [`Node::TypeBinding`] or panic.
+    #[track_caller]
+    pub fn expect_type_binding(self) -> &'hir TypeBinding<'hir> {
+        let Node::TypeBinding(this) = self else { self.expect_failed("a type binding") };
+        this
+    }
+
+    /// Expect a [`Node::TraitRef`] or panic.
+    #[track_caller]
+    pub fn expect_trait_ref(self) -> &'hir TraitRef<'hir> {
+        let Node::TraitRef(this) = self else { self.expect_failed("a trait reference") };
+        this
+    }
+
+    /// Expect a [`Node::Pat`] or panic.
+    #[track_caller]
+    pub fn expect_pat(self) -> &'hir Pat<'hir> {
+        let Node::Pat(this) = self else { self.expect_failed("a pattern") };
+        this
+    }
+
+    /// Expect a [`Node::PatField`] or panic.
+    #[track_caller]
+    pub fn expect_pat_field(self) -> &'hir PatField<'hir> {
+        let Node::PatField(this) = self else { self.expect_failed("a pattern field") };
+        this
+    }
+
+    /// Expect a [`Node::Arm`] or panic.
+    #[track_caller]
+    pub fn expect_arm(self) -> &'hir Arm<'hir> {
+        let Node::Arm(this) = self else { self.expect_failed("an arm") };
+        this
+    }
+
+    /// Expect a [`Node::Block`] or panic.
+    #[track_caller]
+    pub fn expect_block(self) -> &'hir Block<'hir> {
+        let Node::Block(this) = self else { self.expect_failed("a block") };
+        this
+    }
+
+    /// Expect a [`Node::Local`] or panic.
+    #[track_caller]
+    pub fn expect_local(self) -> &'hir Local<'hir> {
+        let Node::Local(this) = self else { self.expect_failed("a local") };
+        this
+    }
+
+    /// Expect a [`Node::Ctor`] or panic.
+    #[track_caller]
+    pub fn expect_ctor(self) -> &'hir VariantData<'hir> {
+        let Node::Ctor(this) = self else { self.expect_failed("a constructor") };
+        this
+    }
+
+    /// Expect a [`Node::Lifetime`] or panic.
+    #[track_caller]
+    pub fn expect_lifetime(self) -> &'hir Lifetime {
+        let Node::Lifetime(this) = self else { self.expect_failed("a lifetime") };
+        this
+    }
+
+    /// Expect a [`Node::GenericParam`] or panic.
+    #[track_caller]
+    pub fn expect_generic_param(self) -> &'hir GenericParam<'hir> {
+        let Node::GenericParam(this) = self else { self.expect_failed("a generic parameter") };
+        this
+    }
+
+    /// Expect a [`Node::Crate`] or panic.
+    #[track_caller]
+    pub fn expect_crate(self) -> &'hir Mod<'hir> {
+        let Node::Crate(this) = self else { self.expect_failed("a crate") };
+        this
+    }
+
+    /// Expect a [`Node::Infer`] or panic.
+    #[track_caller]
+    pub fn expect_infer(self) -> &'hir InferArg {
+        let Node::Infer(this) = self else { self.expect_failed("an infer") };
+        this
+    }
+
+    #[track_caller]
+    fn expect_failed(&self, expected: &'static str) -> ! {
+        panic!("expected {expected} node, found {self:?}")
+    }
 }
 
 // Some nodes are used a lot. Make sure they don't unintentionally get bigger.
diff --git a/compiler/rustc_hir/src/intravisit.rs b/compiler/rustc_hir/src/intravisit.rs
index 02641b7cf..cc0f64017 100644
--- a/compiler/rustc_hir/src/intravisit.rs
+++ b/compiler/rustc_hir/src/intravisit.rs
@@ -67,6 +67,7 @@
 use crate::hir::*;
 use rustc_ast::walk_list;
 use rustc_ast::{Attribute, Label};
+use rustc_span::def_id::LocalDefId;
 use rustc_span::symbol::{Ident, Symbol};
 use rustc_span::Span;
 
@@ -364,7 +365,7 @@ pub trait Visitor<'v>: Sized {
     fn visit_fn_decl(&mut self, fd: &'v FnDecl<'v>) {
         walk_fn_decl(self, fd)
     }
-    fn visit_fn(&mut self, fk: FnKind<'v>, fd: &'v FnDecl<'v>, b: BodyId, _: Span, id: HirId) {
+    fn visit_fn(&mut self, fk: FnKind<'v>, fd: &'v FnDecl<'v>, b: BodyId, _: Span, id: LocalDefId) {
         walk_fn(self, fk, fd, b, id)
     }
     fn visit_use(&mut self, path: &'v UsePath<'v>, hir_id: HirId) {
@@ -468,13 +469,16 @@ pub fn walk_item<'v, V: Visitor<'v>>(visitor: &mut V, item: &'v Item<'v>) {
             visitor.visit_ty(typ);
             visitor.visit_nested_body(body);
         }
-        ItemKind::Fn(ref sig, ref generics, body_id) => visitor.visit_fn(
-            FnKind::ItemFn(item.ident, generics, sig.header),
-            sig.decl,
-            body_id,
-            item.span,
-            item.hir_id(),
-        ),
+        ItemKind::Fn(ref sig, ref generics, body_id) => {
+            visitor.visit_id(item.hir_id());
+            visitor.visit_fn(
+                FnKind::ItemFn(item.ident, generics, sig.header),
+                sig.decl,
+                body_id,
+                item.span,
+                item.owner_id.def_id,
+            )
+        }
         ItemKind::Macro(..) => {
             visitor.visit_id(item.hir_id());
         }
@@ -733,7 +737,7 @@ pub fn walk_expr<'v, V: Visitor<'v>>(visitor: &mut V, expression: &'v Expr<'v>)
             walk_list!(visitor, visit_arm, arms);
         }
         ExprKind::Closure(&Closure {
-            def_id: _,
+            def_id,
             binder: _,
             bound_generic_params,
             fn_decl,
@@ -745,7 +749,7 @@ pub fn walk_expr<'v, V: Visitor<'v>>(visitor: &mut V, expression: &'v Expr<'v>)
             constness: _,
         }) => {
             walk_list!(visitor, visit_generic_param, bound_generic_params);
-            visitor.visit_fn(FnKind::Closure, fn_decl, body, expression.span, expression.hir_id)
+            visitor.visit_fn(FnKind::Closure, fn_decl, body, expression.span, def_id)
         }
         ExprKind::Block(ref block, ref opt_label) => {
             walk_list!(visitor, visit_label, opt_label);
@@ -786,7 +790,7 @@ pub fn walk_expr<'v, V: Visitor<'v>>(visitor: &mut V, expression: &'v Expr<'v>)
         ExprKind::Yield(ref subexpression, _) => {
             visitor.visit_expr(subexpression);
         }
-        ExprKind::Lit(_) | ExprKind::Err => {}
+        ExprKind::Lit(_) | ExprKind::Err(_) => {}
     }
 }
 
@@ -840,7 +844,7 @@ pub fn walk_ty<'v, V: Visitor<'v>>(visitor: &mut V, typ: &'v Ty<'v>) {
             visitor.visit_lifetime(lifetime);
         }
         TyKind::Typeof(ref expression) => visitor.visit_anon_const(expression),
-        TyKind::Infer | TyKind::Err => {}
+        TyKind::Infer | TyKind::Err(_) => {}
     }
 }
 
@@ -923,9 +927,8 @@ pub fn walk_fn<'v, V: Visitor<'v>>(
     function_kind: FnKind<'v>,
     function_declaration: &'v FnDecl<'v>,
     body_id: BodyId,
-    id: HirId,
+    _: LocalDefId,
 ) {
-    visitor.visit_id(id);
     visitor.visit_fn_decl(function_declaration);
     walk_fn_kind(visitor, function_kind);
     visitor.visit_nested_body(body_id)
@@ -953,26 +956,30 @@ pub fn walk_trait_item<'v, V: Visitor<'v>>(visitor: &mut V, trait_item: &'v Trai
     let TraitItem { ident, generics, ref defaultness, ref kind, span, owner_id: _ } = *trait_item;
     let hir_id = trait_item.hir_id();
     visitor.visit_ident(ident);
-    visitor.visit_generics(generics);
-    visitor.visit_defaultness(defaultness);
+    visitor.visit_generics(&generics);
+    visitor.visit_defaultness(&defaultness);
+    visitor.visit_id(hir_id);
     match *kind {
         TraitItemKind::Const(ref ty, default) => {
-            visitor.visit_id(hir_id);
             visitor.visit_ty(ty);
             walk_list!(visitor, visit_nested_body, default);
         }
         TraitItemKind::Fn(ref sig, TraitFn::Required(param_names)) => {
-            visitor.visit_id(hir_id);
             visitor.visit_fn_decl(sig.decl);
             for &param_name in param_names {
                 visitor.visit_ident(param_name);
             }
         }
         TraitItemKind::Fn(ref sig, TraitFn::Provided(body_id)) => {
-            visitor.visit_fn(FnKind::Method(ident, sig), sig.decl, body_id, span, hir_id);
+            visitor.visit_fn(
+                FnKind::Method(ident, sig),
+                sig.decl,
+                body_id,
+                span,
+                trait_item.owner_id.def_id,
+            );
         }
         TraitItemKind::Type(bounds, ref default) => {
-            visitor.visit_id(hir_id);
             walk_list!(visitor, visit_param_bound, bounds);
             walk_list!(visitor, visit_ty, default);
         }
@@ -1002,9 +1009,9 @@ pub fn walk_impl_item<'v, V: Visitor<'v>>(visitor: &mut V, impl_item: &'v ImplIt
     visitor.visit_ident(ident);
     visitor.visit_generics(generics);
     visitor.visit_defaultness(defaultness);
+    visitor.visit_id(impl_item.hir_id());
     match *kind {
         ImplItemKind::Const(ref ty, body) => {
-            visitor.visit_id(impl_item.hir_id());
             visitor.visit_ty(ty);
             visitor.visit_nested_body(body);
         }
@@ -1014,11 +1021,10 @@ pub fn walk_impl_item<'v, V: Visitor<'v>>(visitor: &mut V, impl_item: &'v ImplIt
                 sig.decl,
                 body_id,
                 impl_item.span,
-                impl_item.hir_id(),
+                impl_item.owner_id.def_id,
             );
         }
         ImplItemKind::Type(ref ty) => {
-            visitor.visit_id(impl_item.hir_id());
             visitor.visit_ty(ty);
         }
     }
diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs
index 54fa5702f..60fa5a99e 100644
--- a/compiler/rustc_hir/src/lang_items.rs
+++ b/compiler/rustc_hir/src/lang_items.rs
@@ -244,6 +244,14 @@ language_item_table! {
     /// libstd panic entry point. Necessary for const eval to be able to catch it
     BeginPanic,              sym::begin_panic,         begin_panic_fn,             Target::Fn,             GenericRequirement::None;
 
+    // Lang items needed for `format_args!()`.
+    FormatAlignment,         sym::format_alignment,    format_alignment,           Target::Enum,           GenericRequirement::None;
+    FormatArgument,          sym::format_argument,     format_argument,            Target::Struct,         GenericRequirement::None;
+    FormatArguments,         sym::format_arguments,    format_arguments,           Target::Struct,         GenericRequirement::None;
+    FormatCount,             sym::format_count,        format_count,               Target::Enum,           GenericRequirement::None;
+    FormatPlaceholder,       sym::format_placeholder,  format_placeholder,         Target::Struct,         GenericRequirement::None;
+    FormatUnsafeArg,         sym::format_unsafe_arg,   format_unsafe_arg,          Target::Struct,         GenericRequirement::None;
+
     ExchangeMalloc,          sym::exchange_malloc,     exchange_malloc_fn,         Target::Fn,             GenericRequirement::None;
     BoxFree,                 sym::box_free,            box_free_fn,                Target::Fn,             GenericRequirement::Minimum(1);
     DropInPlace,             sym::drop_in_place,       drop_in_place_fn,           Target::Fn,             GenericRequirement::Minimum(1);
@@ -279,7 +287,7 @@ language_item_table! {
     TryTraitBranch,          sym::branch,              branch_fn,                  Target::Method(MethodKind::Trait { body: false }), GenericRequirement::None;
     TryTraitFromYeet,        sym::from_yeet,           from_yeet_fn,               Target::Fn,             GenericRequirement::None;
 
-    PointerSized,            sym::pointer_sized,       pointer_sized,              Target::Trait,          GenericRequirement::Exact(0);
+    PointerLike,             sym::pointer_like,        pointer_like,               Target::Trait,          GenericRequirement::Exact(0);
 
     Poll,                    sym::Poll,                poll,                       Target::Enum,           GenericRequirement::None;
     PollReady,               sym::Ready,               poll_ready_variant,         Target::Variant,        GenericRequirement::None;
@@ -294,8 +302,6 @@ language_item_table! {
     Context,                 sym::Context,             context,                    Target::Struct,         GenericRequirement::None;
     FuturePoll,              sym::poll,                future_poll_fn,             Target::Method(MethodKind::Trait { body: false }), GenericRequirement::None;
 
-    FromFrom,                sym::from,                from_fn,                    Target::Method(MethodKind::Trait { body: false }), GenericRequirement::None;
-
     OptionSome,              sym::Some,                option_some_variant,        Target::Variant,        GenericRequirement::None;
     OptionNone,              sym::None,                option_none_variant,        Target::Variant,        GenericRequirement::None;
 
diff --git a/compiler/rustc_hir/src/stable_hash_impls.rs b/compiler/rustc_hir/src/stable_hash_impls.rs
index 23423e8f3..85d0e02d0 100644
--- a/compiler/rustc_hir/src/stable_hash_impls.rs
+++ b/compiler/rustc_hir/src/stable_hash_impls.rs
@@ -100,13 +100,8 @@ impl<'tcx, HirCtx: crate::HashStableContext> HashStable<HirCtx> for OwnerNodes<'
         // `local_id_to_def_id` is also ignored because is dependent on the body, then just hashing
         // the body satisfies the condition of two nodes being different have different
         // `hash_stable` results.
-        let OwnerNodes {
-            hash_including_bodies,
-            hash_without_bodies: _,
-            nodes: _,
-            bodies: _,
-            local_id_to_def_id: _,
-        } = *self;
+        let OwnerNodes { hash_including_bodies, hash_without_bodies: _, nodes: _, bodies: _ } =
+            *self;
         hash_including_bodies.hash_stable(hcx, hasher);
     }
 }
diff --git a/compiler/rustc_hir/src/target.rs b/compiler/rustc_hir/src/target.rs
index 5917d5e34..961deac54 100644
--- a/compiler/rustc_hir/src/target.rs
+++ b/compiler/rustc_hir/src/target.rs
@@ -116,7 +116,7 @@ impl Target {
             DefKind::Union => Target::Union,
             DefKind::Trait => Target::Trait,
             DefKind::TraitAlias => Target::TraitAlias,
-            DefKind::Impl => Target::Impl,
+            DefKind::Impl { .. } => Target::Impl,
             _ => panic!("impossible case reached"),
         }
     }
diff --git a/compiler/rustc_hir_analysis/Cargo.toml b/compiler/rustc_hir_analysis/Cargo.toml
index 0761d8cdb..fab16b80f 100644
--- a/compiler/rustc_hir_analysis/Cargo.toml
+++ b/compiler/rustc_hir_analysis/Cargo.toml
@@ -9,15 +9,12 @@ doctest = false
 
 [dependencies]
 rustc_arena = { path = "../rustc_arena" }
-tracing = "0.1"
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_attr = { path = "../rustc_attr" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
-rustc_graphviz = { path = "../rustc_graphviz" }
 rustc_hir = { path = "../rustc_hir" }
-rustc_hir_pretty = { path = "../rustc_hir_pretty" }
 rustc_target = { path = "../rustc_target" }
 rustc_session = { path = "../rustc_session" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
@@ -27,6 +24,7 @@ rustc_index = { path = "../rustc_index" }
 rustc_infer = { path = "../rustc_infer" }
 rustc_trait_selection = { path = "../rustc_trait_selection" }
 rustc_lint = { path = "../rustc_lint" }
-rustc_serialize = { path = "../rustc_serialize" }
 rustc_type_ir = { path = "../rustc_type_ir" }
 rustc_feature = { path = "../rustc_feature" }
+thin-vec = "0.2.12"
+tracing = "0.1"
diff --git a/compiler/rustc_error_messages/locales/en-US/hir_analysis.ftl b/compiler/rustc_hir_analysis/locales/en-US.ftl
index 41f458f6c..1d313945b 100644
--- a/compiler/rustc_error_messages/locales/en-US/hir_analysis.ftl
+++ b/compiler/rustc_hir_analysis/locales/en-US.ftl
@@ -33,20 +33,7 @@ hir_analysis_field_already_declared =
     .label = field already declared
     .previous_decl_label = `{$field_name}` first declared here
 
-hir_analysis_copy_impl_on_type_with_dtor =
-    the trait `Copy` may not be implemented for this type; the type has a destructor
-    .label = `Copy` not allowed on types with destructors
-
-hir_analysis_multiple_relaxed_default_bounds =
-    type parameter has more than one relaxed default bound, only one is supported
-
-hir_analysis_copy_impl_on_non_adt =
-    the trait `Copy` may not be implemented for this type
-    .label = type is not a structure or enumeration
-
-hir_analysis_trait_object_declared_with_no_traits =
-    at least one trait is required for an object type
-    .alias_span = this alias does not contain a trait
+hir_analysis_expected_used_symbol = expected `used`, `used(compiler)` or `used(linker)`
 
 hir_analysis_ambiguous_lifetime_bound =
     ambiguous lifetime bound, explicit lifetime bound required
@@ -68,6 +55,17 @@ hir_analysis_value_of_associated_struct_already_specified =
 hir_analysis_unconstrained_opaque_type = unconstrained opaque type
     .note = `{$name}` must be used in combination with a concrete type within the same {$what}
 
+hir_analysis_manual_implementation =
+    manual implementations of `{$trait_name}` are experimental
+    .label = manual implementations of `{$trait_name}` are experimental
+    .help = add `#![feature(unboxed_closures)]` to the crate attributes to enable
+
+hir_analysis_substs_on_overridden_impl = could not resolve substs on overridden impl
+
+hir_analysis_trait_object_declared_with_no_traits =
+    at least one trait is required for an object type
+    .alias_span = this alias does not contain a trait
+
 hir_analysis_missing_type_params =
     the type {$parameterCount ->
         [one] parameter
@@ -90,20 +88,16 @@ hir_analysis_missing_type_params =
     } to {$parameters}
     .note = because of the default `Self` reference, type parameters must be specified on object types
 
-hir_analysis_manual_implementation =
-    manual implementations of `{$trait_name}` are experimental
-    .label = manual implementations of `{$trait_name}` are experimental
-    .help = add `#![feature(unboxed_closures)]` to the crate attributes to enable
-
-hir_analysis_substs_on_overridden_impl = could not resolve substs on overridden impl
+hir_analysis_copy_impl_on_type_with_dtor =
+    the trait `Copy` may not be implemented for this type; the type has a destructor
+    .label = `Copy` not allowed on types with destructors
 
-hir_analysis_unused_extern_crate =
-    unused extern crate
-    .suggestion = remove it
+hir_analysis_multiple_relaxed_default_bounds =
+    type parameter has more than one relaxed default bound, only one is supported
 
-hir_analysis_extern_crate_not_idiomatic =
-    `extern crate` is not idiomatic in the new edition
-    .suggestion = convert it to a `{$msg_code}`
+hir_analysis_copy_impl_on_non_adt =
+    the trait `Copy` may not be implemented for this type
+    .label = type is not a structure or enumeration
 
 hir_analysis_const_impl_for_non_const_trait =
     const `impl` for trait `{$trait_name}` which is not marked with `#[const_trait]`
@@ -124,3 +118,40 @@ hir_analysis_linkage_type =
 hir_analysis_auto_deref_reached_recursion_limit = reached the recursion limit while auto-dereferencing `{$ty}`
     .label = deref recursion limit reached
     .help = consider increasing the recursion limit by adding a `#![recursion_limit = "{$suggested_limit}"]` attribute to your crate (`{$crate_name}`)
+
+hir_analysis_where_clause_on_main = `main` function is not allowed to have a `where` clause
+    .label = `main` cannot have a `where` clause
+
+hir_analysis_track_caller_on_main = `main` function is not allowed to be `#[track_caller]`
+    .suggestion = remove this annotation
+
+hir_analysis_start_not_track_caller = `start` is not allowed to be `#[track_caller]`
+    .label = `start` is not allowed to be `#[track_caller]`
+
+hir_analysis_start_not_async = `start` is not allowed to be `async`
+    .label = `start` is not allowed to be `async`
+
+hir_analysis_start_function_where = start function is not allowed to have a `where` clause
+    .label = start function cannot have a `where` clause
+
+hir_analysis_start_function_parameters = start function is not allowed to have type parameters
+    .label = start function cannot have type parameters
+
+hir_analysis_main_function_return_type_generic = `main` function return type is not allowed to have generic parameters
+
+hir_analysis_main_function_async = `main` function is not allowed to be `async`
+    .label = `main` function is not allowed to be `async`
+
+hir_analysis_main_function_generic_parameters = `main` function is not allowed to have generic parameters
+    .label = `main` cannot have generic parameters
+
+hir_analysis_variadic_function_compatible_convention = C-variadic function must have a compatible calling convention, like {$conventions}
+    .label = C-variadic function must have a compatible calling convention
+
+hir_analysis_cannot_capture_late_bound_ty_in_anon_const =
+    cannot capture late-bound type parameter in a constant
+    .label = parameter defined here
+
+hir_analysis_cannot_capture_late_bound_const_in_anon_const =
+    cannot capture late-bound const parameter in a constant
+    .label = parameter defined here
diff --git a/compiler/rustc_hir_analysis/src/astconv/errors.rs b/compiler/rustc_hir_analysis/src/astconv/errors.rs
index 232ef2079..c49e4d9d5 100644
--- a/compiler/rustc_hir_analysis/src/astconv/errors.rs
+++ b/compiler/rustc_hir_analysis/src/astconv/errors.rs
@@ -1,12 +1,13 @@
 use crate::astconv::AstConv;
 use crate::errors::{ManualImplementation, MissingTypeParams};
 use rustc_data_structures::fx::FxHashMap;
-use rustc_errors::{pluralize, struct_span_err, Applicability, ErrorGuaranteed};
+use rustc_errors::{pluralize, struct_span_err, Applicability, Diagnostic, ErrorGuaranteed};
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
-use rustc_middle::ty;
+use rustc_infer::traits::FulfillmentError;
+use rustc_middle::ty::{self, Ty};
 use rustc_session::parse::feature_err;
-use rustc_span::lev_distance::find_best_match_for_name;
+use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::symbol::{sym, Ident};
 use rustc_span::{Span, Symbol, DUMMY_SP};
 
@@ -221,6 +222,228 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         err.emit()
     }
 
+    pub(crate) fn complain_about_ambiguous_inherent_assoc_type(
+        &self,
+        name: Ident,
+        candidates: Vec<DefId>,
+        span: Span,
+    ) -> ErrorGuaranteed {
+        let mut err = struct_span_err!(
+            self.tcx().sess,
+            name.span,
+            E0034,
+            "multiple applicable items in scope"
+        );
+        err.span_label(name.span, format!("multiple `{name}` found"));
+        self.note_ambiguous_inherent_assoc_type(&mut err, candidates, span);
+        err.emit()
+    }
+
+    // FIXME(fmease): Heavily adapted from `rustc_hir_typeck::method::suggest`. Deduplicate.
+    fn note_ambiguous_inherent_assoc_type(
+        &self,
+        err: &mut Diagnostic,
+        candidates: Vec<DefId>,
+        span: Span,
+    ) {
+        let tcx = self.tcx();
+
+        // Dynamic limit to avoid hiding just one candidate, which is silly.
+        let limit = if candidates.len() == 5 { 5 } else { 4 };
+
+        for (index, &item) in candidates.iter().take(limit).enumerate() {
+            let impl_ = tcx.impl_of_method(item).unwrap();
+
+            let note_span = if item.is_local() {
+                Some(tcx.def_span(item))
+            } else if impl_.is_local() {
+                Some(tcx.def_span(impl_))
+            } else {
+                None
+            };
+
+            let title = if candidates.len() > 1 {
+                format!("candidate #{}", index + 1)
+            } else {
+                "the candidate".into()
+            };
+
+            let impl_ty = tcx.at(span).type_of(impl_).subst_identity();
+            let note = format!("{title} is defined in an impl for the type `{impl_ty}`");
+
+            if let Some(span) = note_span {
+                err.span_note(span, &note);
+            } else {
+                err.note(&note);
+            }
+        }
+        if candidates.len() > limit {
+            err.note(&format!("and {} others", candidates.len() - limit));
+        }
+    }
+
+    // FIXME(inherent_associated_types): Find similarly named associated types and suggest them.
+    pub(crate) fn complain_about_inherent_assoc_type_not_found(
+        &self,
+        name: Ident,
+        self_ty: Ty<'tcx>,
+        candidates: Vec<(DefId, (DefId, DefId))>,
+        fulfillment_errors: Vec<FulfillmentError<'tcx>>,
+        span: Span,
+    ) -> ErrorGuaranteed {
+        // FIXME(fmease): This was copied in parts from an old version of `rustc_hir_typeck::method::suggest`.
+        // Either
+        // * update this code by applying changes similar to #106702 or by taking a
+        //   Vec<(DefId, (DefId, DefId), Option<Vec<FulfillmentError<'tcx>>>)> or
+        // * deduplicate this code across the two crates.
+
+        let tcx = self.tcx();
+
+        let adt_did = self_ty.ty_adt_def().map(|def| def.did());
+        let add_def_label = |err: &mut Diagnostic| {
+            if let Some(did) = adt_did {
+                err.span_label(
+                    tcx.def_span(did),
+                    format!("associated item `{name}` not found for this {}", tcx.def_descr(did)),
+                );
+            }
+        };
+
+        if fulfillment_errors.is_empty() {
+            // FIXME(fmease): Copied from `rustc_hir_typeck::method::probe`. Deduplicate.
+
+            let limit = if candidates.len() == 5 { 5 } else { 4 };
+            let type_candidates = candidates
+                .iter()
+                .take(limit)
+                .map(|&(impl_, _)| format!("- `{}`", tcx.at(span).type_of(impl_).subst_identity()))
+                .collect::<Vec<_>>()
+                .join("\n");
+            let additional_types = if candidates.len() > limit {
+                format!("\nand {} more types", candidates.len() - limit)
+            } else {
+                String::new()
+            };
+
+            let mut err = struct_span_err!(
+                tcx.sess,
+                name.span,
+                E0220,
+                "associated type `{name}` not found for `{self_ty}` in the current scope"
+            );
+            err.span_label(name.span, format!("associated item not found in `{self_ty}`"));
+            err.note(&format!(
+                "the associated type was found for\n{type_candidates}{additional_types}",
+            ));
+            add_def_label(&mut err);
+            return err.emit();
+        }
+
+        let mut bound_spans = Vec::new();
+
+        let mut bound_span_label = |self_ty: Ty<'_>, obligation: &str, quiet: &str| {
+            let msg = format!(
+                "doesn't satisfy `{}`",
+                if obligation.len() > 50 { quiet } else { obligation }
+            );
+            match &self_ty.kind() {
+                // Point at the type that couldn't satisfy the bound.
+                ty::Adt(def, _) => bound_spans.push((tcx.def_span(def.did()), msg)),
+                // Point at the trait object that couldn't satisfy the bound.
+                ty::Dynamic(preds, _, _) => {
+                    for pred in preds.iter() {
+                        match pred.skip_binder() {
+                            ty::ExistentialPredicate::Trait(tr) => {
+                                bound_spans.push((tcx.def_span(tr.def_id), msg.clone()))
+                            }
+                            ty::ExistentialPredicate::Projection(_)
+                            | ty::ExistentialPredicate::AutoTrait(_) => {}
+                        }
+                    }
+                }
+                // Point at the closure that couldn't satisfy the bound.
+                ty::Closure(def_id, _) => {
+                    bound_spans.push((tcx.def_span(*def_id), format!("doesn't satisfy `{quiet}`")))
+                }
+                _ => {}
+            }
+        };
+
+        let format_pred = |pred: ty::Predicate<'tcx>| {
+            let bound_predicate = pred.kind();
+            match bound_predicate.skip_binder() {
+                ty::PredicateKind::Clause(ty::Clause::Projection(pred)) => {
+                    let pred = bound_predicate.rebind(pred);
+                    // `<Foo as Iterator>::Item = String`.
+                    let projection_ty = pred.skip_binder().projection_ty;
+
+                    let substs_with_infer_self = tcx.mk_substs_from_iter(
+                        std::iter::once(tcx.mk_ty_var(ty::TyVid::from_u32(0)).into())
+                            .chain(projection_ty.substs.iter().skip(1)),
+                    );
+
+                    let quiet_projection_ty =
+                        tcx.mk_alias_ty(projection_ty.def_id, substs_with_infer_self);
+
+                    let term = pred.skip_binder().term;
+
+                    let obligation = format!("{projection_ty} = {term}");
+                    let quiet = format!("{quiet_projection_ty} = {term}");
+
+                    bound_span_label(projection_ty.self_ty(), &obligation, &quiet);
+                    Some((obligation, projection_ty.self_ty()))
+                }
+                ty::PredicateKind::Clause(ty::Clause::Trait(poly_trait_ref)) => {
+                    let p = poly_trait_ref.trait_ref;
+                    let self_ty = p.self_ty();
+                    let path = p.print_only_trait_path();
+                    let obligation = format!("{self_ty}: {path}");
+                    let quiet = format!("_: {path}");
+                    bound_span_label(self_ty, &obligation, &quiet);
+                    Some((obligation, self_ty))
+                }
+                _ => None,
+            }
+        };
+
+        // FIXME(fmease): `rustc_hir_typeck::method::suggest` uses a `skip_list` to filter out some bounds.
+        // I would do the same here if it didn't mean more code duplication.
+        let mut bounds: Vec<_> = fulfillment_errors
+            .into_iter()
+            .map(|error| error.root_obligation.predicate)
+            .filter_map(format_pred)
+            .map(|(p, _)| format!("`{}`", p))
+            .collect();
+        bounds.sort();
+        bounds.dedup();
+
+        let mut err = tcx.sess.struct_span_err(
+            name.span,
+            &format!("the associated type `{name}` exists for `{self_ty}`, but its trait bounds were not satisfied")
+        );
+        if !bounds.is_empty() {
+            err.note(&format!(
+                "the following trait bounds were not satisfied:\n{}",
+                bounds.join("\n")
+            ));
+        }
+        err.span_label(
+            name.span,
+            format!("associated type cannot be referenced on `{self_ty}` due to unsatisfied trait bounds")
+        );
+
+        bound_spans.sort();
+        bound_spans.dedup();
+        for (span, msg) in bound_spans {
+            if !tcx.sess.source_map().is_span_accessible(span) {
+                continue;
+            }
+            err.span_label(span, &msg);
+        }
+        add_def_label(&mut err);
+        err.emit()
+    }
+
     /// When there are any missing associated types, emit an E0191 error and attempt to supply a
     /// reasonable suggestion on how to write it. For the case of multiple associated types in the
     /// same trait bound have the same name (as they come from different supertraits), we instead
diff --git a/compiler/rustc_hir_analysis/src/astconv/generics.rs b/compiler/rustc_hir_analysis/src/astconv/generics.rs
index 7a499327d..7f6518ffd 100644
--- a/compiler/rustc_hir_analysis/src/astconv/generics.rs
+++ b/compiler/rustc_hir_analysis/src/astconv/generics.rs
@@ -6,7 +6,7 @@ use crate::astconv::{
 use crate::errors::AssocTypeBindingNotAllowed;
 use crate::structured_errors::{GenericArgsInfo, StructuredDiagnostic, WrongNumberOfGenericArgs};
 use rustc_ast::ast::ParamKindOrd;
-use rustc_errors::{struct_span_err, Applicability, Diagnostic, MultiSpan};
+use rustc_errors::{struct_span_err, Applicability, Diagnostic, ErrorGuaranteed, MultiSpan};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::DefId;
@@ -26,7 +26,7 @@ fn generic_arg_mismatch_err(
     param: &GenericParamDef,
     possible_ordering_error: bool,
     help: Option<&str>,
-) {
+) -> ErrorGuaranteed {
     let sess = tcx.sess;
     let mut err = struct_span_err!(
         sess,
@@ -70,18 +70,18 @@ fn generic_arg_mismatch_err(
         ) => match path.res {
             Res::Err => {
                 add_braces_suggestion(arg, &mut err);
-                err.set_primary_message("unresolved item provided when a constant was expected")
+                return err
+                    .set_primary_message("unresolved item provided when a constant was expected")
                     .emit();
-                return;
             }
             Res::Def(DefKind::TyParam, src_def_id) => {
                 if let Some(param_local_id) = param.def_id.as_local() {
                     let param_name = tcx.hir().ty_param_name(param_local_id);
-                    let param_type = tcx.type_of(param.def_id);
+                    let param_type = tcx.type_of(param.def_id).subst_identity();
                     if param_type.is_suggestable(tcx, false) {
                         err.span_suggestion(
                             tcx.def_span(src_def_id),
-                            "consider changing this type parameter to be a `const` generic",
+                            "consider changing this type parameter to a const parameter",
                             format!("const {}: {}", param_name, param_type),
                             Applicability::MaybeIncorrect,
                         );
@@ -97,7 +97,7 @@ fn generic_arg_mismatch_err(
         (
             GenericArg::Type(hir::Ty { kind: hir::TyKind::Array(_, len), .. }),
             GenericParamDefKind::Const { .. },
-        ) if tcx.type_of(param.def_id) == tcx.types.usize => {
+        ) if tcx.type_of(param.def_id).skip_binder() == tcx.types.usize => {
             let snippet = sess.source_map().span_to_snippet(tcx.hir().span(len.hir_id()));
             if let Ok(snippet) = snippet {
                 err.span_suggestion(
@@ -137,7 +137,7 @@ fn generic_arg_mismatch_err(
         }
     }
 
-    err.emit();
+    err.emit()
 }
 
 /// Creates the relevant generic argument substitutions
@@ -370,7 +370,7 @@ pub fn create_substs_for_generic_args<'tcx, 'a>(
         }
     }
 
-    tcx.intern_substs(&substs)
+    tcx.mk_substs(&substs)
 }
 
 /// Checks that the correct number of generic arguments have been provided.
@@ -385,10 +385,9 @@ pub fn check_generic_arg_count_for_call(
 ) -> GenericArgCountResult {
     let empty_args = hir::GenericArgs::none();
     let gen_args = seg.args.unwrap_or(&empty_args);
-    let gen_pos = if is_method_call == IsMethodCall::Yes {
-        GenericArgPosition::MethodCall
-    } else {
-        GenericArgPosition::Value
+    let gen_pos = match is_method_call {
+        IsMethodCall::Yes => GenericArgPosition::MethodCall,
+        IsMethodCall::No => GenericArgPosition::Value,
     };
     let has_self = generics.parent.is_none() && generics.has_self;
 
diff --git a/compiler/rustc_hir_analysis/src/astconv/mod.rs b/compiler/rustc_hir_analysis/src/astconv/mod.rs
index 6435b05ce..f5ce02c9e 100644
--- a/compiler/rustc_hir_analysis/src/astconv/mod.rs
+++ b/compiler/rustc_hir_analysis/src/astconv/mod.rs
@@ -14,7 +14,7 @@ use crate::errors::{
     AmbiguousLifetimeBound, MultipleRelaxedDefaultBounds, TraitObjectDeclaredWithNoTraits,
     TypeofReservedKeywordUsed, ValueOfAssociatedStructAlreadySpecified,
 };
-use crate::middle::resolve_lifetime as rl;
+use crate::middle::resolve_bound_vars as rbv;
 use crate::require_c_abi_if_c_variadic;
 use rustc_ast::TraitObjectSyntax;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
@@ -27,34 +27,37 @@ use rustc_hir::def::{CtorOf, DefKind, Namespace, Res};
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::intravisit::{walk_generics, Visitor as _};
 use rustc_hir::{GenericArg, GenericArgs, OpaqueTyOrigin};
-use rustc_infer::infer::TyCtxtInferExt;
+use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
+use rustc_infer::infer::{InferCtxt, TyCtxtInferExt};
+use rustc_infer::traits::ObligationCause;
+use rustc_middle::infer::unify_key::{ConstVariableOrigin, ConstVariableOriginKind};
 use rustc_middle::middle::stability::AllowUnstable;
 use rustc_middle::ty::subst::{self, GenericArgKind, InternalSubsts, SubstsRef};
+use rustc_middle::ty::DynKind;
 use rustc_middle::ty::GenericParamDefKind;
-use rustc_middle::ty::{self, Const, DefIdTree, IsSuggestable, Ty, TyCtxt, TypeVisitable};
-use rustc_middle::ty::{DynKind, EarlyBinder};
+use rustc_middle::ty::{self, Const, DefIdTree, IsSuggestable, Ty, TyCtxt, TypeVisitableExt};
 use rustc_session::lint::builtin::{AMBIGUOUS_ASSOCIATED_ITEMS, BARE_TRAIT_OBJECTS};
+use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::edition::Edition;
-use rustc_span::lev_distance::find_best_match_for_name;
 use rustc_span::symbol::{kw, Ident, Symbol};
-use rustc_span::{sym, Span};
+use rustc_span::{sym, Span, DUMMY_SP};
 use rustc_target::spec::abi;
-use rustc_trait_selection::traits;
-use rustc_trait_selection::traits::astconv_object_safety_violations;
 use rustc_trait_selection::traits::error_reporting::{
     report_object_safety_error, suggestions::NextTypeParamName,
 };
 use rustc_trait_selection::traits::wf::object_region_bounds;
+use rustc_trait_selection::traits::{self, astconv_object_safety_violations, ObligationCtxt};
 
 use smallvec::{smallvec, SmallVec};
 use std::collections::BTreeSet;
+use std::fmt::Display;
 use std::slice;
 
 #[derive(Debug)]
 pub struct PathSeg(pub DefId, pub usize);
 
 pub trait AstConv<'tcx> {
-    fn tcx<'a>(&'a self) -> TyCtxt<'tcx>;
+    fn tcx(&self) -> TyCtxt<'tcx>;
 
     fn item_def_id(&self) -> DefId;
 
@@ -131,6 +134,8 @@ pub trait AstConv<'tcx> {
     {
         self
     }
+
+    fn infcx(&self) -> Option<&InferCtxt<'tcx>>;
 }
 
 #[derive(Debug)]
@@ -223,36 +228,37 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         let tcx = self.tcx();
         let lifetime_name = |def_id| tcx.hir().name(tcx.hir().local_def_id_to_hir_id(def_id));
 
-        match tcx.named_region(lifetime.hir_id) {
-            Some(rl::Region::Static) => tcx.lifetimes.re_static,
+        match tcx.named_bound_var(lifetime.hir_id) {
+            Some(rbv::ResolvedArg::StaticLifetime) => tcx.lifetimes.re_static,
 
-            Some(rl::Region::LateBound(debruijn, index, def_id)) => {
+            Some(rbv::ResolvedArg::LateBound(debruijn, index, def_id)) => {
                 let name = lifetime_name(def_id.expect_local());
                 let br = ty::BoundRegion {
                     var: ty::BoundVar::from_u32(index),
                     kind: ty::BrNamed(def_id, name),
                 };
-                tcx.mk_region(ty::ReLateBound(debruijn, br))
+                tcx.mk_re_late_bound(debruijn, br)
             }
 
-            Some(rl::Region::EarlyBound(def_id)) => {
+            Some(rbv::ResolvedArg::EarlyBound(def_id)) => {
                 let name = tcx.hir().ty_param_name(def_id.expect_local());
                 let item_def_id = tcx.hir().ty_param_owner(def_id.expect_local());
                 let generics = tcx.generics_of(item_def_id);
                 let index = generics.param_def_id_to_index[&def_id];
-                tcx.mk_region(ty::ReEarlyBound(ty::EarlyBoundRegion { def_id, index, name }))
+                tcx.mk_re_early_bound(ty::EarlyBoundRegion { def_id, index, name })
             }
 
-            Some(rl::Region::Free(scope, id)) => {
+            Some(rbv::ResolvedArg::Free(scope, id)) => {
                 let name = lifetime_name(id.expect_local());
-                tcx.mk_region(ty::ReFree(ty::FreeRegion {
-                    scope,
-                    bound_region: ty::BrNamed(id, name),
-                }))
+                tcx.mk_re_free(scope, ty::BrNamed(id, name))
 
                 // (*) -- not late-bound, won't change
             }
 
+            Some(rbv::ResolvedArg::Error(_)) => {
+                bug!("only ty/ct should resolve as ResolvedArg::Error")
+            }
+
             None => {
                 self.re_infer(def, lifetime.ident.span).unwrap_or_else(|| {
                     debug!(?lifetime, "unelided lifetime in signature");
@@ -261,11 +267,10 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     // elision. `resolve_lifetime` should have
                     // reported an error in this case -- but if
                     // not, let's error out.
-                    tcx.sess.delay_span_bug(lifetime.ident.span, "unelided lifetime in signature");
-
-                    // Supply some dummy value. We don't have an
-                    // `re_error`, annoyingly, so use `'static`.
-                    tcx.lifetimes.re_static
+                    tcx.mk_re_error_with_message(
+                        lifetime.ident.span,
+                        "unelided lifetime in signature",
+                    )
                 })
             }
         }
@@ -377,7 +382,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         // here and so associated type bindings will be handled regardless of whether there are any
         // non-`Self` generic parameters.
         if generics.params.is_empty() {
-            return (tcx.intern_substs(parent_substs), arg_count);
+            return (tcx.mk_substs(parent_substs), arg_count);
         }
 
         struct SubstsForAstPathCtxt<'a, 'tcx> {
@@ -425,7 +430,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     }
                     if let (hir::TyKind::Infer, false) = (&ty.kind, self.astconv.allow_ty_infer()) {
                         self.inferred_params.push(ty.span);
-                        tcx.ty_error().into()
+                        tcx.ty_error_misc().into()
                     } else {
                         self.astconv.ast_ty_to_ty(ty).into()
                     }
@@ -452,7 +457,11 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                         .into()
                     }
                     (&GenericParamDefKind::Const { .. }, hir::GenericArg::Infer(inf)) => {
-                        let ty = tcx.at(self.span).type_of(param.def_id);
+                        let ty = tcx
+                            .at(self.span)
+                            .type_of(param.def_id)
+                            .no_bound_vars()
+                            .expect("const parameter types cannot be generic");
                         if self.astconv.allow_ty_infer() {
                             self.astconv.ct_infer(ty, Some(param), inf.span).into()
                         } else {
@@ -479,11 +488,10 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                             debug!(?param, "unelided lifetime in signature");
 
                             // This indicates an illegal lifetime in a non-assoc-trait position
-                            tcx.sess.delay_span_bug(self.span, "unelided lifetime in signature");
-
-                            // Supply some dummy value. We don't have an
-                            // `re_error`, annoyingly, so use `'static`.
-                            tcx.lifetimes.re_static
+                            tcx.mk_re_error_with_message(
+                                self.span,
+                                "unelided lifetime in signature",
+                            )
                         })
                         .into(),
                     GenericParamDefKind::Type { has_default, .. } => {
@@ -495,20 +503,24 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                                 _ => false,
                             }) {
                                 // Avoid ICE #86756 when type error recovery goes awry.
-                                return tcx.ty_error().into();
+                                return tcx.ty_error_misc().into();
                             }
-                            tcx.at(self.span).bound_type_of(param.def_id).subst(tcx, substs).into()
+                            tcx.at(self.span).type_of(param.def_id).subst(tcx, substs).into()
                         } else if infer_args {
                             self.astconv.ty_infer(Some(param), self.span).into()
                         } else {
                             // We've already errored above about the mismatch.
-                            tcx.ty_error().into()
+                            tcx.ty_error_misc().into()
                         }
                     }
                     GenericParamDefKind::Const { has_default } => {
-                        let ty = tcx.at(self.span).type_of(param.def_id);
-                        if ty.references_error() {
-                            return tcx.const_error(ty).into();
+                        let ty = tcx
+                            .at(self.span)
+                            .type_of(param.def_id)
+                            .no_bound_vars()
+                            .expect("const parameter types cannot be generic");
+                        if let Err(guar) = ty.error_reported() {
+                            return tcx.const_error_with_guaranteed(ty, guar).into();
                         }
                         if !infer_args && has_default {
                             tcx.const_param_default(param.def_id).subst(tcx, substs.unwrap()).into()
@@ -1084,11 +1096,11 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 // those that do.
                 self.one_bound_for_assoc_type(
                     || traits::supertraits(tcx, trait_ref),
-                    || trait_ref.print_only_trait_path().to_string(),
+                    trait_ref.print_only_trait_path(),
                     binding.item_name,
                     path_span,
-                    || match binding.kind {
-                        ConvertedBindingKind::Equality(ty) => Some(ty.to_string()),
+                    match binding.kind {
+                        ConvertedBindingKind::Equality(term) => Some(term),
                         _ => None,
                     },
                 )?
@@ -1206,7 +1218,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     | (hir::def::DefKind::AssocConst, ty::TermKind::Const(_)) => (),
                     (_, _) => {
                         let got = if let Some(_) = term.ty() { "type" } else { "constant" };
-                        let expected = def_kind.descr(assoc_item_def_id);
+                        let expected = tcx.def_descr(assoc_item_def_id);
                         let mut err = tcx.sess.struct_span_err(
                             binding.span,
                             &format!("expected {expected} bound, found {got}"),
@@ -1228,12 +1240,10 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                         }
                         let reported = err.emit();
                         term = match def_kind {
-                            hir::def::DefKind::AssocTy => {
-                                tcx.ty_error_with_guaranteed(reported).into()
-                            }
+                            hir::def::DefKind::AssocTy => tcx.ty_error(reported).into(),
                             hir::def::DefKind::AssocConst => tcx
                                 .const_error_with_guaranteed(
-                                    tcx.bound_type_of(assoc_item_def_id)
+                                    tcx.type_of(assoc_item_def_id)
                                         .subst(tcx, projection_ty.skip_binder().substs),
                                     reported,
                                 )
@@ -1256,7 +1266,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 //
                 // Calling `skip_binder` is okay, because `add_bounds` expects the `param_ty`
                 // parameter to have a skipped binder.
-                let param_ty = tcx.mk_ty(ty::Alias(ty::Projection, projection_ty.skip_binder()));
+                let param_ty = tcx.mk_alias(ty::Projection, projection_ty.skip_binder());
                 self.add_bounds(param_ty, ast_bounds.iter(), bounds, candidate.bound_vars());
             }
         }
@@ -1270,7 +1280,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         item_segment: &hir::PathSegment<'_>,
     ) -> Ty<'tcx> {
         let substs = self.ast_path_substs_for_ty(span, did, item_segment);
-        self.tcx().at(span).bound_type_of(did).subst(self.tcx(), substs)
+        self.tcx().at(span).type_of(did).subst(self.tcx(), substs)
     }
 
     fn conv_object_ty_poly_trait_ref(
@@ -1323,9 +1333,10 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     ty::Clause::TypeOutlives(_) => {
                         // Do nothing, we deal with regions separately
                     }
-                    ty::Clause::RegionOutlives(_) => bug!(),
+                    ty::Clause::RegionOutlives(_) | ty::Clause::ConstArgHasType(..) => bug!(),
                 },
                 ty::PredicateKind::WellFormed(_)
+                | ty::PredicateKind::AliasEq(..)
                 | ty::PredicateKind::ObjectSafe(_)
                 | ty::PredicateKind::ClosureKind(_, _, _)
                 | ty::PredicateKind::Subtype(_)
@@ -1385,7 +1396,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 .map(|trait_ref| tcx.def_span(trait_ref));
             let reported =
                 tcx.sess.emit_err(TraitObjectDeclaredWithNoTraits { span, trait_alias_span });
-            return tcx.ty_error_with_guaranteed(reported);
+            return tcx.ty_error(reported);
         }
 
         // Check that there are no gross object safety violations;
@@ -1402,7 +1413,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     &object_safety_violations,
                 )
                 .emit();
-                return tcx.ty_error_with_guaranteed(reported);
+                return tcx.ty_error(reported);
             }
         }
 
@@ -1511,15 +1522,15 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                         if arg == dummy_self.into() {
                             let param = &generics.params[index];
                             missing_type_params.push(param.name);
-                            return tcx.ty_error().into();
+                            return tcx.ty_error_misc().into();
                         } else if arg.walk().any(|arg| arg == dummy_self.into()) {
                             references_self = true;
-                            return tcx.ty_error().into();
+                            return tcx.ty_error_misc().into();
                         }
                         arg
                     })
                     .collect();
-                let substs = tcx.intern_substs(&substs[..]);
+                let substs = tcx.mk_substs(&substs);
 
                 let span = i.bottom().1;
                 let empty_generic_args = hir_trait_bounds.iter().any(|hir_bound| {
@@ -1540,7 +1551,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                         i.bottom().1,
                         E0038,
                         "the {} `{}` cannot be made into an object",
-                        tcx.def_kind(def_id).descr(def_id),
+                        tcx.def_descr(def_id),
                         tcx.item_name(def_id),
                     );
                     err.note(
@@ -1567,7 +1578,8 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     false
                 });
                 if references_self {
-                    tcx.sess
+                    let guar = tcx
+                        .sess
                         .delay_span_bug(span, "trait object projection bounds reference `Self`");
                     let substs: Vec<_> = b
                         .projection_ty
@@ -1575,12 +1587,12 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                         .iter()
                         .map(|arg| {
                             if arg.walk().any(|arg| arg == dummy_self.into()) {
-                                return tcx.ty_error().into();
+                                return tcx.ty_error(guar).into();
                             }
                             arg
                         })
                         .collect();
-                    b.projection_ty.substs = tcx.intern_substs(&substs[..]);
+                    b.projection_ty.substs = tcx.mk_substs(&substs);
                 }
 
                 ty::ExistentialProjection::erase_self_ty(tcx, b)
@@ -1602,14 +1614,14 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             .collect::<SmallVec<[_; 8]>>();
         v.sort_by(|a, b| a.skip_binder().stable_cmp(tcx, &b.skip_binder()));
         v.dedup();
-        let existential_predicates = tcx.mk_poly_existential_predicates(v.into_iter());
+        let existential_predicates = tcx.mk_poly_existential_predicates(&v);
 
         // Use explicitly-specified region bound.
         let region_bound = if !lifetime.is_elided() {
             self.ast_region_to_region(lifetime, None)
         } else {
             self.compute_object_lifetime_bound(span, existential_predicates).unwrap_or_else(|| {
-                if tcx.named_region(lifetime.hir_id).is_some() {
+                if tcx.named_bound_var(lifetime.hir_id).is_some() {
                     self.ast_region_to_region(lifetime, None)
                 } else {
                     self.re_infer(None, span).unwrap_or_else(|| {
@@ -1620,14 +1632,14 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                             "the lifetime bound for this object type cannot be deduced \
                              from context; please supply an explicit bound"
                         );
-                        if borrowed {
+                        let e = if borrowed {
                             // We will have already emitted an error E0106 complaining about a
                             // missing named lifetime in `&dyn Trait`, so we elide this one.
-                            err.delay_as_bug();
+                            err.delay_as_bug()
                         } else {
-                            err.emit();
-                        }
-                        tcx.lifetimes.re_static
+                            err.emit()
+                        };
+                        tcx.mk_re_error(e)
                     })
                 }
             })
@@ -1778,10 +1790,10 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     assoc_name,
                 )
             },
-            || param_name.to_string(),
+            param_name,
             assoc_name,
             span,
-            || None,
+            None,
         )
     }
 
@@ -1791,10 +1803,10 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
     fn one_bound_for_assoc_type<I>(
         &self,
         all_candidates: impl Fn() -> I,
-        ty_param_name: impl Fn() -> String,
+        ty_param_name: impl Display,
         assoc_name: Ident,
         span: Span,
-        is_equality: impl Fn() -> Option<String>,
+        is_equality: Option<ty::Term<'tcx>>,
     ) -> Result<ty::PolyTraitRef<'tcx>, ErrorGuaranteed>
     where
         I: Iterator<Item = ty::PolyTraitRef<'tcx>>,
@@ -1810,7 +1822,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             (None, None) => {
                 let reported = self.complain_about_assoc_type_not_found(
                     all_candidates,
-                    &ty_param_name(),
+                    &ty_param_name.to_string(),
                     assoc_name,
                     span,
                 );
@@ -1822,7 +1834,6 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         if let Some(bound2) = next_cand {
             debug!(?bound2);
 
-            let is_equality = is_equality();
             let bounds = IntoIterator::into_iter([bound, bound2]).chain(matching_candidates);
             let mut err = if is_equality.is_some() {
                 // More specific Error Index entry.
@@ -1832,7 +1843,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     E0222,
                     "ambiguous associated type `{}` in bounds of `{}`",
                     assoc_name,
-                    ty_param_name()
+                    ty_param_name
                 )
             } else {
                 struct_span_err!(
@@ -1841,7 +1852,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     E0221,
                     "ambiguous associated type `{}` in bounds of `{}`",
                     assoc_name,
-                    ty_param_name()
+                    ty_param_name
                 )
             };
             err.span_label(span, format!("ambiguous associated type `{}`", assoc_name));
@@ -1875,18 +1886,14 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                         err.span_suggestion_verbose(
                             span.with_hi(assoc_name.span.lo()),
                             "use fully qualified syntax to disambiguate",
-                            format!(
-                                "<{} as {}>::",
-                                ty_param_name(),
-                                bound.print_only_trait_path(),
-                            ),
+                            format!("<{} as {}>::", ty_param_name, bound.print_only_trait_path()),
                             Applicability::MaybeIncorrect,
                         );
                     }
                 } else {
                     err.note(&format!(
                         "associated type `{}` could derive from `{}`",
-                        ty_param_name(),
+                        ty_param_name,
                         bound.print_only_trait_path(),
                     ));
                 }
@@ -1895,7 +1902,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 err.help(&format!(
                     "consider introducing a new type parameter `T` and adding `where` constraints:\
                      \n    where\n        T: {},\n{}",
-                    ty_param_name(),
+                    ty_param_name,
                     where_bounds.join(",\n"),
                 ));
             }
@@ -1934,7 +1941,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             Res::Err
         };
 
-        // Check if we have an enum variant.
+        // Check if we have an enum variant or an inherent associated type.
         let mut variant_resolution = None;
         if let Some(adt_def) = self.probe_adt(span, qself_ty) {
             if adt_def.is_enum() {
@@ -1975,7 +1982,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                             ) = &qself.kind {
                                 // If the path segment already has type params, we want to overwrite
                                 // them.
-                                match &path.segments[..] {
+                                match &path.segments {
                                     // `segment` is the previous to last element on the path,
                                     // which would normally be the `enum` itself, while the last
                                     // `_` `PathSegment` corresponds to the variant.
@@ -2033,23 +2040,15 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 }
             }
 
-            // see if we can satisfy using an inherent associated type
-            for &impl_ in tcx.inherent_impls(adt_def.did()) {
-                let Some(assoc_ty_did) = self.lookup_assoc_ty(assoc_ident, hir_ref_id, span, impl_) else {
-                    continue;
-                };
-                let ty::Adt(_, adt_substs) = qself_ty.kind() else {
-                    // FIXME(inherent_associated_types)
-                    bug!("unimplemented: non-adt self of inherent assoc ty");
-                };
-                let item_substs = self.create_substs_for_associated_item(
-                    span,
-                    assoc_ty_did,
-                    assoc_segment,
-                    adt_substs,
-                );
-                let ty = tcx.bound_type_of(assoc_ty_did).subst(tcx, item_substs);
-                return Ok((ty, DefKind::AssocTy, assoc_ty_did));
+            if let Some((ty, did)) = self.lookup_inherent_assoc_ty(
+                assoc_ident,
+                assoc_segment,
+                adt_def.did(),
+                qself_ty,
+                hir_ref_id,
+                span,
+            )? {
+                return Ok((ty, DefKind::AssocTy, did));
             }
         }
 
@@ -2067,10 +2066,10 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
 
                 self.one_bound_for_assoc_type(
                     || traits::supertraits(tcx, ty::Binder::dummy(trait_ref.subst_identity())),
-                    || "Self".to_string(),
+                    kw::SelfUpper,
                     assoc_ident,
                     span,
-                    || None,
+                    None,
                 )?
             }
             (
@@ -2132,48 +2131,8 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     )
                     .emit() // Already reported in an earlier stage.
                 } else {
-                    // Find all the `impl`s that `qself_ty` has for any trait that has the
-                    // associated type, so that we suggest the right one.
-                    let infcx = tcx.infer_ctxt().build();
-                    // We create a fresh `ty::ParamEnv` instead of the one for `self.item_def_id()`
-                    // to avoid a cycle error in `src/test/ui/resolve/issue-102946.rs`.
-                    let param_env = ty::ParamEnv::empty();
-                    let traits: Vec<_> = self
-                        .tcx()
-                        .all_traits()
-                        .filter(|trait_def_id| {
-                            // Consider only traits with the associated type
-                            tcx.associated_items(*trait_def_id)
-                                .in_definition_order()
-                                .any(|i| {
-                                    i.kind.namespace() == Namespace::TypeNS
-                                        && i.ident(tcx).normalize_to_macros_2_0() == assoc_ident
-                                        && matches!(i.kind, ty::AssocKind::Type)
-                                })
-                            // Consider only accessible traits
-                            && tcx.visibility(*trait_def_id)
-                                .is_accessible_from(self.item_def_id(), tcx)
-                            && tcx.all_impls(*trait_def_id)
-                                .any(|impl_def_id| {
-                                    let trait_ref = tcx.impl_trait_ref(impl_def_id);
-                                    trait_ref.map_or(false, |trait_ref| {
-                                        let impl_ = trait_ref.subst(
-                                            tcx,
-                                            infcx.fresh_substs_for_item(span, impl_def_id),
-                                        );
-                                        infcx
-                                            .can_eq(
-                                                param_env,
-                                                tcx.erase_regions(impl_.self_ty()),
-                                                tcx.erase_regions(qself_ty),
-                                            )
-                                            .is_ok()
-                                    })
-                                    && tcx.impl_polarity(impl_def_id) != ty::ImplPolarity::Negative
-                                })
-                        })
-                        .map(|trait_def_id| tcx.def_path_str(trait_def_id))
-                        .collect();
+                    let traits: Vec<_> =
+                        self.probe_traits_that_match_assoc_ty(qself_ty, assoc_ident);
 
                     // Don't print `TyErr` to the user.
                     self.report_ambiguous_associated_type(
@@ -2210,7 +2169,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                             "`{}` could{} refer to the {} defined here",
                             assoc_ident,
                             also,
-                            kind.descr(def_id)
+                            tcx.def_kind_descr(kind, def_id)
                         );
                         lint.span_note(tcx.def_span(def_id), &note_msg);
                     };
@@ -2232,15 +2191,137 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         Ok((ty, DefKind::AssocTy, assoc_ty_did))
     }
 
+    fn lookup_inherent_assoc_ty(
+        &self,
+        name: Ident,
+        segment: &hir::PathSegment<'_>,
+        adt_did: DefId,
+        self_ty: Ty<'tcx>,
+        block: hir::HirId,
+        span: Span,
+    ) -> Result<Option<(Ty<'tcx>, DefId)>, ErrorGuaranteed> {
+        let tcx = self.tcx();
+
+        let candidates: Vec<_> = tcx
+            .inherent_impls(adt_did)
+            .iter()
+            .filter_map(|&impl_| Some((impl_, self.lookup_assoc_ty_unchecked(name, block, impl_)?)))
+            .collect();
+
+        if candidates.is_empty() {
+            return Ok(None);
+        }
+
+        // In contexts that have no inference context, just make a new one.
+        // We do need a local variable to store it, though.
+        let infcx_;
+        let infcx = match self.infcx() {
+            Some(infcx) => infcx,
+            None => {
+                assert!(!self_ty.needs_infer());
+                infcx_ = tcx.infer_ctxt().ignoring_regions().build();
+                &infcx_
+            }
+        };
+
+        let param_env = tcx.param_env(block.owner.to_def_id());
+        let cause = ObligationCause::misc(span, block.owner.def_id);
+        let mut fulfillment_errors = Vec::new();
+        let mut applicable_candidates: Vec<_> = candidates
+            .iter()
+            .filter_map(|&(impl_, (assoc_item, def_scope))| {
+                infcx.probe(|_| {
+                    let ocx = ObligationCtxt::new_in_snapshot(&infcx);
+
+                    let impl_ty = tcx.type_of(impl_);
+                    let impl_substs = infcx.fresh_item_substs(impl_);
+                    let impl_ty = impl_ty.subst(tcx, impl_substs);
+                    let impl_ty = ocx.normalize(&cause, param_env, impl_ty);
+
+                    // Check that the Self-types can be related.
+                    // FIXME(fmease): Should we use `eq` here?
+                    ocx.sup(&ObligationCause::dummy(), param_env, impl_ty, self_ty).ok()?;
+
+                    // Check whether the impl imposes obligations we have to worry about.
+                    let impl_bounds = tcx.predicates_of(impl_);
+                    let impl_bounds = impl_bounds.instantiate(tcx, impl_substs);
+
+                    let impl_bounds = ocx.normalize(&cause, param_env, impl_bounds);
+
+                    let impl_obligations = traits::predicates_for_generics(
+                        |_, _| cause.clone(),
+                        param_env,
+                        impl_bounds,
+                    );
+
+                    ocx.register_obligations(impl_obligations);
+
+                    let mut errors = ocx.select_where_possible();
+                    if !errors.is_empty() {
+                        fulfillment_errors.append(&mut errors);
+                        return None;
+                    }
+
+                    // FIXME(fmease): Unsolved vars can escape this InferCtxt snapshot.
+                    Some((assoc_item, def_scope, infcx.resolve_vars_if_possible(impl_substs)))
+                })
+            })
+            .collect();
+
+        if applicable_candidates.len() > 1 {
+            return Err(self.complain_about_ambiguous_inherent_assoc_type(
+                name,
+                applicable_candidates.into_iter().map(|(candidate, ..)| candidate).collect(),
+                span,
+            ));
+        }
+
+        if let Some((assoc_item, def_scope, impl_substs)) = applicable_candidates.pop() {
+            self.check_assoc_ty(assoc_item, name, def_scope, block, span);
+
+            // FIXME(inherent_associated_types): To fully *confirm* the *probed* candidate, we still
+            // need to relate the Self-type with fresh item substs & register region obligations for
+            // regionck to prove/disprove.
+
+            let item_substs =
+                self.create_substs_for_associated_item(span, assoc_item, segment, impl_substs);
+
+            // FIXME(fmease, #106722): Check if the bounds on the parameters of the
+            // associated type hold, if any.
+            let ty = tcx.type_of(assoc_item).subst(tcx, item_substs);
+
+            return Ok(Some((ty, assoc_item)));
+        }
+
+        Err(self.complain_about_inherent_assoc_type_not_found(
+            name,
+            self_ty,
+            candidates,
+            fulfillment_errors,
+            span,
+        ))
+    }
+
     fn lookup_assoc_ty(
         &self,
-        ident: Ident,
+        name: Ident,
         block: hir::HirId,
         span: Span,
         scope: DefId,
     ) -> Option<DefId> {
+        let (item, def_scope) = self.lookup_assoc_ty_unchecked(name, block, scope)?;
+        self.check_assoc_ty(item, name, def_scope, block, span);
+        Some(item)
+    }
+
+    fn lookup_assoc_ty_unchecked(
+        &self,
+        name: Ident,
+        block: hir::HirId,
+        scope: DefId,
+    ) -> Option<(DefId, DefId)> {
         let tcx = self.tcx();
-        let (ident, def_scope) = tcx.adjust_ident_and_get_scope(ident, scope, block);
+        let (ident, def_scope) = tcx.adjust_ident_and_get_scope(name, scope, block);
 
         // We have already adjusted the item name above, so compare with `ident.normalize_to_macros_2_0()` instead
         // of calling `find_by_name_and_kind`.
@@ -2249,20 +2330,86 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 && i.ident(tcx).normalize_to_macros_2_0() == ident
         })?;
 
+        Some((item.def_id, def_scope))
+    }
+
+    fn check_assoc_ty(
+        &self,
+        item: DefId,
+        name: Ident,
+        def_scope: DefId,
+        block: hir::HirId,
+        span: Span,
+    ) {
+        let tcx = self.tcx();
         let kind = DefKind::AssocTy;
-        if !item.visibility(tcx).is_accessible_from(def_scope, tcx) {
-            let kind = kind.descr(item.def_id);
-            let msg = format!("{kind} `{ident}` is private");
-            let def_span = self.tcx().def_span(item.def_id);
+
+        if !tcx.visibility(item).is_accessible_from(def_scope, tcx) {
+            let kind = tcx.def_kind_descr(kind, item);
+            let msg = format!("{kind} `{name}` is private");
+            let def_span = tcx.def_span(item);
             tcx.sess
                 .struct_span_err_with_code(span, &msg, rustc_errors::error_code!(E0624))
                 .span_label(span, &format!("private {kind}"))
                 .span_label(def_span, &format!("{kind} defined here"))
                 .emit();
         }
-        tcx.check_stability(item.def_id, Some(block), span, None);
+        tcx.check_stability(item, Some(block), span, None);
+    }
+
+    fn probe_traits_that_match_assoc_ty(
+        &self,
+        qself_ty: Ty<'tcx>,
+        assoc_ident: Ident,
+    ) -> Vec<String> {
+        let tcx = self.tcx();
+
+        // In contexts that have no inference context, just make a new one.
+        // We do need a local variable to store it, though.
+        let infcx_;
+        let infcx = if let Some(infcx) = self.infcx() {
+            infcx
+        } else {
+            assert!(!qself_ty.needs_infer());
+            infcx_ = tcx.infer_ctxt().build();
+            &infcx_
+        };
 
-        Some(item.def_id)
+        tcx.all_traits()
+            .filter(|trait_def_id| {
+                // Consider only traits with the associated type
+                tcx.associated_items(*trait_def_id)
+                        .in_definition_order()
+                        .any(|i| {
+                            i.kind.namespace() == Namespace::TypeNS
+                                && i.ident(tcx).normalize_to_macros_2_0() == assoc_ident
+                                && matches!(i.kind, ty::AssocKind::Type)
+                        })
+                    // Consider only accessible traits
+                    && tcx.visibility(*trait_def_id)
+                        .is_accessible_from(self.item_def_id(), tcx)
+                    && tcx.all_impls(*trait_def_id)
+                        .any(|impl_def_id| {
+                            let trait_ref = tcx.impl_trait_ref(impl_def_id);
+                            trait_ref.map_or(false, |trait_ref| {
+                                let impl_ = trait_ref.subst(
+                                    tcx,
+                                    infcx.fresh_substs_for_item(DUMMY_SP, impl_def_id),
+                                );
+                                infcx
+                                    .can_eq(
+                                        ty::ParamEnv::empty(),
+                                        impl_.self_ty(),
+                                        // Must fold past escaping bound vars too,
+                                        // since we have those at this point in astconv.
+                                        tcx.fold_regions(qself_ty, |_, _| tcx.lifetimes.re_erased),
+                                    )
+                            })
+                            && tcx.impl_polarity(impl_def_id) != ty::ImplPolarity::Negative
+                        })
+            })
+            .map(|trait_def_id| tcx.def_path_str(trait_def_id))
+            .collect()
     }
 
     fn qpath_to_ty(
@@ -2323,7 +2470,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 &[path_str],
                 item_segment.ident.name,
             );
-            return tcx.ty_error_with_guaranteed(reported)
+            return tcx.ty_error(reported)
         };
 
         debug!("qpath_to_ty: self_type={:?}", self_ty);
@@ -2589,6 +2736,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         &self,
         opt_self_ty: Option<Ty<'tcx>>,
         path: &hir::Path<'_>,
+        hir_id: hir::HirId,
         permit_variants: bool,
     ) -> Ty<'tcx> {
         let tcx = self.tcx();
@@ -2652,17 +2800,32 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     }
                 });
 
-                let def_id = def_id.expect_local();
-                let item_def_id = tcx.hir().ty_param_owner(def_id);
-                let generics = tcx.generics_of(item_def_id);
-                let index = generics.param_def_id_to_index[&def_id.to_def_id()];
-                tcx.mk_ty_param(index, tcx.hir().ty_param_name(def_id))
+                match tcx.named_bound_var(hir_id) {
+                    Some(rbv::ResolvedArg::LateBound(debruijn, index, _)) => {
+                        let name =
+                            tcx.hir().name(tcx.hir().local_def_id_to_hir_id(def_id.expect_local()));
+                        let br = ty::BoundTy {
+                            var: ty::BoundVar::from_u32(index),
+                            kind: ty::BoundTyKind::Param(def_id, name),
+                        };
+                        tcx.mk_bound(debruijn, br)
+                    }
+                    Some(rbv::ResolvedArg::EarlyBound(_)) => {
+                        let def_id = def_id.expect_local();
+                        let item_def_id = tcx.hir().ty_param_owner(def_id);
+                        let generics = tcx.generics_of(item_def_id);
+                        let index = generics.param_def_id_to_index[&def_id.to_def_id()];
+                        tcx.mk_ty_param(index, tcx.hir().ty_param_name(def_id))
+                    }
+                    Some(rbv::ResolvedArg::Error(guar)) => tcx.ty_error(guar),
+                    arg => bug!("unexpected bound var resolution for {hir_id:?}: {arg:?}"),
+                }
             }
             Res::SelfTyParam { .. } => {
                 // `Self` in trait or type alias.
                 assert_eq!(opt_self_ty, None);
                 self.prohibit_generics(path.segments.iter(), |err| {
-                    if let [hir::PathSegment { args: Some(args), ident, .. }] = &path.segments[..] {
+                    if let [hir::PathSegment { args: Some(args), ident, .. }] = &path.segments {
                         err.span_suggestion_verbose(
                             ident.span.shrink_to_hi().to(args.span_ext),
                             "the `Self` type doesn't accept type parameters",
@@ -2677,7 +2840,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 // `Self` in impl (we know the concrete type).
                 assert_eq!(opt_self_ty, None);
                 // Try to evaluate any array length constants.
-                let ty = tcx.at(span).type_of(def_id);
+                let ty = tcx.at(span).type_of(def_id).subst_identity();
                 let span_of_impl = tcx.span_of_impl(def_id);
                 self.prohibit_generics(path.segments.iter(), |err| {
                     let def_id = match *ty.kind() {
@@ -2766,7 +2929,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     {
                         err.span_note(impl_.self_ty.span, "not a concrete type");
                     }
-                    tcx.ty_error_with_guaranteed(err.emit())
+                    tcx.ty_error(err.emit())
                 } else {
                     ty
                 }
@@ -2819,7 +2982,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     .sess
                     .delay_span_bug(path.span, "path with `Res::Err` but no error emitted");
                 self.set_tainted_by_errors(e);
-                self.tcx().ty_error_with_guaranteed(e)
+                self.tcx().ty_error(e)
             }
             _ => span_bug!(span, "unexpected resolution: {:?}", path.res),
         }
@@ -2855,7 +3018,9 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 tcx.mk_ref(r, ty::TypeAndMut { ty: t, mutbl: mt.mutbl })
             }
             hir::TyKind::Never => tcx.types.never,
-            hir::TyKind::Tup(fields) => tcx.mk_tup(fields.iter().map(|t| self.ast_ty_to_ty(t))),
+            hir::TyKind::Tup(fields) => {
+                tcx.mk_tup_from_iter(fields.iter().map(|t| self.ast_ty_to_ty(t)))
+            }
             hir::TyKind::BareFn(bf) => {
                 require_c_abi_if_c_variadic(tcx, bf.decl, bf.abi, ast_ty.span);
 
@@ -2874,12 +3039,13 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     TraitObjectSyntax::Dyn | TraitObjectSyntax::None => ty::Dyn,
                     TraitObjectSyntax::DynStar => ty::DynStar,
                 };
+
                 self.conv_object_ty_poly_trait_ref(ast_ty.span, bounds, lifetime, borrowed, repr)
             }
             hir::TyKind::Path(hir::QPath::Resolved(maybe_qself, path)) => {
                 debug!(?maybe_qself, ?path);
                 let opt_self_ty = maybe_qself.as_ref().map(|qself| self.ast_ty_to_ty(qself));
-                self.res_to_ty(opt_self_ty, path, false)
+                self.res_to_ty(opt_self_ty, path, ast_ty.hir_id, false)
             }
             &hir::TyKind::OpaqueDef(item_id, lifetimes, in_trait) => {
                 let opaque_ty = tcx.hir().item(item_id);
@@ -2897,7 +3063,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 let ty = self.ast_ty_to_ty_inner(qself, false, true);
                 self.associated_path_to_ty(ast_ty.hir_id, ast_ty.span, ty, qself, segment, false)
                     .map(|(ty, _, _)| ty)
-                    .unwrap_or_else(|_| tcx.ty_error())
+                    .unwrap_or_else(|guar| tcx.ty_error(guar))
             }
             &hir::TyKind::Path(hir::QPath::LangItem(lang_item, span, _)) => {
                 let def_id = tcx.require_lang_item(lang_item, Some(span));
@@ -2911,7 +3077,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     None,
                     ty::BoundConstness::NotConst,
                 );
-                EarlyBinder(tcx.at(span).type_of(def_id)).subst(tcx, substs)
+                tcx.at(span).type_of(def_id).subst(tcx, substs)
             }
             hir::TyKind::Array(ty, length) => {
                 let length = match length {
@@ -2921,20 +3087,20 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     }
                 };
 
-                tcx.mk_ty(ty::Array(self.ast_ty_to_ty(ty), length))
+                tcx.mk_array_with_const_len(self.ast_ty_to_ty(ty), length)
             }
             hir::TyKind::Typeof(e) => {
-                let ty_erased = tcx.type_of(e.def_id);
+                let ty_erased = tcx.type_of(e.def_id).subst_identity();
                 let ty = tcx.fold_regions(ty_erased, |r, _| {
                     if r.is_erased() { tcx.lifetimes.re_static } else { r }
                 });
                 let span = ast_ty.span;
-                tcx.sess.emit_err(TypeofReservedKeywordUsed {
-                    span,
-                    ty,
-                    opt_sugg: Some((span, Applicability::MachineApplicable))
-                        .filter(|_| ty.is_suggestable(tcx, false)),
-                });
+                let (ty, opt_sugg) = if let Some(ty) = ty.make_suggestable(tcx, false) {
+                    (ty, Some((span, Applicability::MachineApplicable)))
+                } else {
+                    (ty, None)
+                };
+                tcx.sess.emit_err(TypeofReservedKeywordUsed { span, ty, opt_sugg });
 
                 ty
             }
@@ -2945,7 +3111,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 // handled specially and will not descend into this routine.
                 self.ty_infer(None, ast_ty.span)
             }
-            hir::TyKind::Err => tcx.ty_error(),
+            hir::TyKind::Err(guar) => tcx.ty_error(*guar),
         };
 
         self.record_ty(ast_ty.hir_id, result_ty, ast_ty.span);
@@ -3052,7 +3218,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
 
         debug!(?output_ty);
 
-        let fn_ty = tcx.mk_fn_sig(input_tys.into_iter(), output_ty, decl.c_variadic, unsafety, abi);
+        let fn_ty = tcx.mk_fn_sig(input_tys, output_ty, decl.c_variadic, unsafety, abi);
         let bare_fn_ty = ty::Binder::bind_with_vars(fn_ty, bound_vars);
 
         if !self.allow_ty_infer() && !(visitor.0.is_empty() && infer_replacements.is_empty()) {
@@ -3124,8 +3290,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
 
         let hir::Node::ImplItem(hir::ImplItem { kind: hir::ImplItemKind::Fn(..), ident, .. }) =
             hir.get(fn_hir_id) else { return None };
-        let hir::Node::Item(hir::Item { kind: hir::ItemKind::Impl(i), .. }) =
-                hir.get_parent(fn_hir_id) else { bug!("ImplItem should have Impl parent") };
+        let i = hir.get_parent(fn_hir_id).expect_item().expect_impl();
 
         let trait_ref = self.instantiate_mono_trait_ref(
             i.of_trait.as_ref()?,
@@ -3140,7 +3305,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             trait_ref.def_id,
         )?;
 
-        let fn_sig = tcx.bound_fn_sig(assoc.def_id).subst(
+        let fn_sig = tcx.fn_sig(assoc.def_id).subst(
             tcx,
             trait_ref.substs.extend_to(tcx, assoc.def_id, |param, _| tcx.mk_param_from_def(param)),
         );
@@ -3336,3 +3501,36 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         }
     }
 }
+
+pub trait InferCtxtExt<'tcx> {
+    fn fresh_item_substs(&self, def_id: DefId) -> SubstsRef<'tcx>;
+}
+
+impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
+    fn fresh_item_substs(&self, def_id: DefId) -> SubstsRef<'tcx> {
+        InternalSubsts::for_item(self.tcx, def_id, |param, _| match param.kind {
+            GenericParamDefKind::Lifetime => self.tcx.lifetimes.re_erased.into(),
+            GenericParamDefKind::Type { .. } => self
+                .next_ty_var(TypeVariableOrigin {
+                    kind: TypeVariableOriginKind::SubstitutionPlaceholder,
+                    span: self.tcx.def_span(def_id),
+                })
+                .into(),
+            GenericParamDefKind::Const { .. } => {
+                let span = self.tcx.def_span(def_id);
+                let origin = ConstVariableOrigin {
+                    kind: ConstVariableOriginKind::SubstitutionPlaceholder,
+                    span,
+                };
+                self.next_const_var(
+                    self.tcx
+                        .type_of(param.def_id)
+                        .no_bound_vars()
+                        .expect("const parameter types cannot be generic"),
+                    origin,
+                )
+                .into()
+            }
+        })
+    }
+}
diff --git a/compiler/rustc_hir_analysis/src/autoderef.rs b/compiler/rustc_hir_analysis/src/autoderef.rs
index 730560cc6..ba2d4319a 100644
--- a/compiler/rustc_hir_analysis/src/autoderef.rs
+++ b/compiler/rustc_hir_analysis/src/autoderef.rs
@@ -2,11 +2,11 @@ use crate::errors::AutoDerefReachedRecursionLimit;
 use crate::traits::query::evaluate_obligation::InferCtxtExt;
 use crate::traits::NormalizeExt;
 use crate::traits::{self, TraitEngine, TraitEngineExt};
-use rustc_hir as hir;
 use rustc_infer::infer::InferCtxt;
-use rustc_middle::ty::TypeVisitable;
+use rustc_middle::ty::TypeVisitableExt;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_session::Limit;
+use rustc_span::def_id::LocalDefId;
 use rustc_span::def_id::LOCAL_CRATE;
 use rustc_span::Span;
 
@@ -28,7 +28,7 @@ pub struct Autoderef<'a, 'tcx> {
     // Meta infos:
     infcx: &'a InferCtxt<'tcx>,
     span: Span,
-    body_id: hir::HirId,
+    body_id: LocalDefId,
     param_env: ty::ParamEnv<'tcx>,
 
     // Current state:
@@ -96,14 +96,14 @@ impl<'a, 'tcx> Autoderef<'a, 'tcx> {
     pub fn new(
         infcx: &'a InferCtxt<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
-        body_id: hir::HirId,
+        body_def_id: LocalDefId,
         span: Span,
         base_ty: Ty<'tcx>,
     ) -> Autoderef<'a, 'tcx> {
         Autoderef {
             infcx,
             span,
-            body_id,
+            body_id: body_def_id,
             param_env,
             state: AutoderefSnapshot {
                 steps: vec![],
diff --git a/compiler/rustc_hir_analysis/src/check/check.rs b/compiler/rustc_hir_analysis/src/check/check.rs
index abc1c2d7b..848828175 100644
--- a/compiler/rustc_hir_analysis/src/check/check.rs
+++ b/compiler/rustc_hir_analysis/src/check/check.rs
@@ -14,21 +14,24 @@ use rustc_hir::{ItemKind, Node, PathSegment};
 use rustc_infer::infer::opaque_types::ConstrainOpaqueTypeRegionVisitor;
 use rustc_infer::infer::outlives::env::OutlivesEnvironment;
 use rustc_infer::infer::{DefiningAnchor, RegionVariableOrigin, TyCtxtInferExt};
-use rustc_infer::traits::Obligation;
+use rustc_infer::traits::{Obligation, TraitEngineExt as _};
 use rustc_lint::builtin::REPR_TRANSPARENT_EXTERNAL_PRIVATE_FIELDS;
 use rustc_middle::hir::nested_filter;
 use rustc_middle::middle::stability::EvalResult;
 use rustc_middle::ty::layout::{LayoutError, MAX_SIMD_LANES};
 use rustc_middle::ty::subst::GenericArgKind;
 use rustc_middle::ty::util::{Discr, IntTypeExt};
-use rustc_middle::ty::{self, AdtDef, ParamEnv, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable};
+use rustc_middle::ty::{
+    self, AdtDef, DefIdTree, ParamEnv, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable,
+    TypeVisitableExt,
+};
 use rustc_session::lint::builtin::{UNINHABITED_STATIC, UNSUPPORTED_CALLING_CONVENTIONS};
 use rustc_span::symbol::sym;
 use rustc_span::{self, Span};
 use rustc_target::spec::abi::Abi;
 use rustc_trait_selection::traits::error_reporting::on_unimplemented::OnUnimplementedDirective;
 use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt as _;
-use rustc_trait_selection::traits::{self, ObligationCtxt};
+use rustc_trait_selection::traits::{self, ObligationCtxt, TraitEngine, TraitEngineExt as _};
 
 use std::ops::ControlFlow;
 
@@ -91,7 +94,7 @@ fn check_union(tcx: TyCtxt<'_>, def_id: LocalDefId) {
 
 /// Check that the fields of the `union` do not need dropping.
 fn check_union_fields(tcx: TyCtxt<'_>, span: Span, item_def_id: LocalDefId) -> bool {
-    let item_type = tcx.type_of(item_def_id);
+    let item_type = tcx.type_of(item_def_id).subst_identity();
     if let ty::Adt(def, substs) = item_type.kind() {
         assert!(def.is_union());
 
@@ -121,7 +124,7 @@ fn check_union_fields(tcx: TyCtxt<'_>, span: Span, item_def_id: LocalDefId) -> b
 
         let param_env = tcx.param_env(item_def_id);
         for field in &def.non_enum_variant().fields {
-            let field_ty = field.ty(tcx, substs);
+            let field_ty = tcx.normalize_erasing_regions(param_env, field.ty(tcx, substs));
 
             if !allowed_union_field(field_ty, tcx, param_env) {
                 let (field_span, ty_span) = match tcx.hir().get_if_local(field.did) {
@@ -168,22 +171,14 @@ fn check_static_inhabited(tcx: TyCtxt<'_>, def_id: LocalDefId) {
     // would be enough to check this for `extern` statics, as statics with an initializer will
     // have UB during initialization if they are uninhabited, but there also seems to be no good
     // reason to allow any statics to be uninhabited.
-    let ty = tcx.type_of(def_id);
+    let ty = tcx.type_of(def_id).subst_identity();
     let span = tcx.def_span(def_id);
     let layout = match tcx.layout_of(ParamEnv::reveal_all().and(ty)) {
         Ok(l) => l,
         // Foreign statics that overflow their allowed size should emit an error
         Err(LayoutError::SizeOverflow(_))
-            if {
-                let node = tcx.hir().get_by_def_id(def_id);
-                matches!(
-                    node,
-                    hir::Node::ForeignItem(hir::ForeignItem {
-                        kind: hir::ForeignItemKind::Static(..),
-                        ..
-                    })
-                )
-            } =>
+            if matches!(tcx.def_kind(def_id), DefKind::Static(_)
+                if tcx.def_kind(tcx.local_parent(def_id)) == DefKind::ForeignMod) =>
         {
             tcx.sess
                 .struct_span_err(span, "extern static is too large for the current architecture")
@@ -215,7 +210,7 @@ fn check_static_inhabited(tcx: TyCtxt<'_>, def_id: LocalDefId) {
 fn check_opaque(tcx: TyCtxt<'_>, id: hir::ItemId) {
     let item = tcx.hir().item(id);
     let hir::ItemKind::OpaqueTy(hir::OpaqueTy { origin, .. }) = item.kind else {
-        tcx.sess.delay_span_bug(tcx.hir().span(id.hir_id()), "expected opaque item");
+        tcx.sess.delay_span_bug(item.span, "expected opaque item");
         return;
     };
 
@@ -233,7 +228,7 @@ fn check_opaque(tcx: TyCtxt<'_>, id: hir::ItemId) {
     if !tcx.features().impl_trait_projections {
         check_opaque_for_inheriting_lifetimes(tcx, item.owner_id.def_id, span);
     }
-    if tcx.type_of(item.owner_id.def_id).references_error() {
+    if tcx.type_of(item.owner_id.def_id).subst_identity().references_error() {
         return;
     }
     if check_opaque_for_cycles(tcx, item.owner_id.def_id, substs, span, &origin).is_err() {
@@ -261,7 +256,7 @@ pub(super) fn check_opaque_for_inheriting_lifetimes(
         selftys: Vec<(Span, Option<String>)>,
     }
 
-    impl<'tcx> ty::visit::TypeVisitor<'tcx> for ProhibitOpaqueVisitor<'tcx> {
+    impl<'tcx> ty::visit::TypeVisitor<TyCtxt<'tcx>> for ProhibitOpaqueVisitor<'tcx> {
         type BreakTy = Ty<'tcx>;
 
         fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
@@ -412,7 +407,6 @@ fn check_opaque_meets_bounds<'tcx>(
     span: Span,
     origin: &hir::OpaqueTyOrigin,
 ) {
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
     let defining_use_anchor = match *origin {
         hir::OpaqueTyOrigin::FnReturn(did) | hir::OpaqueTyOrigin::AsyncFn(did) => did,
         hir::OpaqueTyOrigin::TyAlias => def_id,
@@ -432,17 +426,18 @@ fn check_opaque_meets_bounds<'tcx>(
     //
     // FIXME: Consider wrapping the hidden type in an existential `Binder` and instantiating it
     // here rather than using ReErased.
-    let hidden_ty = tcx.bound_type_of(def_id.to_def_id()).subst(tcx, substs);
+    let hidden_ty = tcx.type_of(def_id.to_def_id()).subst(tcx, substs);
     let hidden_ty = tcx.fold_regions(hidden_ty, |re, _dbi| match re.kind() {
         ty::ReErased => infcx.next_region_var(RegionVariableOrigin::MiscVariable(span)),
         _ => re,
     });
 
-    let misc_cause = traits::ObligationCause::misc(span, hir_id);
+    let misc_cause = traits::ObligationCause::misc(span, def_id);
 
     match ocx.eq(&misc_cause, param_env, opaque_ty, hidden_ty) {
         Ok(()) => {}
         Err(ty_err) => {
+            let ty_err = ty_err.to_string(tcx);
             tcx.sess.delay_span_bug(
                 span,
                 &format!("could not unify `{hidden_ty}` with revealed type:\n{ty_err}"),
@@ -498,7 +493,7 @@ fn is_enum_of_nonnullable_ptr<'tcx>(
 
 fn check_static_linkage(tcx: TyCtxt<'_>, def_id: LocalDefId) {
     if tcx.codegen_fn_attrs(def_id).import_linkage.is_some() {
-        if match tcx.type_of(def_id).kind() {
+        if match tcx.type_of(def_id).subst_identity().kind() {
             ty::RawPtr(_) => false,
             ty::Adt(adt_def, substs) => !is_enum_of_nonnullable_ptr(tcx, *adt_def, *substs),
             _ => true,
@@ -529,45 +524,34 @@ fn check_item_type(tcx: TyCtxt<'_>, id: hir::ItemId) {
             check_enum(tcx, id.owner_id.def_id);
         }
         DefKind::Fn => {} // entirely within check_item_body
-        DefKind::Impl => {
-            let it = tcx.hir().item(id);
-            let hir::ItemKind::Impl(impl_) = it.kind else { return };
-            debug!("ItemKind::Impl {} with id {:?}", it.ident, it.owner_id);
-            if let Some(impl_trait_ref) = tcx.impl_trait_ref(it.owner_id) {
+        DefKind::Impl { of_trait } => {
+            if of_trait && let Some(impl_trait_ref) = tcx.impl_trait_ref(id.owner_id) {
                 check_impl_items_against_trait(
                     tcx,
-                    it.span,
-                    it.owner_id.def_id,
+                    id.owner_id.def_id,
                     impl_trait_ref.subst_identity(),
-                    &impl_.items,
                 );
-                check_on_unimplemented(tcx, it);
+                check_on_unimplemented(tcx, id);
             }
         }
         DefKind::Trait => {
-            let it = tcx.hir().item(id);
-            let hir::ItemKind::Trait(_, _, _, _, items) = it.kind else {
-                return;
-            };
-            check_on_unimplemented(tcx, it);
-
-            for item in items.iter() {
-                let item = tcx.hir().trait_item(item.id);
-                match &item.kind {
-                    hir::TraitItemKind::Fn(sig, _) => {
-                        let abi = sig.header.abi;
-                        fn_maybe_err(tcx, item.ident.span, abi);
+            let assoc_items = tcx.associated_items(id.owner_id);
+            check_on_unimplemented(tcx, id);
+
+            for &assoc_item in assoc_items.in_definition_order() {
+                match assoc_item.kind {
+                    ty::AssocKind::Fn => {
+                        let abi = tcx.fn_sig(assoc_item.def_id).skip_binder().abi();
+                        fn_maybe_err(tcx, assoc_item.ident(tcx).span, abi);
                     }
-                    hir::TraitItemKind::Type(.., Some(default)) => {
-                        let assoc_item = tcx.associated_item(item.owner_id);
+                    ty::AssocKind::Type if assoc_item.defaultness(tcx).has_value() => {
                         let trait_substs =
-                            InternalSubsts::identity_for_item(tcx, it.owner_id.to_def_id());
+                            InternalSubsts::identity_for_item(tcx, id.owner_id.to_def_id());
                         let _: Result<_, rustc_errors::ErrorGuaranteed> = check_type_bounds(
                             tcx,
                             assoc_item,
                             assoc_item,
-                            default.span,
-                            tcx.mk_trait_ref(it.owner_id.to_def_id(), trait_substs),
+                            tcx.mk_trait_ref(id.owner_id.to_def_id(), trait_substs),
                         );
                     }
                     _ => {}
@@ -595,7 +579,7 @@ fn check_item_type(tcx: TyCtxt<'_>, id: hir::ItemId) {
             }
         }
         DefKind::TyAlias => {
-            let pty_ty = tcx.type_of(id.owner_id);
+            let pty_ty = tcx.type_of(id.owner_id).subst_identity();
             let generics = tcx.generics_of(id.owner_id);
             check_type_params_are_used(tcx, &generics, pty_ty);
         }
@@ -606,59 +590,66 @@ fn check_item_type(tcx: TyCtxt<'_>, id: hir::ItemId) {
             };
             check_abi(tcx, it.hir_id(), it.span, abi);
 
-            if abi == Abi::RustIntrinsic {
-                for item in items {
-                    let item = tcx.hir().foreign_item(item.id);
-                    intrinsic::check_intrinsic_type(tcx, item);
-                }
-            } else if abi == Abi::PlatformIntrinsic {
-                for item in items {
-                    let item = tcx.hir().foreign_item(item.id);
-                    intrinsic::check_platform_intrinsic_type(tcx, item);
+            match abi {
+                Abi::RustIntrinsic => {
+                    for item in items {
+                        let item = tcx.hir().foreign_item(item.id);
+                        intrinsic::check_intrinsic_type(tcx, item);
+                    }
                 }
-            } else {
-                for item in items {
-                    let def_id = item.id.owner_id.def_id;
-                    let generics = tcx.generics_of(def_id);
-                    let own_counts = generics.own_counts();
-                    if generics.params.len() - own_counts.lifetimes != 0 {
-                        let (kinds, kinds_pl, egs) = match (own_counts.types, own_counts.consts) {
-                            (_, 0) => ("type", "types", Some("u32")),
-                            // We don't specify an example value, because we can't generate
-                            // a valid value for any type.
-                            (0, _) => ("const", "consts", None),
-                            _ => ("type or const", "types or consts", None),
-                        };
-                        struct_span_err!(
-                            tcx.sess,
-                            item.span,
-                            E0044,
-                            "foreign items may not have {kinds} parameters",
-                        )
-                        .span_label(item.span, &format!("can't have {kinds} parameters"))
-                        .help(
-                            // FIXME: once we start storing spans for type arguments, turn this
-                            // into a suggestion.
-                            &format!(
-                                "replace the {} parameters with concrete {}{}",
-                                kinds,
-                                kinds_pl,
-                                egs.map(|egs| format!(" like `{}`", egs)).unwrap_or_default(),
-                            ),
-                        )
-                        .emit();
+
+                Abi::PlatformIntrinsic => {
+                    for item in items {
+                        let item = tcx.hir().foreign_item(item.id);
+                        intrinsic::check_platform_intrinsic_type(tcx, item);
                     }
+                }
 
-                    let item = tcx.hir().foreign_item(item.id);
-                    match &item.kind {
-                        hir::ForeignItemKind::Fn(fn_decl, _, _) => {
-                            require_c_abi_if_c_variadic(tcx, fn_decl, abi, item.span);
+                _ => {
+                    for item in items {
+                        let def_id = item.id.owner_id.def_id;
+                        let generics = tcx.generics_of(def_id);
+                        let own_counts = generics.own_counts();
+                        if generics.params.len() - own_counts.lifetimes != 0 {
+                            let (kinds, kinds_pl, egs) = match (own_counts.types, own_counts.consts)
+                            {
+                                (_, 0) => ("type", "types", Some("u32")),
+                                // We don't specify an example value, because we can't generate
+                                // a valid value for any type.
+                                (0, _) => ("const", "consts", None),
+                                _ => ("type or const", "types or consts", None),
+                            };
+                            struct_span_err!(
+                                tcx.sess,
+                                item.span,
+                                E0044,
+                                "foreign items may not have {kinds} parameters",
+                            )
+                            .span_label(item.span, &format!("can't have {kinds} parameters"))
+                            .help(
+                                // FIXME: once we start storing spans for type arguments, turn this
+                                // into a suggestion.
+                                &format!(
+                                    "replace the {} parameters with concrete {}{}",
+                                    kinds,
+                                    kinds_pl,
+                                    egs.map(|egs| format!(" like `{}`", egs)).unwrap_or_default(),
+                                ),
+                            )
+                            .emit();
                         }
-                        hir::ForeignItemKind::Static(..) => {
-                            check_static_inhabited(tcx, def_id);
-                            check_static_linkage(tcx, def_id);
+
+                        let item = tcx.hir().foreign_item(item.id);
+                        match &item.kind {
+                            hir::ForeignItemKind::Fn(fn_decl, _, _) => {
+                                require_c_abi_if_c_variadic(tcx, fn_decl, abi, item.span);
+                            }
+                            hir::ForeignItemKind::Static(..) => {
+                                check_static_inhabited(tcx, def_id);
+                                check_static_linkage(tcx, def_id);
+                            }
+                            _ => {}
                         }
-                        _ => {}
                     }
                 }
             }
@@ -666,13 +657,13 @@ fn check_item_type(tcx: TyCtxt<'_>, id: hir::ItemId) {
         DefKind::GlobalAsm => {
             let it = tcx.hir().item(id);
             let hir::ItemKind::GlobalAsm(asm) = it.kind else { span_bug!(it.span, "DefKind::GlobalAsm but got {:#?}", it) };
-            InlineAsmCtxt::new_global_asm(tcx).check_asm(asm, id.hir_id());
+            InlineAsmCtxt::new_global_asm(tcx).check_asm(asm, id.owner_id.def_id);
         }
         _ => {}
     }
 }
 
-pub(super) fn check_on_unimplemented(tcx: TyCtxt<'_>, item: &hir::Item<'_>) {
+pub(super) fn check_on_unimplemented(tcx: TyCtxt<'_>, item: hir::ItemId) {
     // an error would be reported if this fails.
     let _ = OnUnimplementedDirective::of_item(tcx, item.owner_id.to_def_id());
 }
@@ -680,9 +671,9 @@ pub(super) fn check_on_unimplemented(tcx: TyCtxt<'_>, item: &hir::Item<'_>) {
 pub(super) fn check_specialization_validity<'tcx>(
     tcx: TyCtxt<'tcx>,
     trait_def: &ty::TraitDef,
-    trait_item: &ty::AssocItem,
+    trait_item: ty::AssocItem,
     impl_id: DefId,
-    impl_item: &hir::ImplItemRef,
+    impl_item: DefId,
 ) {
     let Ok(ancestors) = trait_def.ancestors(tcx, impl_id) else { return };
     let mut ancestor_impls = ancestors.skip(1).filter_map(|parent| {
@@ -728,10 +719,8 @@ pub(super) fn check_specialization_validity<'tcx>(
 
 fn check_impl_items_against_trait<'tcx>(
     tcx: TyCtxt<'tcx>,
-    full_impl_span: Span,
     impl_id: LocalDefId,
     impl_trait_ref: ty::TraitRef<'tcx>,
-    impl_item_refs: &[hir::ImplItemRef],
 ) {
     // If the trait reference itself is erroneous (so the compilation is going
     // to fail), skip checking the items here -- the `impl_item` table in `tcx`
@@ -740,12 +729,14 @@ fn check_impl_items_against_trait<'tcx>(
         return;
     }
 
+    let impl_item_refs = tcx.associated_item_def_ids(impl_id);
+
     // Negative impls are not expected to have any items
     match tcx.impl_polarity(impl_id) {
         ty::ImplPolarity::Reservation | ty::ImplPolarity::Positive => {}
         ty::ImplPolarity::Negative => {
             if let [first_item_ref, ..] = impl_item_refs {
-                let first_item_span = tcx.hir().impl_item(first_item_ref.id).span;
+                let first_item_span = tcx.def_span(first_item_ref);
                 struct_span_err!(
                     tcx.sess,
                     first_item_span,
@@ -760,50 +751,34 @@ fn check_impl_items_against_trait<'tcx>(
 
     let trait_def = tcx.trait_def(impl_trait_ref.def_id);
 
-    for impl_item in impl_item_refs {
-        let ty_impl_item = tcx.associated_item(impl_item.id.owner_id);
+    for &impl_item in impl_item_refs {
+        let ty_impl_item = tcx.associated_item(impl_item);
         let ty_trait_item = if let Some(trait_item_id) = ty_impl_item.trait_item_def_id {
             tcx.associated_item(trait_item_id)
         } else {
             // Checked in `associated_item`.
-            tcx.sess.delay_span_bug(impl_item.span, "missing associated item in trait");
+            tcx.sess.delay_span_bug(tcx.def_span(impl_item), "missing associated item in trait");
             continue;
         };
-        let impl_item_full = tcx.hir().impl_item(impl_item.id);
-        match impl_item_full.kind {
-            hir::ImplItemKind::Const(..) => {
+        match ty_impl_item.kind {
+            ty::AssocKind::Const => {
                 let _ = tcx.compare_impl_const((
-                    impl_item.id.owner_id.def_id,
+                    impl_item.expect_local(),
                     ty_impl_item.trait_item_def_id.unwrap(),
                 ));
             }
-            hir::ImplItemKind::Fn(..) => {
-                let opt_trait_span = tcx.hir().span_if_local(ty_trait_item.def_id);
-                compare_impl_method(
-                    tcx,
-                    &ty_impl_item,
-                    &ty_trait_item,
-                    impl_trait_ref,
-                    opt_trait_span,
-                );
+            ty::AssocKind::Fn => {
+                compare_impl_method(tcx, ty_impl_item, ty_trait_item, impl_trait_ref);
             }
-            hir::ImplItemKind::Type(impl_ty) => {
-                let opt_trait_span = tcx.hir().span_if_local(ty_trait_item.def_id);
-                compare_impl_ty(
-                    tcx,
-                    &ty_impl_item,
-                    impl_ty.span,
-                    &ty_trait_item,
-                    impl_trait_ref,
-                    opt_trait_span,
-                );
+            ty::AssocKind::Type => {
+                compare_impl_ty(tcx, ty_impl_item, ty_trait_item, impl_trait_ref);
             }
         }
 
         check_specialization_validity(
             tcx,
             trait_def,
-            &ty_trait_item,
+            ty_trait_item,
             impl_id.to_def_id(),
             impl_item,
         );
@@ -817,8 +792,10 @@ fn check_impl_items_against_trait<'tcx>(
             trait_def.must_implement_one_of.as_deref();
 
         for &trait_item_id in tcx.associated_item_def_ids(impl_trait_ref.def_id) {
-            let is_implemented = ancestors
-                .leaf_def(tcx, trait_item_id)
+            let leaf_def = ancestors.leaf_def(tcx, trait_item_id);
+
+            let is_implemented = leaf_def
+                .as_ref()
                 .map_or(false, |node_item| node_item.item.defaultness(tcx).has_value());
 
             if !is_implemented && tcx.impl_defaultness(impl_id).is_final() {
@@ -826,11 +803,13 @@ fn check_impl_items_against_trait<'tcx>(
             }
 
             // true if this item is specifically implemented in this impl
-            let is_implemented_here = ancestors
-                .leaf_def(tcx, trait_item_id)
+            let is_implemented_here = leaf_def
+                .as_ref()
                 .map_or(false, |node_item| !node_item.defining_node.is_from_trait());
 
             if !is_implemented_here {
+                let full_impl_span =
+                    tcx.hir().span_with_body(tcx.hir().local_def_id_to_hir_id(impl_id));
                 match tcx.eval_default_body_stability(trait_item_id, full_impl_span) {
                     EvalResult::Deny { feature, reason, issue, .. } => default_body_is_unstable(
                         tcx,
@@ -854,9 +833,41 @@ fn check_impl_items_against_trait<'tcx>(
                     }
                 }
             }
+
+            if let Some(leaf_def) = &leaf_def
+                && !leaf_def.is_final()
+                && let def_id = leaf_def.item.def_id
+                && tcx.impl_method_has_trait_impl_trait_tys(def_id)
+            {
+                let def_kind = tcx.def_kind(def_id);
+                let descr = tcx.def_kind_descr(def_kind, def_id);
+                let (msg, feature) = if tcx.asyncness(def_id).is_async() {
+                    (
+                        format!("async {descr} in trait cannot be specialized"),
+                        sym::async_fn_in_trait,
+                    )
+                } else {
+                    (
+                        format!(
+                            "{descr} with return-position `impl Trait` in trait cannot be specialized"
+                        ),
+                        sym::return_position_impl_trait_in_trait,
+                    )
+                };
+                tcx.sess
+                    .struct_span_err(tcx.def_span(def_id), msg)
+                    .note(format!(
+                        "specialization behaves in inconsistent and \
+                        surprising ways with `#![feature({feature})]`, \
+                        and for now is disallowed"
+                    ))
+                    .emit();
+            }
         }
 
         if !missing_items.is_empty() {
+            let full_impl_span =
+                tcx.hir().span_with_body(tcx.hir().local_def_id_to_hir_id(impl_id));
             missing_items_err(tcx, tcx.def_span(impl_id), &missing_items, full_impl_span);
         }
 
@@ -876,7 +887,7 @@ fn check_impl_items_against_trait<'tcx>(
 }
 
 pub fn check_simd(tcx: TyCtxt<'_>, sp: Span, def_id: LocalDefId) {
-    let t = tcx.type_of(def_id);
+    let t = tcx.type_of(def_id).subst_identity();
     if let ty::Adt(def, substs) = t.kind()
         && def.is_struct()
     {
@@ -894,7 +905,7 @@ pub fn check_simd(tcx: TyCtxt<'_>, sp: Span, def_id: LocalDefId) {
         }
 
         let len = if let ty::Array(_ty, c) = e.kind() {
-            c.try_eval_usize(tcx, tcx.param_env(def.did()))
+            c.try_eval_target_usize(tcx, tcx.param_env(def.did()))
         } else {
             Some(fields.len() as u64)
         };
@@ -996,7 +1007,7 @@ pub(super) fn check_packed(tcx: TyCtxt<'_>, sp: Span, def: ty::AdtDef<'_>) {
                             &if first {
                                 format!(
                                     "`{}` contains a field of type `{}`",
-                                    tcx.type_of(def.did()),
+                                    tcx.type_of(def.did()).subst_identity(),
                                     ident
                                 )
                             } else {
@@ -1018,7 +1029,7 @@ pub(super) fn check_packed_inner(
     def_id: DefId,
     stack: &mut Vec<DefId>,
 ) -> Option<Vec<(DefId, Span)>> {
-    if let ty::Adt(def, substs) = tcx.type_of(def_id).kind() {
+    if let ty::Adt(def, substs) = tcx.type_of(def_id).subst_identity().kind() {
         if def.is_struct() || def.is_union() {
             if def.repr().align.is_some() {
                 return Some(vec![(def.did(), DUMMY_SP)]);
@@ -1440,7 +1451,7 @@ fn opaque_type_cycle_error(
                     opaques: Vec<DefId>,
                     closures: Vec<DefId>,
                 }
-                impl<'tcx> ty::visit::TypeVisitor<'tcx> for OpaqueTypeCollector {
+                impl<'tcx> ty::visit::TypeVisitor<TyCtxt<'tcx>> for OpaqueTypeCollector {
                     fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
                         match *t.kind() {
                             ty::Alias(ty::Opaque, ty::AliasTy { def_id: def, .. }) => {
@@ -1461,7 +1472,8 @@ fn opaque_type_cycle_error(
                 for def_id in visitor.opaques {
                     let ty_span = tcx.def_span(def_id);
                     if !seen.contains(&ty_span) {
-                        err.span_label(ty_span, &format!("returning this opaque type `{ty}`"));
+                        let descr = if ty.is_impl_trait() { "opaque " } else { "" };
+                        err.span_label(ty_span, &format!("returning this {descr}type `{ty}`"));
                         seen.insert(ty_span);
                     }
                     err.span_label(sp, &format!("returning here with type `{ty}`"));
@@ -1481,7 +1493,7 @@ fn opaque_type_cycle_error(
                                     span,
                                     format!(
                                         "{} captures itself here",
-                                        tcx.def_kind(closure_def_id).descr(closure_def_id)
+                                        tcx.def_descr(closure_def_id)
                                     ),
                                 );
                             }
@@ -1508,3 +1520,34 @@ fn opaque_type_cycle_error(
     }
     err.emit()
 }
+
+pub(super) fn check_generator_obligations(tcx: TyCtxt<'_>, def_id: LocalDefId) {
+    debug_assert!(tcx.sess.opts.unstable_opts.drop_tracking_mir);
+    debug_assert!(matches!(tcx.def_kind(def_id), DefKind::Generator));
+
+    let typeck = tcx.typeck(def_id);
+    let param_env = tcx.param_env(def_id);
+
+    let generator_interior_predicates = &typeck.generator_interior_predicates[&def_id];
+    debug!(?generator_interior_predicates);
+
+    let infcx = tcx
+        .infer_ctxt()
+        // typeck writeback gives us predicates with their regions erased.
+        // As borrowck already has checked lifetimes, we do not need to do it again.
+        .ignoring_regions()
+        // Bind opaque types to `def_id` as they should have been checked by borrowck.
+        .with_opaque_type_inference(DefiningAnchor::Bind(def_id))
+        .build();
+
+    let mut fulfillment_cx = <dyn TraitEngine<'_>>::new(infcx.tcx);
+    for (predicate, cause) in generator_interior_predicates {
+        let obligation = Obligation::new(tcx, cause.clone(), param_env, *predicate);
+        fulfillment_cx.register_predicate_obligation(&infcx, obligation);
+    }
+    let errors = fulfillment_cx.select_all_or_error(&infcx);
+    debug!(?errors);
+    if !errors.is_empty() {
+        infcx.err_ctxt().report_fulfillment_errors(&errors, None);
+    }
+}
diff --git a/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs b/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs
index cfebcceef..691d3f8d9 100644
--- a/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs
+++ b/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs
@@ -8,7 +8,7 @@ use rustc_errors::{
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::intravisit;
-use rustc_hir::{GenericParamKind, ImplItemKind, TraitItemKind};
+use rustc_hir::{GenericParamKind, ImplItemKind};
 use rustc_infer::infer::outlives::env::OutlivesEnvironment;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
 use rustc_infer::infer::{self, InferCtxt, TyCtxtInferExt};
@@ -16,7 +16,8 @@ use rustc_infer::traits::util;
 use rustc_middle::ty::error::{ExpectedFound, TypeError};
 use rustc_middle::ty::util::ExplicitSelf;
 use rustc_middle::ty::{
-    self, DefIdTree, InternalSubsts, Ty, TypeFoldable, TypeFolder, TypeSuperFoldable, TypeVisitable,
+    self, DefIdTree, InternalSubsts, Ty, TypeFoldable, TypeFolder, TypeSuperFoldable,
+    TypeVisitableExt,
 };
 use rustc_middle::ty::{GenericParamDefKind, ToPredicate, TyCtxt};
 use rustc_span::Span;
@@ -33,31 +34,26 @@ use std::iter;
 /// # Parameters
 ///
 /// - `impl_m`: type of the method we are checking
-/// - `impl_m_span`: span to use for reporting errors
 /// - `trait_m`: the method in the trait
 /// - `impl_trait_ref`: the TraitRef corresponding to the trait implementation
 pub(super) fn compare_impl_method<'tcx>(
     tcx: TyCtxt<'tcx>,
-    impl_m: &ty::AssocItem,
-    trait_m: &ty::AssocItem,
+    impl_m: ty::AssocItem,
+    trait_m: ty::AssocItem,
     impl_trait_ref: ty::TraitRef<'tcx>,
-    trait_item_span: Option<Span>,
 ) {
     debug!("compare_impl_method(impl_trait_ref={:?})", impl_trait_ref);
 
-    let impl_m_span = tcx.def_span(impl_m.def_id);
-
     let _: Result<_, ErrorGuaranteed> = try {
-        compare_self_type(tcx, impl_m, impl_m_span, trait_m, impl_trait_ref)?;
-        compare_number_of_generics(tcx, impl_m, trait_m, trait_item_span, false)?;
+        compare_self_type(tcx, impl_m, trait_m, impl_trait_ref)?;
+        compare_number_of_generics(tcx, impl_m, trait_m, false)?;
         compare_generic_param_kinds(tcx, impl_m, trait_m, false)?;
-        compare_number_of_method_arguments(tcx, impl_m, impl_m_span, trait_m, trait_item_span)?;
+        compare_number_of_method_arguments(tcx, impl_m, trait_m)?;
         compare_synthetic_generics(tcx, impl_m, trait_m)?;
-        compare_asyncness(tcx, impl_m, impl_m_span, trait_m, trait_item_span)?;
+        compare_asyncness(tcx, impl_m, trait_m)?;
         compare_method_predicate_entailment(
             tcx,
             impl_m,
-            impl_m_span,
             trait_m,
             impl_trait_ref,
             CheckImpliedWfMode::Check,
@@ -131,12 +127,11 @@ pub(super) fn compare_impl_method<'tcx>(
 ///
 /// Finally we register each of these predicates as an obligation and check that
 /// they hold.
-#[instrument(level = "debug", skip(tcx, impl_m_span, impl_trait_ref))]
+#[instrument(level = "debug", skip(tcx, impl_trait_ref))]
 fn compare_method_predicate_entailment<'tcx>(
     tcx: TyCtxt<'tcx>,
-    impl_m: &ty::AssocItem,
-    impl_m_span: Span,
-    trait_m: &ty::AssocItem,
+    impl_m: ty::AssocItem,
+    trait_m: ty::AssocItem,
     impl_trait_ref: ty::TraitRef<'tcx>,
     check_implied_wf: CheckImpliedWfMode,
 ) -> Result<(), ErrorGuaranteed> {
@@ -147,12 +142,13 @@ fn compare_method_predicate_entailment<'tcx>(
     //
     // FIXME(@lcnr): remove that after removing `cause.body_id` from
     // obligations.
-    let impl_m_hir_id = tcx.hir().local_def_id_to_hir_id(impl_m.def_id.expect_local());
+    let impl_m_def_id = impl_m.def_id.expect_local();
+    let impl_m_span = tcx.def_span(impl_m_def_id);
     let cause = ObligationCause::new(
         impl_m_span,
-        impl_m_hir_id,
+        impl_m_def_id,
         ObligationCauseCode::CompareImplItemObligation {
-            impl_item_def_id: impl_m.def_id.expect_local(),
+            impl_item_def_id: impl_m_def_id,
             trait_item_def_id: trait_m.def_id,
             kind: impl_m.kind,
         },
@@ -198,9 +194,9 @@ fn compare_method_predicate_entailment<'tcx>(
     // Construct trait parameter environment and then shift it into the placeholder viewpoint.
     // The key step here is to update the caller_bounds's predicates to be
     // the new hybrid bounds we computed.
-    let normalize_cause = traits::ObligationCause::misc(impl_m_span, impl_m_hir_id);
+    let normalize_cause = traits::ObligationCause::misc(impl_m_span, impl_m_def_id);
     let param_env = ty::ParamEnv::new(
-        tcx.intern_predicates(&hybrid_preds.predicates),
+        tcx.mk_predicates(&hybrid_preds.predicates),
         Reveal::UserFacing,
         hir::Constness::NotConst,
     );
@@ -213,14 +209,14 @@ fn compare_method_predicate_entailment<'tcx>(
 
     let impl_m_own_bounds = impl_m_predicates.instantiate_own(tcx, impl_to_placeholder_substs);
     for (predicate, span) in impl_m_own_bounds {
-        let normalize_cause = traits::ObligationCause::misc(span, impl_m_hir_id);
+        let normalize_cause = traits::ObligationCause::misc(span, impl_m_def_id);
         let predicate = ocx.normalize(&normalize_cause, param_env, predicate);
 
         let cause = ObligationCause::new(
             span,
-            impl_m_hir_id,
+            impl_m_def_id,
             ObligationCauseCode::CompareImplItemObligation {
-                impl_item_def_id: impl_m.def_id.expect_local(),
+                impl_item_def_id: impl_m_def_id,
                 trait_item_def_id: trait_m.def_id,
                 kind: impl_m.kind,
             },
@@ -246,18 +242,18 @@ fn compare_method_predicate_entailment<'tcx>(
 
     let mut wf_tys = FxIndexSet::default();
 
-    let unnormalized_impl_sig = infcx.replace_bound_vars_with_fresh_vars(
+    let unnormalized_impl_sig = infcx.instantiate_binder_with_fresh_vars(
         impl_m_span,
         infer::HigherRankedType,
-        tcx.fn_sig(impl_m.def_id),
+        tcx.fn_sig(impl_m.def_id).subst_identity(),
     );
     let unnormalized_impl_fty = tcx.mk_fn_ptr(ty::Binder::dummy(unnormalized_impl_sig));
 
-    let norm_cause = ObligationCause::misc(impl_m_span, impl_m_hir_id);
+    let norm_cause = ObligationCause::misc(impl_m_span, impl_m_def_id);
     let impl_sig = ocx.normalize(&norm_cause, param_env, unnormalized_impl_sig);
     debug!("compare_impl_method: impl_fty={:?}", impl_sig);
 
-    let trait_sig = tcx.bound_fn_sig(trait_m.def_id).subst(tcx, trait_to_placeholder_substs);
+    let trait_sig = tcx.fn_sig(trait_m.def_id).subst(tcx, trait_to_placeholder_substs);
     let trait_sig = tcx.liberate_late_bound_regions(impl_m.def_id, trait_sig);
 
     // Next, add all inputs and output as well-formed tys. Importantly,
@@ -311,10 +307,10 @@ fn compare_method_predicate_entailment<'tcx>(
     if !errors.is_empty() {
         match check_implied_wf {
             CheckImpliedWfMode::Check => {
+                let impl_m_hir_id = tcx.hir().local_def_id_to_hir_id(impl_m_def_id);
                 return compare_method_predicate_entailment(
                     tcx,
                     impl_m,
-                    impl_m_span,
                     trait_m,
                     impl_trait_ref,
                     CheckImpliedWfMode::Skip,
@@ -336,7 +332,7 @@ fn compare_method_predicate_entailment<'tcx>(
     let outlives_env = OutlivesEnvironment::with_bounds(
         param_env,
         Some(infcx),
-        infcx.implied_bounds_tys(param_env, impl_m_hir_id, wf_tys.clone()),
+        infcx.implied_bounds_tys(param_env, impl_m_def_id, wf_tys.clone()),
     );
     infcx.process_registered_region_obligations(
         outlives_env.region_bound_pairs(),
@@ -346,12 +342,12 @@ fn compare_method_predicate_entailment<'tcx>(
     if !errors.is_empty() {
         // FIXME(compiler-errors): This can be simplified when IMPLIED_BOUNDS_ENTAILMENT
         // becomes a hard error (i.e. ideally we'd just call `resolve_regions_and_report_errors`
+        let impl_m_hir_id = tcx.hir().local_def_id_to_hir_id(impl_m_def_id);
         match check_implied_wf {
             CheckImpliedWfMode::Check => {
                 return compare_method_predicate_entailment(
                     tcx,
                     impl_m,
-                    impl_m_span,
                     trait_m,
                     impl_trait_ref,
                     CheckImpliedWfMode::Skip,
@@ -371,7 +367,7 @@ fn compare_method_predicate_entailment<'tcx>(
             }
             CheckImpliedWfMode::Skip => {
                 if infcx.tainted_by_errors().is_none() {
-                    infcx.err_ctxt().report_region_errors(impl_m.def_id.expect_local(), &errors);
+                    infcx.err_ctxt().report_region_errors(impl_m_def_id, &errors);
                 }
                 return Err(tcx
                     .sess
@@ -386,8 +382,8 @@ fn compare_method_predicate_entailment<'tcx>(
 fn extract_bad_args_for_implies_lint<'tcx>(
     tcx: TyCtxt<'tcx>,
     errors: &[infer::RegionResolutionError<'tcx>],
-    (trait_m, trait_sig): (&ty::AssocItem, ty::FnSig<'tcx>),
-    (impl_m, impl_sig): (&ty::AssocItem, ty::FnSig<'tcx>),
+    (trait_m, trait_sig): (ty::AssocItem, ty::FnSig<'tcx>),
+    (impl_m, impl_sig): (ty::AssocItem, ty::FnSig<'tcx>),
     hir_id: hir::HirId,
 ) -> Vec<(Span, Option<String>)> {
     let mut blame_generics = vec![];
@@ -420,8 +416,8 @@ fn extract_bad_args_for_implies_lint<'tcx>(
 
     // Map late-bound regions from trait to impl, so the names are right.
     let mapping = std::iter::zip(
-        tcx.fn_sig(trait_m.def_id).bound_vars(),
-        tcx.fn_sig(impl_m.def_id).bound_vars(),
+        tcx.fn_sig(trait_m.def_id).skip_binder().bound_vars(),
+        tcx.fn_sig(impl_m.def_id).skip_binder().bound_vars(),
     )
     .filter_map(|(impl_bv, trait_bv)| {
         if let ty::BoundVariableKind::Region(impl_bv) = impl_bv
@@ -462,21 +458,17 @@ struct RemapLateBound<'a, 'tcx> {
     mapping: &'a FxHashMap<ty::BoundRegionKind, ty::BoundRegionKind>,
 }
 
-impl<'tcx> TypeFolder<'tcx> for RemapLateBound<'_, 'tcx> {
-    fn tcx(&self) -> TyCtxt<'tcx> {
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for RemapLateBound<'_, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
     fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
         if let ty::ReFree(fr) = *r {
-            self.tcx.mk_region(ty::ReFree(ty::FreeRegion {
-                bound_region: self
-                    .mapping
-                    .get(&fr.bound_region)
-                    .copied()
-                    .unwrap_or(fr.bound_region),
-                ..fr
-            }))
+            self.tcx.mk_re_free(
+                fr.scope,
+                self.mapping.get(&fr.bound_region).copied().unwrap_or(fr.bound_region),
+            )
         } else {
             r
         }
@@ -485,7 +477,7 @@ impl<'tcx> TypeFolder<'tcx> for RemapLateBound<'_, 'tcx> {
 
 fn emit_implied_wf_lint<'tcx>(
     tcx: TyCtxt<'tcx>,
-    impl_m: &ty::AssocItem,
+    impl_m: ty::AssocItem,
     hir_id: hir::HirId,
     bad_args: Vec<(Span, Option<String>)>,
 ) {
@@ -532,13 +524,11 @@ enum CheckImpliedWfMode {
 
 fn compare_asyncness<'tcx>(
     tcx: TyCtxt<'tcx>,
-    impl_m: &ty::AssocItem,
-    impl_m_span: Span,
-    trait_m: &ty::AssocItem,
-    trait_item_span: Option<Span>,
+    impl_m: ty::AssocItem,
+    trait_m: ty::AssocItem,
 ) -> Result<(), ErrorGuaranteed> {
     if tcx.asyncness(trait_m.def_id) == hir::IsAsync::Async {
-        match tcx.fn_sig(impl_m.def_id).skip_binder().output().kind() {
+        match tcx.fn_sig(impl_m.def_id).skip_binder().skip_binder().output().kind() {
             ty::Alias(ty::Opaque, ..) => {
                 // allow both `async fn foo()` and `fn foo() -> impl Future`
             }
@@ -547,9 +537,9 @@ fn compare_asyncness<'tcx>(
             }
             _ => {
                 return Err(tcx.sess.emit_err(crate::errors::AsyncTraitImplShouldBeAsync {
-                    span: impl_m_span,
+                    span: tcx.def_span(impl_m.def_id),
                     method_name: trait_m.name,
-                    trait_item_span,
+                    trait_item_span: tcx.hir().span_if_local(trait_m.def_id),
                 }));
             }
         };
@@ -604,19 +594,20 @@ pub(super) fn collect_return_position_impl_trait_in_trait_tys<'tcx>(
 
     // First, check a few of the same things as `compare_impl_method`,
     // just so we don't ICE during substitution later.
-    compare_number_of_generics(tcx, impl_m, trait_m, tcx.hir().span_if_local(impl_m.def_id), true)?;
+    compare_number_of_generics(tcx, impl_m, trait_m, true)?;
     compare_generic_param_kinds(tcx, impl_m, trait_m, true)?;
     check_region_bounds_on_impl_item(tcx, impl_m, trait_m, true)?;
 
     let trait_to_impl_substs = impl_trait_ref.substs;
 
-    let impl_m_hir_id = tcx.hir().local_def_id_to_hir_id(impl_m.def_id.expect_local());
+    let impl_m_def_id = impl_m.def_id.expect_local();
+    let impl_m_hir_id = tcx.hir().local_def_id_to_hir_id(impl_m_def_id);
     let return_span = tcx.hir().fn_decl_by_hir_id(impl_m_hir_id).unwrap().output.span();
     let cause = ObligationCause::new(
         return_span,
-        impl_m_hir_id,
+        impl_m_def_id,
         ObligationCauseCode::CompareImplItemObligation {
-            impl_item_def_id: impl_m.def_id.expect_local(),
+            impl_item_def_id: impl_m_def_id,
             trait_item_def_id: trait_m.def_id,
             kind: impl_m.kind,
         },
@@ -633,14 +624,14 @@ pub(super) fn collect_return_position_impl_trait_in_trait_tys<'tcx>(
     let ocx = ObligationCtxt::new(infcx);
 
     // Normalize the impl signature with fresh variables for lifetime inference.
-    let norm_cause = ObligationCause::misc(return_span, impl_m_hir_id);
+    let norm_cause = ObligationCause::misc(return_span, impl_m_def_id);
     let impl_sig = ocx.normalize(
         &norm_cause,
         param_env,
-        infcx.replace_bound_vars_with_fresh_vars(
+        infcx.instantiate_binder_with_fresh_vars(
             return_span,
             infer::HigherRankedType,
-            tcx.fn_sig(impl_m.def_id),
+            tcx.fn_sig(impl_m.def_id).subst_identity(),
         ),
     );
     impl_sig.error_reported()?;
@@ -650,13 +641,20 @@ pub(super) fn collect_return_position_impl_trait_in_trait_tys<'tcx>(
     // the ImplTraitInTraitCollector, which gathers all of the RPITITs and replaces
     // them with inference variables.
     // We will use these inference variables to collect the hidden types of RPITITs.
-    let mut collector = ImplTraitInTraitCollector::new(&ocx, return_span, param_env, impl_m_hir_id);
+    let mut collector = ImplTraitInTraitCollector::new(&ocx, return_span, param_env, impl_m_def_id);
     let unnormalized_trait_sig = tcx
         .liberate_late_bound_regions(
             impl_m.def_id,
-            tcx.bound_fn_sig(trait_m.def_id).subst(tcx, trait_to_placeholder_substs),
+            tcx.fn_sig(trait_m.def_id).subst(tcx, trait_to_placeholder_substs),
         )
         .fold_with(&mut collector);
+
+    debug_assert_ne!(
+        collector.types.len(),
+        0,
+        "expect >1 RPITITs in call to `collect_return_position_impl_trait_in_trait_tys`"
+    );
+
     let trait_sig = ocx.normalize(&norm_cause, param_env, unnormalized_trait_sig);
     trait_sig.error_reported()?;
     let trait_return_ty = trait_sig.output();
@@ -732,12 +730,11 @@ pub(super) fn collect_return_position_impl_trait_in_trait_tys<'tcx>(
     let outlives_environment = OutlivesEnvironment::with_bounds(
         param_env,
         Some(infcx),
-        infcx.implied_bounds_tys(param_env, impl_m_hir_id, wf_tys),
+        infcx.implied_bounds_tys(param_env, impl_m_def_id, wf_tys),
     );
-    infcx.err_ctxt().check_region_obligations_and_report_errors(
-        impl_m.def_id.expect_local(),
-        &outlives_environment,
-    )?;
+    infcx
+        .err_ctxt()
+        .check_region_obligations_and_report_errors(impl_m_def_id, &outlives_environment)?;
 
     let mut collected_tys = FxHashMap::default();
     for (def_id, (ty, substs)) in collector.types {
@@ -784,19 +781,13 @@ pub(super) fn collect_return_position_impl_trait_in_trait_tys<'tcx>(
                     }
                     let Some(ty::ReEarlyBound(e)) = map.get(&region.into()).map(|r| r.expect_region().kind())
                     else {
-                        tcx
-                            .sess
-                            .delay_span_bug(
-                                return_span,
-                                "expected ReFree to map to ReEarlyBound"
-                            );
-                        return tcx.lifetimes.re_static;
+                        return tcx.mk_re_error_with_message(return_span, "expected ReFree to map to ReEarlyBound")
                     };
-                    tcx.mk_region(ty::ReEarlyBound(ty::EarlyBoundRegion {
+                    tcx.mk_re_early_bound(ty::EarlyBoundRegion {
                         def_id: e.def_id,
                         name: e.name,
                         index: (e.index as usize - num_trait_substs + num_impl_substs) as u32,
-                    }))
+                    })
                 });
                 debug!(%ty);
                 collected_tys.insert(def_id, ty);
@@ -806,7 +797,7 @@ pub(super) fn collect_return_position_impl_trait_in_trait_tys<'tcx>(
                     return_span,
                     format!("could not fully resolve: {ty} => {err:?}"),
                 );
-                collected_tys.insert(def_id, tcx.ty_error_with_guaranteed(reported));
+                collected_tys.insert(def_id, tcx.ty_error(reported));
             }
         }
     }
@@ -819,7 +810,7 @@ struct ImplTraitInTraitCollector<'a, 'tcx> {
     types: FxHashMap<DefId, (Ty<'tcx>, ty::SubstsRef<'tcx>)>,
     span: Span,
     param_env: ty::ParamEnv<'tcx>,
-    body_id: hir::HirId,
+    body_id: LocalDefId,
 }
 
 impl<'a, 'tcx> ImplTraitInTraitCollector<'a, 'tcx> {
@@ -827,20 +818,20 @@ impl<'a, 'tcx> ImplTraitInTraitCollector<'a, 'tcx> {
         ocx: &'a ObligationCtxt<'a, 'tcx>,
         span: Span,
         param_env: ty::ParamEnv<'tcx>,
-        body_id: hir::HirId,
+        body_id: LocalDefId,
     ) -> Self {
         ImplTraitInTraitCollector { ocx, types: FxHashMap::default(), span, param_env, body_id }
     }
 }
 
-impl<'tcx> TypeFolder<'tcx> for ImplTraitInTraitCollector<'_, 'tcx> {
-    fn tcx<'a>(&'a self) -> TyCtxt<'tcx> {
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ImplTraitInTraitCollector<'_, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.ocx.infcx.tcx
     }
 
     fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
         if let ty::Alias(ty::Projection, proj) = ty.kind()
-            && self.tcx().def_kind(proj.def_id) == DefKind::ImplTraitPlaceholder
+            && self.interner().def_kind(proj.def_id) == DefKind::ImplTraitPlaceholder
         {
             if let Some((ty, _)) = self.types.get(&proj.def_id) {
                 return *ty;
@@ -856,7 +847,7 @@ impl<'tcx> TypeFolder<'tcx> for ImplTraitInTraitCollector<'_, 'tcx> {
             });
             self.types.insert(proj.def_id, (infer_ty, proj.substs));
             // Recurse into bounds
-            for (pred, pred_span) in self.tcx().bound_explicit_item_bounds(proj.def_id).subst_iter_copied(self.tcx(), proj.substs) {
+            for (pred, pred_span) in self.interner().bound_explicit_item_bounds(proj.def_id).subst_iter_copied(self.interner(), proj.substs) {
                 let pred = pred.fold_with(self);
                 let pred = self.ocx.normalize(
                     &ObligationCause::misc(self.span, self.body_id),
@@ -865,7 +856,7 @@ impl<'tcx> TypeFolder<'tcx> for ImplTraitInTraitCollector<'_, 'tcx> {
                 );
 
                 self.ocx.register_obligation(traits::Obligation::new(
-                    self.tcx(),
+                    self.interner(),
                     ObligationCause::new(
                         self.span,
                         self.body_id,
@@ -886,8 +877,8 @@ fn report_trait_method_mismatch<'tcx>(
     infcx: &InferCtxt<'tcx>,
     mut cause: ObligationCause<'tcx>,
     terr: TypeError<'tcx>,
-    (trait_m, trait_sig): (&ty::AssocItem, ty::FnSig<'tcx>),
-    (impl_m, impl_sig): (&ty::AssocItem, ty::FnSig<'tcx>),
+    (trait_m, trait_sig): (ty::AssocItem, ty::FnSig<'tcx>),
+    (impl_m, impl_sig): (ty::AssocItem, ty::FnSig<'tcx>),
     impl_trait_ref: ty::TraitRef<'tcx>,
 ) -> ErrorGuaranteed {
     let tcx = infcx.tcx;
@@ -916,7 +907,7 @@ fn report_trait_method_mismatch<'tcx>(
             // When the `impl` receiver is an arbitrary self type, like `self: Box<Self>`, the
             // span points only at the type `Box<Self`>, but we want to cover the whole
             // argument pattern and type.
-            let ImplItemKind::Fn(ref sig, body) = tcx.hir().expect_impl_item(impl_m.def_id.expect_local()).kind else { bug!("{impl_m:?} is not a method") };
+            let (sig, body) = tcx.hir().expect_impl_item(impl_m.def_id.expect_local()).expect_fn();
             let span = tcx
                 .hir()
                 .body_param_names(body)
@@ -980,8 +971,8 @@ fn report_trait_method_mismatch<'tcx>(
 
 fn check_region_bounds_on_impl_item<'tcx>(
     tcx: TyCtxt<'tcx>,
-    impl_m: &ty::AssocItem,
-    trait_m: &ty::AssocItem,
+    impl_m: ty::AssocItem,
+    trait_m: ty::AssocItem,
     delay: bool,
 ) -> Result<(), ErrorGuaranteed> {
     let impl_generics = tcx.generics_of(impl_m.def_id);
@@ -1055,7 +1046,7 @@ fn check_region_bounds_on_impl_item<'tcx>(
             .sess
             .create_err(LifetimesOrBoundsMismatchOnTrait {
                 span,
-                item_kind: assoc_item_kind_str(impl_m),
+                item_kind: assoc_item_kind_str(&impl_m),
                 ident: impl_m.ident(tcx),
                 generics_span,
                 bounds_span,
@@ -1073,17 +1064,17 @@ fn extract_spans_for_error_reporting<'tcx>(
     infcx: &infer::InferCtxt<'tcx>,
     terr: TypeError<'_>,
     cause: &ObligationCause<'tcx>,
-    impl_m: &ty::AssocItem,
-    trait_m: &ty::AssocItem,
+    impl_m: ty::AssocItem,
+    trait_m: ty::AssocItem,
 ) -> (Span, Option<Span>) {
     let tcx = infcx.tcx;
     let mut impl_args = {
-        let ImplItemKind::Fn(sig, _) = &tcx.hir().expect_impl_item(impl_m.def_id.expect_local()).kind else { bug!("{:?} is not a method", impl_m) };
+        let (sig, _) = tcx.hir().expect_impl_item(impl_m.def_id.expect_local()).expect_fn();
         sig.decl.inputs.iter().map(|t| t.span).chain(iter::once(sig.decl.output.span()))
     };
 
     let trait_args = trait_m.def_id.as_local().map(|def_id| {
-        let TraitItemKind::Fn(sig, _) = &tcx.hir().expect_trait_item(def_id).kind else { bug!("{:?} is not a TraitItemKind::Fn", trait_m) };
+        let (sig, _) = tcx.hir().expect_trait_item(def_id).expect_fn();
         sig.decl.inputs.iter().map(|t| t.span).chain(iter::once(sig.decl.output.span()))
     });
 
@@ -1097,9 +1088,8 @@ fn extract_spans_for_error_reporting<'tcx>(
 
 fn compare_self_type<'tcx>(
     tcx: TyCtxt<'tcx>,
-    impl_m: &ty::AssocItem,
-    impl_m_span: Span,
-    trait_m: &ty::AssocItem,
+    impl_m: ty::AssocItem,
+    trait_m: ty::AssocItem,
     impl_trait_ref: ty::TraitRef<'tcx>,
 ) -> Result<(), ErrorGuaranteed> {
     // Try to give more informative error messages about self typing
@@ -1110,17 +1100,17 @@ fn compare_self_type<'tcx>(
     // inscrutable, particularly for cases where one method has no
     // self.
 
-    let self_string = |method: &ty::AssocItem| {
+    let self_string = |method: ty::AssocItem| {
         let untransformed_self_ty = match method.container {
             ty::ImplContainer => impl_trait_ref.self_ty(),
             ty::TraitContainer => tcx.types.self_param,
         };
-        let self_arg_ty = tcx.fn_sig(method.def_id).input(0);
+        let self_arg_ty = tcx.fn_sig(method.def_id).subst_identity().input(0);
         let param_env = ty::ParamEnv::reveal_all();
 
         let infcx = tcx.infer_ctxt().build();
         let self_arg_ty = tcx.liberate_late_bound_regions(method.def_id, self_arg_ty);
-        let can_eq_self = |ty| infcx.can_eq(param_env, untransformed_self_ty, ty).is_ok();
+        let can_eq_self = |ty| infcx.can_eq(param_env, untransformed_self_ty, ty);
         match ExplicitSelf::determine(self_arg_ty, can_eq_self) {
             ExplicitSelf::ByValue => "self".to_owned(),
             ExplicitSelf::ByReference(_, hir::Mutability::Not) => "&self".to_owned(),
@@ -1134,6 +1124,7 @@ fn compare_self_type<'tcx>(
 
         (false, true) => {
             let self_descr = self_string(impl_m);
+            let impl_m_span = tcx.def_span(impl_m.def_id);
             let mut err = struct_span_err!(
                 tcx.sess,
                 impl_m_span,
@@ -1153,6 +1144,7 @@ fn compare_self_type<'tcx>(
 
         (true, false) => {
             let self_descr = self_string(trait_m);
+            let impl_m_span = tcx.def_span(impl_m.def_id);
             let mut err = struct_span_err!(
                 tcx.sess,
                 impl_m_span,
@@ -1198,9 +1190,8 @@ fn compare_self_type<'tcx>(
 /// [`compare_generic_param_kinds`]. This function also does not handle lifetime parameters
 fn compare_number_of_generics<'tcx>(
     tcx: TyCtxt<'tcx>,
-    impl_: &ty::AssocItem,
-    trait_: &ty::AssocItem,
-    trait_span: Option<Span>,
+    impl_: ty::AssocItem,
+    trait_: ty::AssocItem,
     delay: bool,
 ) -> Result<(), ErrorGuaranteed> {
     let trait_own_counts = tcx.generics_of(trait_.def_id).own_counts();
@@ -1220,7 +1211,7 @@ fn compare_number_of_generics<'tcx>(
         ("const", trait_own_counts.consts, impl_own_counts.consts),
     ];
 
-    let item_kind = assoc_item_kind_str(impl_);
+    let item_kind = assoc_item_kind_str(&impl_);
 
     let mut err_occurred = None;
     for (kind, trait_count, impl_count) in matchings {
@@ -1260,6 +1251,7 @@ fn compare_number_of_generics<'tcx>(
                     .collect();
                 (Some(arg_spans), impl_trait_spans)
             } else {
+                let trait_span = tcx.hir().span_if_local(trait_.def_id);
                 (trait_span.map(|s| vec![s]), vec![])
             };
 
@@ -1341,22 +1333,20 @@ fn compare_number_of_generics<'tcx>(
 
 fn compare_number_of_method_arguments<'tcx>(
     tcx: TyCtxt<'tcx>,
-    impl_m: &ty::AssocItem,
-    impl_m_span: Span,
-    trait_m: &ty::AssocItem,
-    trait_item_span: Option<Span>,
+    impl_m: ty::AssocItem,
+    trait_m: ty::AssocItem,
 ) -> Result<(), ErrorGuaranteed> {
     let impl_m_fty = tcx.fn_sig(impl_m.def_id);
     let trait_m_fty = tcx.fn_sig(trait_m.def_id);
-    let trait_number_args = trait_m_fty.inputs().skip_binder().len();
-    let impl_number_args = impl_m_fty.inputs().skip_binder().len();
+    let trait_number_args = trait_m_fty.skip_binder().inputs().skip_binder().len();
+    let impl_number_args = impl_m_fty.skip_binder().inputs().skip_binder().len();
 
     if trait_number_args != impl_number_args {
         let trait_span = trait_m
             .def_id
             .as_local()
             .and_then(|def_id| {
-                let TraitItemKind::Fn(trait_m_sig, _) = &tcx.hir().expect_trait_item(def_id).kind else { bug!("{:?} is not a method", impl_m) };
+                let (trait_m_sig, _) = &tcx.hir().expect_trait_item(def_id).expect_fn();
                 let pos = trait_number_args.saturating_sub(1);
                 trait_m_sig.decl.inputs.get(pos).map(|arg| {
                     if pos == 0 {
@@ -1366,9 +1356,9 @@ fn compare_number_of_method_arguments<'tcx>(
                     }
                 })
             })
-            .or(trait_item_span);
+            .or_else(|| tcx.hir().span_if_local(trait_m.def_id));
 
-        let ImplItemKind::Fn(impl_m_sig, _) = &tcx.hir().expect_impl_item(impl_m.def_id.expect_local()).kind else { bug!("{:?} is not a method", impl_m) };
+        let (impl_m_sig, _) = &tcx.hir().expect_impl_item(impl_m.def_id.expect_local()).expect_fn();
         let pos = impl_number_args.saturating_sub(1);
         let impl_span = impl_m_sig
             .decl
@@ -1381,7 +1371,7 @@ fn compare_number_of_method_arguments<'tcx>(
                     arg.span.with_lo(impl_m_sig.decl.inputs[0].span.lo())
                 }
             })
-            .unwrap_or(impl_m_span);
+            .unwrap_or_else(|| tcx.def_span(impl_m.def_id));
 
         let mut err = struct_span_err!(
             tcx.sess,
@@ -1423,8 +1413,8 @@ fn compare_number_of_method_arguments<'tcx>(
 
 fn compare_synthetic_generics<'tcx>(
     tcx: TyCtxt<'tcx>,
-    impl_m: &ty::AssocItem,
-    trait_m: &ty::AssocItem,
+    impl_m: ty::AssocItem,
+    trait_m: ty::AssocItem,
 ) -> Result<(), ErrorGuaranteed> {
     // FIXME(chrisvittal) Clean up this function, list of FIXME items:
     //     1. Better messages for the span labels
@@ -1504,7 +1494,7 @@ fn compare_synthetic_generics<'tcx>(
                     let _: Option<_> = try {
                         let impl_m = impl_m.def_id.as_local()?;
                         let impl_m = tcx.hir().expect_impl_item(impl_m);
-                        let hir::ImplItemKind::Fn(sig, _) = &impl_m.kind else { unreachable!() };
+                        let (sig, _) = impl_m.expect_fn();
                         let input_tys = sig.decl.inputs;
 
                         struct Visitor(Option<Span>, hir::def_id::LocalDefId);
@@ -1577,8 +1567,8 @@ fn compare_synthetic_generics<'tcx>(
 /// This function does not handle lifetime parameters
 fn compare_generic_param_kinds<'tcx>(
     tcx: TyCtxt<'tcx>,
-    impl_item: &ty::AssocItem,
-    trait_item: &ty::AssocItem,
+    impl_item: ty::AssocItem,
+    trait_item: ty::AssocItem,
     delay: bool,
 ) -> Result<(), ErrorGuaranteed> {
     assert_eq!(impl_item.kind, trait_item.kind);
@@ -1623,7 +1613,11 @@ fn compare_generic_param_kinds<'tcx>(
 
             let make_param_message = |prefix: &str, param: &ty::GenericParamDef| match param.kind {
                 Const { .. } => {
-                    format!("{} const parameter of type `{}`", prefix, tcx.type_of(param.def_id))
+                    format!(
+                        "{} const parameter of type `{}`",
+                        prefix,
+                        tcx.type_of(param.def_id).subst_identity()
+                    )
                 }
                 Type { .. } => format!("{} type parameter", prefix),
                 Lifetime { .. } => unreachable!(),
@@ -1671,14 +1665,12 @@ pub(super) fn compare_impl_const_raw(
 
     // Create a parameter environment that represents the implementation's
     // method.
-    let impl_c_hir_id = tcx.hir().local_def_id_to_hir_id(impl_const_item_def);
-
     // Compute placeholder form of impl and trait const tys.
-    let impl_ty = tcx.type_of(impl_const_item_def.to_def_id());
-    let trait_ty = tcx.bound_type_of(trait_const_item_def).subst(tcx, trait_to_impl_substs);
+    let impl_ty = tcx.type_of(impl_const_item_def.to_def_id()).subst_identity();
+    let trait_ty = tcx.type_of(trait_const_item_def).subst(tcx, trait_to_impl_substs);
     let mut cause = ObligationCause::new(
         impl_c_span,
-        impl_c_hir_id,
+        impl_const_item_def,
         ObligationCauseCode::CompareImplItemObligation {
             impl_item_def_id: impl_const_item_def,
             trait_item_def_id: trait_const_item_def,
@@ -1704,7 +1696,7 @@ pub(super) fn compare_impl_const_raw(
         );
 
         // Locate the Span containing just the type of the offending impl
-        let ImplItemKind::Const(ty, _) = tcx.hir().expect_impl_item(impl_const_item_def).kind else { bug!("{impl_const_item:?} is not a impl const") };
+        let (ty, _) = tcx.hir().expect_impl_item(impl_const_item_def).expect_const();
         cause.span = ty.span;
 
         let mut diag = struct_span_err!(
@@ -1717,7 +1709,7 @@ pub(super) fn compare_impl_const_raw(
 
         let trait_c_span = trait_const_item_def.as_local().map(|trait_c_def_id| {
             // Add a label to the Span containing just the type of the const
-            let TraitItemKind::Const(ty, _) = tcx.hir().expect_trait_item(trait_c_def_id).kind else { bug!("{trait_const_item:?} is not a trait const") };
+            let (ty, _) = tcx.hir().expect_trait_item(trait_c_def_id).expect_const();
             ty.span
         });
 
@@ -1752,23 +1744,17 @@ pub(super) fn compare_impl_const_raw(
 
 pub(super) fn compare_impl_ty<'tcx>(
     tcx: TyCtxt<'tcx>,
-    impl_ty: &ty::AssocItem,
-    impl_ty_span: Span,
-    trait_ty: &ty::AssocItem,
+    impl_ty: ty::AssocItem,
+    trait_ty: ty::AssocItem,
     impl_trait_ref: ty::TraitRef<'tcx>,
-    trait_item_span: Option<Span>,
 ) {
     debug!("compare_impl_type(impl_trait_ref={:?})", impl_trait_ref);
 
     let _: Result<(), ErrorGuaranteed> = try {
-        compare_number_of_generics(tcx, impl_ty, trait_ty, trait_item_span, false)?;
-
+        compare_number_of_generics(tcx, impl_ty, trait_ty, false)?;
         compare_generic_param_kinds(tcx, impl_ty, trait_ty, false)?;
-
-        let sp = tcx.def_span(impl_ty.def_id);
-        compare_type_predicate_entailment(tcx, impl_ty, sp, trait_ty, impl_trait_ref)?;
-
-        check_type_bounds(tcx, trait_ty, impl_ty, impl_ty_span, impl_trait_ref)?;
+        compare_type_predicate_entailment(tcx, impl_ty, trait_ty, impl_trait_ref)?;
+        check_type_bounds(tcx, trait_ty, impl_ty, impl_trait_ref)?;
     };
 }
 
@@ -1776,9 +1762,8 @@ pub(super) fn compare_impl_ty<'tcx>(
 /// instead of associated functions.
 fn compare_type_predicate_entailment<'tcx>(
     tcx: TyCtxt<'tcx>,
-    impl_ty: &ty::AssocItem,
-    impl_ty_span: Span,
-    trait_ty: &ty::AssocItem,
+    impl_ty: ty::AssocItem,
+    trait_ty: ty::AssocItem,
     impl_trait_ref: ty::TraitRef<'tcx>,
 ) -> Result<(), ErrorGuaranteed> {
     let impl_substs = InternalSubsts::identity_for_item(tcx, impl_ty.def_id);
@@ -1799,7 +1784,7 @@ fn compare_type_predicate_entailment<'tcx>(
     // This `HirId` should be used for the `body_id` field on each
     // `ObligationCause` (and the `FnCtxt`). This is what
     // `regionck_item` expects.
-    let impl_ty_hir_id = tcx.hir().local_def_id_to_hir_id(impl_ty.def_id.expect_local());
+    let impl_ty_def_id = impl_ty.def_id.expect_local();
     debug!("compare_type_predicate_entailment: trait_to_impl_substs={:?}", trait_to_impl_substs);
 
     // The predicates declared by the impl definition, the trait and the
@@ -1814,9 +1799,10 @@ fn compare_type_predicate_entailment<'tcx>(
 
     debug!("compare_type_predicate_entailment: bounds={:?}", hybrid_preds);
 
-    let normalize_cause = traits::ObligationCause::misc(impl_ty_span, impl_ty_hir_id);
+    let impl_ty_span = tcx.def_span(impl_ty_def_id);
+    let normalize_cause = traits::ObligationCause::misc(impl_ty_span, impl_ty_def_id);
     let param_env = ty::ParamEnv::new(
-        tcx.intern_predicates(&hybrid_preds.predicates),
+        tcx.mk_predicates(&hybrid_preds.predicates),
         Reveal::UserFacing,
         hir::Constness::NotConst,
     );
@@ -1827,12 +1813,12 @@ fn compare_type_predicate_entailment<'tcx>(
     debug!("compare_type_predicate_entailment: caller_bounds={:?}", param_env.caller_bounds());
 
     for (predicate, span) in impl_ty_own_bounds {
-        let cause = ObligationCause::misc(span, impl_ty_hir_id);
+        let cause = ObligationCause::misc(span, impl_ty_def_id);
         let predicate = ocx.normalize(&cause, param_env, predicate);
 
         let cause = ObligationCause::new(
             span,
-            impl_ty_hir_id,
+            impl_ty_def_id,
             ObligationCauseCode::CompareImplItemObligation {
                 impl_item_def_id: impl_ty.def_id.expect_local(),
                 trait_item_def_id: trait_ty.def_id,
@@ -1877,9 +1863,8 @@ fn compare_type_predicate_entailment<'tcx>(
 #[instrument(level = "debug", skip(tcx))]
 pub(super) fn check_type_bounds<'tcx>(
     tcx: TyCtxt<'tcx>,
-    trait_ty: &ty::AssocItem,
-    impl_ty: &ty::AssocItem,
-    impl_ty_span: Span,
+    trait_ty: ty::AssocItem,
+    impl_ty: ty::AssocItem,
     impl_trait_ref: ty::TraitRef<'tcx>,
 ) -> Result<(), ErrorGuaranteed> {
     // Given
@@ -1888,7 +1873,7 @@ pub(super) fn check_type_bounds<'tcx>(
     //     type Bar<C> =...
     // }
     //
-    // - `impl_trait_ref` would be `<(A, B) as Foo<u32>>
+    // - `impl_trait_ref` would be `<(A, B) as Foo<u32>>`
     // - `impl_ty_substs` would be `[A, B, ^0.0]` (`^0.0` here is the bound var with db 0 and index 0)
     // - `rebased_substs` would be `[(A, B), u32, ^0.0]`, combining the substs from
     //    the *trait* with the generic associated type parameters (as bound vars).
@@ -1930,23 +1915,23 @@ pub(super) fn check_type_bounds<'tcx>(
         smallvec::SmallVec::with_capacity(defs.count());
     InternalSubsts::fill_single(&mut substs, defs, &mut |param, _| match param.kind {
         GenericParamDefKind::Type { .. } => {
-            let kind = ty::BoundTyKind::Param(param.name);
+            let kind = ty::BoundTyKind::Param(param.def_id, param.name);
             let bound_var = ty::BoundVariableKind::Ty(kind);
             bound_vars.push(bound_var);
-            tcx.mk_ty(ty::Bound(
+            tcx.mk_bound(
                 ty::INNERMOST,
                 ty::BoundTy { var: ty::BoundVar::from_usize(bound_vars.len() - 1), kind },
-            ))
+            )
             .into()
         }
         GenericParamDefKind::Lifetime => {
             let kind = ty::BoundRegionKind::BrNamed(param.def_id, param.name);
             let bound_var = ty::BoundVariableKind::Region(kind);
             bound_vars.push(bound_var);
-            tcx.mk_region(ty::ReLateBound(
+            tcx.mk_re_late_bound(
                 ty::INNERMOST,
                 ty::BoundRegion { var: ty::BoundVar::from_usize(bound_vars.len() - 1), kind },
-            ))
+            )
             .into()
         }
         GenericParamDefKind::Const { .. } => {
@@ -1954,17 +1939,17 @@ pub(super) fn check_type_bounds<'tcx>(
             bound_vars.push(bound_var);
             tcx.mk_const(
                 ty::ConstKind::Bound(ty::INNERMOST, ty::BoundVar::from_usize(bound_vars.len() - 1)),
-                tcx.type_of(param.def_id),
+                tcx.type_of(param.def_id).subst_identity(),
             )
             .into()
         }
     });
-    let bound_vars = tcx.mk_bound_variable_kinds(bound_vars.into_iter());
-    let impl_ty_substs = tcx.intern_substs(&substs);
+    let bound_vars = tcx.mk_bound_variable_kinds(&bound_vars);
+    let impl_ty_substs = tcx.mk_substs(&substs);
     let container_id = impl_ty.container_id(tcx);
 
     let rebased_substs = impl_ty_substs.rebase_onto(tcx, container_id, impl_trait_ref.substs);
-    let impl_ty_value = tcx.type_of(impl_ty.def_id);
+    let impl_ty_value = tcx.type_of(impl_ty.def_id).subst_identity();
 
     let param_env = tcx.param_env(impl_ty.def_id);
 
@@ -2000,27 +1985,30 @@ pub(super) fn check_type_bounds<'tcx>(
                 .to_predicate(tcx),
             ),
         };
-        ty::ParamEnv::new(
-            tcx.intern_predicates(&predicates),
-            Reveal::UserFacing,
-            param_env.constness(),
-        )
+        ty::ParamEnv::new(tcx.mk_predicates(&predicates), Reveal::UserFacing, param_env.constness())
     };
     debug!(?normalize_param_env);
 
-    let impl_ty_hir_id = tcx.hir().local_def_id_to_hir_id(impl_ty.def_id.expect_local());
+    let impl_ty_def_id = impl_ty.def_id.expect_local();
     let impl_ty_substs = InternalSubsts::identity_for_item(tcx, impl_ty.def_id);
     let rebased_substs = impl_ty_substs.rebase_onto(tcx, container_id, impl_trait_ref.substs);
 
     let infcx = tcx.infer_ctxt().build();
     let ocx = ObligationCtxt::new(&infcx);
 
-    let assumed_wf_types =
-        ocx.assumed_wf_types(param_env, impl_ty_span, impl_ty.def_id.expect_local());
+    let impl_ty_span = match tcx.hir().get_by_def_id(impl_ty_def_id) {
+        hir::Node::TraitItem(hir::TraitItem {
+            kind: hir::TraitItemKind::Type(_, Some(ty)),
+            ..
+        }) => ty.span,
+        hir::Node::ImplItem(hir::ImplItem { kind: hir::ImplItemKind::Type(ty), .. }) => ty.span,
+        _ => bug!(),
+    };
+    let assumed_wf_types = ocx.assumed_wf_types(param_env, impl_ty_span, impl_ty_def_id);
 
     let normalize_cause = ObligationCause::new(
         impl_ty_span,
-        impl_ty_hir_id,
+        impl_ty_def_id,
         ObligationCauseCode::CheckAssociatedTypeBounds {
             impl_item_def_id: impl_ty.def_id.expect_local(),
             trait_item_def_id: trait_ty.def_id,
@@ -2032,7 +2020,7 @@ pub(super) fn check_type_bounds<'tcx>(
         } else {
             traits::BindingObligation(trait_ty.def_id, span)
         };
-        ObligationCause::new(impl_ty_span, impl_ty_hir_id, code)
+        ObligationCause::new(impl_ty_span, impl_ty_def_id, code)
     };
 
     let obligations = tcx
@@ -2063,7 +2051,7 @@ pub(super) fn check_type_bounds<'tcx>(
 
     // Finally, resolve all regions. This catches wily misuses of
     // lifetime parameters.
-    let implied_bounds = infcx.implied_bounds_tys(param_env, impl_ty_hir_id, assumed_wf_types);
+    let implied_bounds = infcx.implied_bounds_tys(param_env, impl_ty_def_id, assumed_wf_types);
     let outlives_environment =
         OutlivesEnvironment::with_bounds(param_env, Some(&infcx), implied_bounds);
 
diff --git a/compiler/rustc_hir_analysis/src/check/dropck.rs b/compiler/rustc_hir_analysis/src/check/dropck.rs
index 64fd61c13..2bb724138 100644
--- a/compiler/rustc_hir_analysis/src/check/dropck.rs
+++ b/compiler/rustc_hir_analysis/src/check/dropck.rs
@@ -27,7 +27,7 @@ use rustc_middle::ty::{self, Predicate, Ty, TyCtxt};
 ///    cannot do `struct S<T>; impl<T:Clone> Drop for S<T> { ... }`).
 ///
 pub fn check_drop_impl(tcx: TyCtxt<'_>, drop_impl_did: DefId) -> Result<(), ErrorGuaranteed> {
-    let dtor_self_type = tcx.type_of(drop_impl_did);
+    let dtor_self_type = tcx.type_of(drop_impl_did).subst_identity();
     let dtor_predicates = tcx.predicates_of(drop_impl_did);
     match dtor_self_type.kind() {
         ty::Adt(adt_def, self_to_impl_substs) => {
@@ -71,7 +71,7 @@ fn ensure_drop_params_and_item_params_correspond<'tcx>(
 
     let drop_impl_span = tcx.def_span(drop_impl_did);
     let item_span = tcx.def_span(self_type_did);
-    let self_descr = tcx.def_kind(self_type_did).descr(self_type_did);
+    let self_descr = tcx.def_descr(self_type_did);
     let mut err =
         struct_span_err!(tcx.sess, drop_impl_span, E0366, "`Drop` impls cannot be specialized");
     match arg {
@@ -217,7 +217,7 @@ fn ensure_drop_predicates_are_implied_by_item_defn<'tcx>(
 
         if !assumptions_in_impl_context.iter().copied().any(predicate_matches_closure) {
             let item_span = tcx.def_span(self_type_did);
-            let self_descr = tcx.def_kind(self_type_did).descr(self_type_did.to_def_id());
+            let self_descr = tcx.def_descr(self_type_did.to_def_id());
             let reported = struct_span_err!(
                 tcx.sess,
                 predicate_sp,
diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs
index 598dc2dca..054284cce 100644
--- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs
+++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs
@@ -15,8 +15,6 @@ use rustc_middle::ty::{self, TyCtxt};
 use rustc_span::symbol::{kw, sym, Symbol};
 use rustc_target::spec::abi::Abi;
 
-use std::iter;
-
 fn equate_intrinsic_type<'tcx>(
     tcx: TyCtxt<'tcx>,
     it: &hir::ForeignItem<'_>,
@@ -56,8 +54,14 @@ fn equate_intrinsic_type<'tcx>(
         && gen_count_ok(own_counts.consts, 0, "const")
     {
         let fty = tcx.mk_fn_ptr(sig);
-        let cause = ObligationCause::new(it.span, it.hir_id(), ObligationCauseCode::IntrinsicType);
-        require_same_types(tcx, &cause, tcx.mk_fn_ptr(tcx.fn_sig(it.owner_id)), fty);
+        let it_def_id = it.owner_id.def_id;
+        let cause = ObligationCause::new(it.span, it_def_id, ObligationCauseCode::IntrinsicType);
+        require_same_types(
+            tcx,
+            &cause,
+            tcx.mk_fn_ptr(tcx.fn_sig(it.owner_id).subst_identity()),
+            fty,
+        );
     }
 }
 
@@ -133,25 +137,21 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
     let intrinsic_name = tcx.item_name(intrinsic_id);
     let name_str = intrinsic_name.as_str();
 
-    let bound_vars = tcx.mk_bound_variable_kinds(
-        [
-            ty::BoundVariableKind::Region(ty::BrAnon(0, None)),
-            ty::BoundVariableKind::Region(ty::BrEnv),
-        ]
-        .iter()
-        .copied(),
-    );
+    let bound_vars = tcx.mk_bound_variable_kinds(&[
+        ty::BoundVariableKind::Region(ty::BrAnon(0, None)),
+        ty::BoundVariableKind::Region(ty::BrEnv),
+    ]);
     let mk_va_list_ty = |mutbl| {
         tcx.lang_items().va_list().map(|did| {
-            let region = tcx.mk_region(ty::ReLateBound(
+            let region = tcx.mk_re_late_bound(
                 ty::INNERMOST,
                 ty::BoundRegion { var: ty::BoundVar::from_u32(0), kind: ty::BrAnon(0, None) },
-            ));
-            let env_region = tcx.mk_region(ty::ReLateBound(
+            );
+            let env_region = tcx.mk_re_late_bound(
                 ty::INNERMOST,
                 ty::BoundRegion { var: ty::BoundVar::from_u32(1), kind: ty::BrEnv },
-            ));
-            let va_list_ty = tcx.bound_type_of(did).subst(tcx, &[region.into()]);
+            );
+            let va_list_ty = tcx.type_of(did).subst(tcx, &[region.into()]);
             (tcx.mk_ref(env_region, ty::TypeAndMut { ty: va_list_ty, mutbl }), va_list_ty)
         })
     };
@@ -165,7 +165,7 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
             "cxchg" | "cxchgweak" => (
                 1,
                 vec![tcx.mk_mut_ptr(param(0)), param(0), param(0)],
-                tcx.intern_tup(&[param(0), tcx.types.bool]),
+                tcx.mk_tup(&[param(0), tcx.types.bool]),
             ),
             "load" => (1, vec![tcx.mk_imm_ptr(param(0))], param(0)),
             "store" => (1, vec![tcx.mk_mut_ptr(param(0)), param(0)], tcx.mk_unit()),
@@ -317,7 +317,7 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
             | sym::bitreverse => (1, vec![param(0)], param(0)),
 
             sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
-                (1, vec![param(0), param(0)], tcx.intern_tup(&[param(0), tcx.types.bool]))
+                (1, vec![param(0), param(0)], tcx.mk_tup(&[param(0), tcx.types.bool]))
             }
 
             sym::ptr_guaranteed_cmp => {
@@ -371,24 +371,22 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
                     ty::BoundRegion { var: ty::BoundVar::from_u32(0), kind: ty::BrAnon(0, None) };
                 (
                     1,
-                    vec![
-                        tcx.mk_imm_ref(tcx.mk_region(ty::ReLateBound(ty::INNERMOST, br)), param(0)),
-                    ],
-                    tcx.mk_projection(discriminant_def_id, tcx.mk_substs([param(0).into()].iter())),
+                    vec![tcx.mk_imm_ref(tcx.mk_re_late_bound(ty::INNERMOST, br), param(0))],
+                    tcx.mk_projection(discriminant_def_id, tcx.mk_substs(&[param(0).into()])),
                 )
             }
 
             kw::Try => {
                 let mut_u8 = tcx.mk_mut_ptr(tcx.types.u8);
                 let try_fn_ty = ty::Binder::dummy(tcx.mk_fn_sig(
-                    iter::once(mut_u8),
+                    [mut_u8],
                     tcx.mk_unit(),
                     false,
                     hir::Unsafety::Normal,
                     Abi::Rust,
                 ));
                 let catch_fn_ty = ty::Binder::dummy(tcx.mk_fn_sig(
-                    [mut_u8, mut_u8].iter().cloned(),
+                    [mut_u8, mut_u8],
                     tcx.mk_unit(),
                     false,
                     hir::Unsafety::Normal,
@@ -424,8 +422,7 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
             sym::raw_eq => {
                 let br =
                     ty::BoundRegion { var: ty::BoundVar::from_u32(0), kind: ty::BrAnon(0, None) };
-                let param_ty =
-                    tcx.mk_imm_ref(tcx.mk_region(ty::ReLateBound(ty::INNERMOST, br)), param(0));
+                let param_ty = tcx.mk_imm_ref(tcx.mk_re_late_bound(ty::INNERMOST, br), param(0));
                 (1, vec![param_ty; 2], tcx.types.bool)
             }
 
@@ -444,7 +441,7 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
         };
         (n_tps, 0, inputs, output, unsafety)
     };
-    let sig = tcx.mk_fn_sig(inputs.into_iter(), output, false, unsafety, Abi::RustIntrinsic);
+    let sig = tcx.mk_fn_sig(inputs, output, false, unsafety, Abi::RustIntrinsic);
     let sig = ty::Binder::bind_with_vars(sig, bound_vars);
     equate_intrinsic_type(tcx, it, n_tps, n_lts, sig)
 }
@@ -542,13 +539,7 @@ pub fn check_platform_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>)
         }
     };
 
-    let sig = tcx.mk_fn_sig(
-        inputs.into_iter(),
-        output,
-        false,
-        hir::Unsafety::Unsafe,
-        Abi::PlatformIntrinsic,
-    );
+    let sig = tcx.mk_fn_sig(inputs, output, false, hir::Unsafety::Unsafe, Abi::PlatformIntrinsic);
     let sig = ty::Binder::dummy(sig);
     equate_intrinsic_type(tcx, it, n_tps, 0, sig)
 }
diff --git a/compiler/rustc_hir_analysis/src/check/intrinsicck.rs b/compiler/rustc_hir_analysis/src/check/intrinsicck.rs
index 82030d82f..b1d5a27be 100644
--- a/compiler/rustc_hir_analysis/src/check/intrinsicck.rs
+++ b/compiler/rustc_hir_analysis/src/check/intrinsicck.rs
@@ -1,8 +1,9 @@
 use rustc_ast::InlineAsmTemplatePiece;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_hir as hir;
-use rustc_middle::ty::{self, Article, FloatTy, IntTy, Ty, TyCtxt, TypeVisitable, UintTy};
+use rustc_middle::ty::{self, Article, FloatTy, IntTy, Ty, TyCtxt, TypeVisitableExt, UintTy};
 use rustc_session::lint;
+use rustc_span::def_id::LocalDefId;
 use rustc_span::{Symbol, DUMMY_SP};
 use rustc_target::asm::{InlineAsmReg, InlineAsmRegClass, InlineAsmRegOrRegClass, InlineAsmType};
 
@@ -253,10 +254,8 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
         Some(asm_ty)
     }
 
-    pub fn check_asm(&self, asm: &hir::InlineAsm<'tcx>, enclosing_id: hir::HirId) {
-        let hir = self.tcx.hir();
-        let enclosing_def_id = hir.local_def_id(enclosing_id).to_def_id();
-        let target_features = self.tcx.asm_target_features(enclosing_def_id);
+    pub fn check_asm(&self, asm: &hir::InlineAsm<'tcx>, enclosing_id: LocalDefId) {
+        let target_features = self.tcx.asm_target_features(enclosing_id.to_def_id());
         let Some(asm_arch) = self.tcx.sess.asm_arch else {
             self.tcx.sess.delay_span_bug(DUMMY_SP, "target architecture does not support asm");
             return;
@@ -415,7 +414,7 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
                 // Check that sym actually points to a function. Later passes
                 // depend on this.
                 hir::InlineAsmOperand::SymFn { anon_const } => {
-                    let ty = self.tcx.typeck_body(anon_const.body).node_type(anon_const.hir_id);
+                    let ty = self.tcx.type_of(anon_const.def_id).subst_identity();
                     match ty.kind() {
                         ty::Never | ty::Error(_) => {}
                         ty::FnDef(..) => {}
@@ -423,7 +422,7 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
                             let mut err =
                                 self.tcx.sess.struct_span_err(*op_sp, "invalid `sym` operand");
                             err.span_label(
-                                self.tcx.hir().span(anon_const.body.hir_id),
+                                self.tcx.def_span(anon_const.def_id),
                                 &format!("is {} `{}`", ty.kind().article(), ty),
                             );
                             err.help("`sym` operands must refer to either a function or a static");
diff --git a/compiler/rustc_hir_analysis/src/check/mod.rs b/compiler/rustc_hir_analysis/src/check/mod.rs
index 14bca34b7..9acfc1b3d 100644
--- a/compiler/rustc_hir_analysis/src/check/mod.rs
+++ b/compiler/rustc_hir_analysis/src/check/mod.rs
@@ -75,7 +75,6 @@ pub use check::check_abi;
 use check::check_mod_item_types;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_errors::{pluralize, struct_span_err, Applicability, Diagnostic, DiagnosticBuilder};
-use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::intravisit::Visitor;
 use rustc_index::bit_set::BitSet;
@@ -105,6 +104,7 @@ pub fn provide(providers: &mut Providers) {
         region_scope_tree,
         collect_return_position_impl_trait_in_trait_tys,
         compare_impl_const: compare_impl_item::compare_impl_const_raw,
+        check_generator_obligations: check::check_generator_obligations,
         ..*providers
     };
 }
@@ -168,27 +168,24 @@ fn maybe_check_static_with_link_section(tcx: TyCtxt<'_>, id: LocalDefId) {
     }
 }
 
-fn report_forbidden_specialization(
-    tcx: TyCtxt<'_>,
-    impl_item: &hir::ImplItemRef,
-    parent_impl: DefId,
-) {
+fn report_forbidden_specialization(tcx: TyCtxt<'_>, impl_item: DefId, parent_impl: DefId) {
+    let span = tcx.def_span(impl_item);
+    let ident = tcx.item_name(impl_item);
     let mut err = struct_span_err!(
         tcx.sess,
-        impl_item.span,
+        span,
         E0520,
-        "`{}` specializes an item from a parent `impl`, but \
-         that item is not marked `default`",
-        impl_item.ident
+        "`{}` specializes an item from a parent `impl`, but that item is not marked `default`",
+        ident,
     );
-    err.span_label(impl_item.span, format!("cannot specialize default item `{}`", impl_item.ident));
+    err.span_label(span, format!("cannot specialize default item `{}`", ident));
 
     match tcx.span_of_impl(parent_impl) {
         Ok(span) => {
             err.span_label(span, "parent `impl` is here");
             err.note(&format!(
                 "to specialize, `{}` in the parent `impl` must be marked `default`",
-                impl_item.ident
+                ident
             ));
         }
         Err(cname) => {
@@ -202,7 +199,7 @@ fn report_forbidden_specialization(
 fn missing_items_err(
     tcx: TyCtxt<'_>,
     impl_span: Span,
-    missing_items: &[&ty::AssocItem],
+    missing_items: &[ty::AssocItem],
     full_impl_span: Span,
 ) {
     let missing_items_msg = missing_items
@@ -228,7 +225,7 @@ fn missing_items_err(
     let padding =
         tcx.sess.source_map().indentation_before(sugg_sp).unwrap_or_else(|| String::new());
 
-    for trait_item in missing_items {
+    for &trait_item in missing_items {
         let snippet = suggestion_signature(trait_item, tcx);
         let code = format!("{}{}\n{}", padding, snippet, padding);
         let msg = format!("implement the missing item: `{snippet}`");
@@ -275,7 +272,7 @@ fn default_body_is_unstable(
     reason: Option<Symbol>,
     issue: Option<NonZeroU32>,
 ) {
-    let missing_item_name = &tcx.associated_item(item_did).name;
+    let missing_item_name = tcx.associated_item(item_did).name;
     let use_of_unstable_library_feature_note = match reason {
         Some(r) => format!("use of unstable library feature '{feature}': {r}"),
         None => format!("use of unstable library feature '{feature}'"),
@@ -368,7 +365,7 @@ fn fn_sig_suggestion<'tcx>(
     sig: ty::FnSig<'tcx>,
     ident: Ident,
     predicates: ty::GenericPredicates<'tcx>,
-    assoc: &ty::AssocItem,
+    assoc: ty::AssocItem,
 ) -> String {
     let args = sig
         .inputs()
@@ -436,7 +433,7 @@ pub fn ty_kind_suggestion(ty: Ty<'_>) -> Option<&'static str> {
 /// Return placeholder code for the given associated item.
 /// Similar to `ty::AssocItem::suggestion`, but appropriate for use as the code snippet of a
 /// structured suggestion.
-fn suggestion_signature(assoc: &ty::AssocItem, tcx: TyCtxt<'_>) -> String {
+fn suggestion_signature(assoc: ty::AssocItem, tcx: TyCtxt<'_>) -> String {
     match assoc.kind {
         ty::AssocKind::Fn => {
             // We skip the binder here because the binder would deanonymize all
@@ -445,7 +442,7 @@ fn suggestion_signature(assoc: &ty::AssocItem, tcx: TyCtxt<'_>) -> String {
             // regions just fine, showing `fn(&MyType)`.
             fn_sig_suggestion(
                 tcx,
-                tcx.fn_sig(assoc.def_id).skip_binder(),
+                tcx.fn_sig(assoc.def_id).subst_identity().skip_binder(),
                 assoc.ident(tcx),
                 tcx.predicates_of(assoc.def_id),
                 assoc,
@@ -453,7 +450,7 @@ fn suggestion_signature(assoc: &ty::AssocItem, tcx: TyCtxt<'_>) -> String {
         }
         ty::AssocKind::Type => format!("type {} = Type;", assoc.name),
         ty::AssocKind::Const => {
-            let ty = tcx.type_of(assoc.def_id);
+            let ty = tcx.type_of(assoc.def_id).subst_identity();
             let val = ty_kind_suggestion(ty).unwrap_or("value");
             format!("const {}: {} = {};", assoc.name, ty, val)
         }
diff --git a/compiler/rustc_hir_analysis/src/check/wfcheck.rs b/compiler/rustc_hir_analysis/src/check/wfcheck.rs
index 11237afe8..4cccdf30c 100644
--- a/compiler/rustc_hir_analysis/src/check/wfcheck.rs
+++ b/compiler/rustc_hir_analysis/src/check/wfcheck.rs
@@ -16,8 +16,8 @@ use rustc_middle::mir::ConstraintCategory;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::trait_def::TraitSpecializationKind;
 use rustc_middle::ty::{
-    self, AdtKind, DefIdTree, GenericParamDefKind, Ty, TyCtxt, TypeFoldable, TypeSuperVisitable,
-    TypeVisitable, TypeVisitor,
+    self, AdtKind, GenericParamDefKind, Ty, TyCtxt, TypeFoldable, TypeSuperVisitable,
+    TypeVisitable, TypeVisitableExt, TypeVisitor,
 };
 use rustc_middle::ty::{GenericArgKind, InternalSubsts};
 use rustc_session::parse::feature_err;
@@ -37,7 +37,7 @@ use std::ops::{ControlFlow, Deref};
 pub(super) struct WfCheckingCtxt<'a, 'tcx> {
     pub(super) ocx: ObligationCtxt<'a, 'tcx>,
     span: Span,
-    body_id: hir::HirId,
+    body_def_id: LocalDefId,
     param_env: ty::ParamEnv<'tcx>,
 }
 impl<'a, 'tcx> Deref for WfCheckingCtxt<'a, 'tcx> {
@@ -56,10 +56,10 @@ impl<'tcx> WfCheckingCtxt<'_, 'tcx> {
     // `ObligationCtxt::normalize`, but provides a nice `ObligationCauseCode`.
     fn normalize<T>(&self, span: Span, loc: Option<WellFormedLoc>, value: T) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         self.ocx.normalize(
-            &ObligationCause::new(span, self.body_id, ObligationCauseCode::WellFormed(loc)),
+            &ObligationCause::new(span, self.body_def_id, ObligationCauseCode::WellFormed(loc)),
             self.param_env,
             value,
         )
@@ -71,8 +71,11 @@ impl<'tcx> WfCheckingCtxt<'_, 'tcx> {
         loc: Option<WellFormedLoc>,
         arg: ty::GenericArg<'tcx>,
     ) {
-        let cause =
-            traits::ObligationCause::new(span, self.body_id, ObligationCauseCode::WellFormed(loc));
+        let cause = traits::ObligationCause::new(
+            span,
+            self.body_def_id,
+            ObligationCauseCode::WellFormed(loc),
+        );
         // for a type to be WF, we do not need to check if const trait predicates satisfy.
         let param_env = self.param_env.without_const();
         self.ocx.register_obligation(traits::Obligation::new(
@@ -93,11 +96,10 @@ pub(super) fn enter_wf_checking_ctxt<'tcx, F>(
     F: for<'a> FnOnce(&WfCheckingCtxt<'a, 'tcx>),
 {
     let param_env = tcx.param_env(body_def_id);
-    let body_id = tcx.hir().local_def_id_to_hir_id(body_def_id);
     let infcx = &tcx.infer_ctxt().build();
     let ocx = ObligationCtxt::new(infcx);
 
-    let mut wfcx = WfCheckingCtxt { ocx, span, body_id, param_env };
+    let mut wfcx = WfCheckingCtxt { ocx, span, body_def_id, param_env };
 
     if !tcx.features().trivial_bounds {
         wfcx.check_false_global_bounds()
@@ -105,7 +107,7 @@ pub(super) fn enter_wf_checking_ctxt<'tcx, F>(
     f(&mut wfcx);
 
     let assumed_wf_types = wfcx.ocx.assumed_wf_types(param_env, span, body_def_id);
-    let implied_bounds = infcx.implied_bounds_tys(param_env, body_id, assumed_wf_types);
+    let implied_bounds = infcx.implied_bounds_tys(param_env, body_def_id, assumed_wf_types);
 
     let errors = wfcx.select_all_or_error();
     if !errors.is_empty() {
@@ -275,56 +277,6 @@ fn check_trait_item(tcx: TyCtxt<'_>, trait_item: &hir::TraitItem<'_>) {
     };
     check_object_unsafe_self_trait_by_name(tcx, trait_item);
     check_associated_item(tcx, def_id, span, method_sig);
-
-    let encl_trait_def_id = tcx.local_parent(def_id);
-    let encl_trait = tcx.hir().expect_item(encl_trait_def_id);
-    let encl_trait_def_id = encl_trait.owner_id.to_def_id();
-    let fn_lang_item_name = if Some(encl_trait_def_id) == tcx.lang_items().fn_trait() {
-        Some("fn")
-    } else if Some(encl_trait_def_id) == tcx.lang_items().fn_mut_trait() {
-        Some("fn_mut")
-    } else {
-        None
-    };
-
-    if let (Some(fn_lang_item_name), "call") =
-        (fn_lang_item_name, trait_item.ident.name.to_ident_string().as_str())
-    {
-        // We are looking at the `call` function of the `fn` or `fn_mut` lang item.
-        // Do some rudimentary sanity checking to avoid an ICE later (issue #83471).
-        if let Some(hir::FnSig { decl, span, .. }) = method_sig {
-            if let [self_ty, _] = decl.inputs {
-                if !matches!(self_ty.kind, hir::TyKind::Ref(_, _)) {
-                    tcx.sess
-                        .struct_span_err(
-                            self_ty.span,
-                            &format!(
-                                "first argument of `call` in `{fn_lang_item_name}` lang item must be a reference",
-                            ),
-                        )
-                        .emit();
-                }
-            } else {
-                tcx.sess
-                    .struct_span_err(
-                        *span,
-                        &format!(
-                            "`call` function in `{fn_lang_item_name}` lang item takes exactly two arguments",
-                        ),
-                    )
-                    .emit();
-            }
-        } else {
-            tcx.sess
-                .struct_span_err(
-                    trait_item.span,
-                    &format!(
-                        "`call` trait item in `{fn_lang_item_name}` lang item must be a function",
-                    ),
-                )
-                .emit();
-        }
-    }
 }
 
 /// Require that the user writes where clauses on GATs for the implicit
@@ -374,7 +326,6 @@ fn check_gat_where_clauses(tcx: TyCtxt<'_>, associated_items: &[hir::TraitItemRe
                     continue;
                 }
 
-                let item_hir_id = item.id.hir_id();
                 let param_env = tcx.param_env(item_def_id);
 
                 let item_required_bounds = match item.kind {
@@ -385,12 +336,12 @@ fn check_gat_where_clauses(tcx: TyCtxt<'_>, associated_items: &[hir::TraitItemRe
                         // `Self::Iter<'a>` is a GAT we want to gather any potential missing bounds from.
                         let sig: ty::FnSig<'_> = tcx.liberate_late_bound_regions(
                             item_def_id.to_def_id(),
-                            tcx.fn_sig(item_def_id),
+                            tcx.fn_sig(item_def_id).subst_identity(),
                         );
                         gather_gat_bounds(
                             tcx,
                             param_env,
-                            item_hir_id,
+                            item_def_id,
                             sig.inputs_and_output,
                             // We also assume that all of the function signature's parameter types
                             // are well formed.
@@ -412,7 +363,7 @@ fn check_gat_where_clauses(tcx: TyCtxt<'_>, associated_items: &[hir::TraitItemRe
                         gather_gat_bounds(
                             tcx,
                             param_env,
-                            item_hir_id,
+                            item_def_id,
                             tcx.explicit_item_bounds(item_def_id).to_vec(),
                             &FxIndexSet::default(),
                             gat_def_id.def_id,
@@ -458,7 +409,6 @@ fn check_gat_where_clauses(tcx: TyCtxt<'_>, associated_items: &[hir::TraitItemRe
         let gat_item_hir = tcx.hir().expect_trait_item(gat_def_id.def_id);
         debug!(?required_bounds);
         let param_env = tcx.param_env(gat_def_id);
-        let gat_hir = gat_item_hir.hir_id();
 
         let mut unsatisfied_bounds: Vec<_> = required_bounds
             .into_iter()
@@ -466,13 +416,25 @@ fn check_gat_where_clauses(tcx: TyCtxt<'_>, associated_items: &[hir::TraitItemRe
                 ty::PredicateKind::Clause(ty::Clause::RegionOutlives(ty::OutlivesPredicate(
                     a,
                     b,
-                ))) => {
-                    !region_known_to_outlive(tcx, gat_hir, param_env, &FxIndexSet::default(), a, b)
-                }
+                ))) => !region_known_to_outlive(
+                    tcx,
+                    gat_def_id.def_id,
+                    param_env,
+                    &FxIndexSet::default(),
+                    a,
+                    b,
+                ),
                 ty::PredicateKind::Clause(ty::Clause::TypeOutlives(ty::OutlivesPredicate(
                     a,
                     b,
-                ))) => !ty_known_to_outlive(tcx, gat_hir, param_env, &FxIndexSet::default(), a, b),
+                ))) => !ty_known_to_outlive(
+                    tcx,
+                    gat_def_id.def_id,
+                    param_env,
+                    &FxIndexSet::default(),
+                    a,
+                    b,
+                ),
                 _ => bug!("Unexpected PredicateKind"),
             })
             .map(|clause| clause.to_string())
@@ -531,8 +493,9 @@ fn augment_param_env<'tcx>(
         return param_env;
     }
 
-    let bounds =
-        tcx.mk_predicates(param_env.caller_bounds().iter().chain(new_predicates.iter().cloned()));
+    let bounds = tcx.mk_predicates_from_iter(
+        param_env.caller_bounds().iter().chain(new_predicates.iter().cloned()),
+    );
     // FIXME(compiler-errors): Perhaps there is a case where we need to normalize this
     // i.e. traits::normalize_param_env_or_error
     ty::ParamEnv::new(bounds, param_env.reveal(), param_env.constness())
@@ -548,10 +511,10 @@ fn augment_param_env<'tcx>(
 ///     fn into_iter<'a>(&'a self) -> Self::Iter<'a>;
 /// }
 /// ```
-fn gather_gat_bounds<'tcx, T: TypeFoldable<'tcx>>(
+fn gather_gat_bounds<'tcx, T: TypeFoldable<TyCtxt<'tcx>>>(
     tcx: TyCtxt<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
-    item_hir: hir::HirId,
+    item_def_id: hir::OwnerId,
     to_check: T,
     wf_tys: &FxIndexSet<Ty<'tcx>>,
     gat_def_id: LocalDefId,
@@ -584,24 +547,22 @@ fn gather_gat_bounds<'tcx, T: TypeFoldable<'tcx>>(
         // reflected in a where clause on the GAT itself.
         for (ty, ty_idx) in &types {
             // In our example, requires that `Self: 'a`
-            if ty_known_to_outlive(tcx, item_hir, param_env, &wf_tys, *ty, *region_a) {
+            if ty_known_to_outlive(tcx, item_def_id.def_id, param_env, &wf_tys, *ty, *region_a) {
                 debug!(?ty_idx, ?region_a_idx);
                 debug!("required clause: {ty} must outlive {region_a}");
                 // Translate into the generic parameters of the GAT. In
                 // our example, the type was `Self`, which will also be
                 // `Self` in the GAT.
                 let ty_param = gat_generics.param_at(*ty_idx, tcx);
-                let ty_param = tcx
-                    .mk_ty(ty::Param(ty::ParamTy { index: ty_param.index, name: ty_param.name }));
+                let ty_param = tcx.mk_ty_param(ty_param.index, ty_param.name);
                 // Same for the region. In our example, 'a corresponds
                 // to the 'me parameter.
                 let region_param = gat_generics.param_at(*region_a_idx, tcx);
-                let region_param =
-                    tcx.mk_region(ty::RegionKind::ReEarlyBound(ty::EarlyBoundRegion {
-                        def_id: region_param.def_id,
-                        index: region_param.index,
-                        name: region_param.name,
-                    }));
+                let region_param = tcx.mk_re_early_bound(ty::EarlyBoundRegion {
+                    def_id: region_param.def_id,
+                    index: region_param.index,
+                    name: region_param.name,
+                });
                 // The predicate we expect to see. (In our example,
                 // `Self: 'me`.)
                 let clause = ty::PredicateKind::Clause(ty::Clause::TypeOutlives(
@@ -622,25 +583,30 @@ fn gather_gat_bounds<'tcx, T: TypeFoldable<'tcx>>(
             if ty::ReStatic == **region_b || region_a == region_b {
                 continue;
             }
-            if region_known_to_outlive(tcx, item_hir, param_env, &wf_tys, *region_a, *region_b) {
+            if region_known_to_outlive(
+                tcx,
+                item_def_id.def_id,
+                param_env,
+                &wf_tys,
+                *region_a,
+                *region_b,
+            ) {
                 debug!(?region_a_idx, ?region_b_idx);
                 debug!("required clause: {region_a} must outlive {region_b}");
                 // Translate into the generic parameters of the GAT.
                 let region_a_param = gat_generics.param_at(*region_a_idx, tcx);
-                let region_a_param =
-                    tcx.mk_region(ty::RegionKind::ReEarlyBound(ty::EarlyBoundRegion {
-                        def_id: region_a_param.def_id,
-                        index: region_a_param.index,
-                        name: region_a_param.name,
-                    }));
+                let region_a_param = tcx.mk_re_early_bound(ty::EarlyBoundRegion {
+                    def_id: region_a_param.def_id,
+                    index: region_a_param.index,
+                    name: region_a_param.name,
+                });
                 // Same for the region.
                 let region_b_param = gat_generics.param_at(*region_b_idx, tcx);
-                let region_b_param =
-                    tcx.mk_region(ty::RegionKind::ReEarlyBound(ty::EarlyBoundRegion {
-                        def_id: region_b_param.def_id,
-                        index: region_b_param.index,
-                        name: region_b_param.name,
-                    }));
+                let region_b_param = tcx.mk_re_early_bound(ty::EarlyBoundRegion {
+                    def_id: region_b_param.def_id,
+                    index: region_b_param.index,
+                    name: region_b_param.name,
+                });
                 // The predicate we expect to see.
                 let clause = ty::PredicateKind::Clause(ty::Clause::RegionOutlives(
                     ty::OutlivesPredicate(region_a_param, region_b_param),
@@ -658,7 +624,7 @@ fn gather_gat_bounds<'tcx, T: TypeFoldable<'tcx>>(
 /// `ty` outlives `region`.
 fn ty_known_to_outlive<'tcx>(
     tcx: TyCtxt<'tcx>,
-    id: hir::HirId,
+    id: LocalDefId,
     param_env: ty::ParamEnv<'tcx>,
     wf_tys: &FxIndexSet<Ty<'tcx>>,
     ty: Ty<'tcx>,
@@ -675,7 +641,7 @@ fn ty_known_to_outlive<'tcx>(
 /// `region_a` outlives `region_b`
 fn region_known_to_outlive<'tcx>(
     tcx: TyCtxt<'tcx>,
-    id: hir::HirId,
+    id: LocalDefId,
     param_env: ty::ParamEnv<'tcx>,
     wf_tys: &FxIndexSet<Ty<'tcx>>,
     region_a: ty::Region<'tcx>,
@@ -699,7 +665,7 @@ fn region_known_to_outlive<'tcx>(
 /// to be tested), then resolve region and return errors
 fn resolve_regions_with_wf_tys<'tcx>(
     tcx: TyCtxt<'tcx>,
-    id: hir::HirId,
+    id: LocalDefId,
     param_env: ty::ParamEnv<'tcx>,
     wf_tys: &FxIndexSet<Ty<'tcx>>,
     add_constraints: impl for<'a> FnOnce(&'a InferCtxt<'tcx>, &'a RegionBoundPairs<'tcx>),
@@ -743,7 +709,7 @@ struct GATSubstCollector<'tcx> {
 }
 
 impl<'tcx> GATSubstCollector<'tcx> {
-    fn visit<T: TypeFoldable<'tcx>>(
+    fn visit<T: TypeFoldable<TyCtxt<'tcx>>>(
         gat: DefId,
         t: T,
     ) -> (FxHashSet<(ty::Region<'tcx>, usize)>, FxHashSet<(Ty<'tcx>, usize)>) {
@@ -754,7 +720,7 @@ impl<'tcx> GATSubstCollector<'tcx> {
     }
 }
 
-impl<'tcx> TypeVisitor<'tcx> for GATSubstCollector<'tcx> {
+impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for GATSubstCollector<'tcx> {
     type BreakTy = !;
 
     fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
@@ -822,7 +788,7 @@ fn check_object_unsafe_self_trait_by_name(tcx: TyCtxt<'_>, item: &hir::TraitItem
         _ => {}
     }
     if !trait_should_be_self.is_empty() {
-        if tcx.object_safety_violations(trait_def_id).is_empty() {
+        if tcx.check_is_object_safe(trait_def_id) {
             return;
         }
         let sugg = trait_should_be_self.iter().map(|span| (*span, "Self".to_string())).collect();
@@ -859,7 +825,7 @@ fn check_param_wf(tcx: TyCtxt<'_>, param: &hir::GenericParam<'_>) {
 
         // Const parameters are well formed if their type is structural match.
         hir::GenericParamKind::Const { ty: hir_ty, default: _ } => {
-            let ty = tcx.type_of(param.def_id);
+            let ty = tcx.type_of(param.def_id).subst_identity();
 
             if tcx.features().adt_const_params {
                 if let Some(non_structural_match_ty) =
@@ -996,17 +962,17 @@ fn check_associated_item(
 
         let self_ty = match item.container {
             ty::TraitContainer => tcx.types.self_param,
-            ty::ImplContainer => tcx.type_of(item.container_id(tcx)),
+            ty::ImplContainer => tcx.type_of(item.container_id(tcx)).subst_identity(),
         };
 
         match item.kind {
             ty::AssocKind::Const => {
-                let ty = tcx.type_of(item.def_id);
+                let ty = tcx.type_of(item.def_id).subst_identity();
                 let ty = wfcx.normalize(span, Some(WellFormedLoc::Ty(item_id)), ty);
                 wfcx.register_wf_obligation(span, loc, ty.into());
             }
             ty::AssocKind::Fn => {
-                let sig = tcx.fn_sig(item.def_id);
+                let sig = tcx.fn_sig(item.def_id).subst_identity();
                 let hir_sig = sig_if_method.expect("bad signature for method");
                 check_fn_or_method(
                     wfcx,
@@ -1022,7 +988,7 @@ fn check_associated_item(
                     check_associated_type_bounds(wfcx, item, span)
                 }
                 if item.defaultness(tcx).has_value() {
-                    let ty = tcx.type_of(item.def_id);
+                    let ty = tcx.type_of(item.def_id).subst_identity();
                     let ty = wfcx.normalize(span, Some(WellFormedLoc::Ty(item_id)), ty);
                     wfcx.register_wf_obligation(span, loc, ty.into());
                 }
@@ -1053,9 +1019,9 @@ fn check_type_defn<'tcx>(tcx: TyCtxt<'tcx>, item: &hir::Item<'tcx>, all_sized: b
             // All field types must be well-formed.
             for field in &variant.fields {
                 let field_id = field.did.expect_local();
-                let hir::Node::Field(hir::FieldDef { ty: hir_ty, .. }) = tcx.hir().get_by_def_id(field_id)
-                else { bug!() };
-                let ty = wfcx.normalize(hir_ty.span, None, tcx.type_of(field.did));
+                let hir::FieldDef { ty: hir_ty, .. } =
+                    tcx.hir().get_by_def_id(field_id).expect_field();
+                let ty = wfcx.normalize(hir_ty.span, None, tcx.type_of(field.did).subst_identity());
                 wfcx.register_wf_obligation(
                     hir_ty.span,
                     Some(WellFormedLoc::Ty(field_id)),
@@ -1067,7 +1033,7 @@ fn check_type_defn<'tcx>(tcx: TyCtxt<'tcx>, item: &hir::Item<'tcx>, all_sized: b
             // intermediate types must be sized.
             let needs_drop_copy = || {
                 packed && {
-                    let ty = tcx.type_of(variant.fields.last().unwrap().did);
+                    let ty = tcx.type_of(variant.fields.last().unwrap().did).subst_identity();
                     let ty = tcx.erase_regions(ty);
                     if ty.needs_infer() {
                         tcx.sess
@@ -1087,13 +1053,13 @@ fn check_type_defn<'tcx>(tcx: TyCtxt<'tcx>, item: &hir::Item<'tcx>, all_sized: b
             {
                 let last = idx == variant.fields.len() - 1;
                 let field_id = field.did.expect_local();
-                let hir::Node::Field(hir::FieldDef { ty: hir_ty, .. }) = tcx.hir().get_by_def_id(field_id)
-                else { bug!() };
-                let ty = wfcx.normalize(hir_ty.span, None, tcx.type_of(field.did));
+                let hir::FieldDef { ty: hir_ty, .. } =
+                    tcx.hir().get_by_def_id(field_id).expect_field();
+                let ty = wfcx.normalize(hir_ty.span, None, tcx.type_of(field.did).subst_identity());
                 wfcx.register_bound(
                     traits::ObligationCause::new(
                         hir_ty.span,
-                        wfcx.body_id,
+                        wfcx.body_def_id,
                         traits::FieldSized {
                             adt_kind: match item_adt_kind(&item.kind) {
                                 Some(i) => i,
@@ -1113,7 +1079,7 @@ fn check_type_defn<'tcx>(tcx: TyCtxt<'tcx>, item: &hir::Item<'tcx>, all_sized: b
             if let ty::VariantDiscr::Explicit(discr_def_id) = variant.discr {
                 let cause = traits::ObligationCause::new(
                     tcx.def_span(discr_def_id),
-                    wfcx.body_id,
+                    wfcx.body_def_id,
                     traits::MiscObligation,
                 );
                 wfcx.register_obligation(traits::Obligation::new(
@@ -1165,7 +1131,7 @@ fn check_trait(tcx: TyCtxt<'_>, item: &hir::Item<'_>) {
 ///
 /// Assuming the defaults are used, check that all predicates (bounds on the
 /// assoc type and where clauses on the trait) hold.
-fn check_associated_type_bounds(wfcx: &WfCheckingCtxt<'_, '_>, item: &ty::AssocItem, span: Span) {
+fn check_associated_type_bounds(wfcx: &WfCheckingCtxt<'_, '_>, item: ty::AssocItem, span: Span) {
     let bounds = wfcx.tcx().explicit_item_bounds(item.def_id);
 
     debug!("check_associated_type_bounds: bounds={:?}", bounds);
@@ -1174,7 +1140,7 @@ fn check_associated_type_bounds(wfcx: &WfCheckingCtxt<'_, '_>, item: &ty::AssocI
         traits::wf::predicate_obligations(
             wfcx.infcx,
             wfcx.param_env,
-            wfcx.body_id,
+            wfcx.body_def_id,
             normalized_bound,
             bound_span,
         )
@@ -1191,7 +1157,7 @@ fn check_item_fn(
     decl: &hir::FnDecl<'_>,
 ) {
     enter_wf_checking_ctxt(tcx, span, def_id, |wfcx| {
-        let sig = tcx.fn_sig(def_id);
+        let sig = tcx.fn_sig(def_id).subst_identity();
         check_fn_or_method(wfcx, ident.span, sig, decl, def_id);
     })
 }
@@ -1200,7 +1166,7 @@ fn check_item_type(tcx: TyCtxt<'_>, item_id: LocalDefId, ty_span: Span, allow_fo
     debug!("check_item_type: {:?}", item_id);
 
     enter_wf_checking_ctxt(tcx, ty_span, item_id, |wfcx| {
-        let ty = tcx.type_of(item_id);
+        let ty = tcx.type_of(item_id).subst_identity();
         let item_ty = wfcx.normalize(ty_span, Some(WellFormedLoc::Ty(item_id)), ty);
 
         let mut forbid_unsized = true;
@@ -1214,7 +1180,7 @@ fn check_item_type(tcx: TyCtxt<'_>, item_id: LocalDefId, ty_span: Span, allow_fo
         wfcx.register_wf_obligation(ty_span, Some(WellFormedLoc::Ty(item_id)), item_ty.into());
         if forbid_unsized {
             wfcx.register_bound(
-                traits::ObligationCause::new(ty_span, wfcx.body_id, traits::WellFormed(None)),
+                traits::ObligationCause::new(ty_span, wfcx.body_def_id, traits::WellFormed(None)),
                 wfcx.param_env,
                 item_ty,
                 tcx.require_lang_item(LangItem::Sized, None),
@@ -1229,7 +1195,7 @@ fn check_item_type(tcx: TyCtxt<'_>, item_id: LocalDefId, ty_span: Span, allow_fo
 
         if should_check_for_sync {
             wfcx.register_bound(
-                traits::ObligationCause::new(ty_span, wfcx.body_id, traits::SharedStatic),
+                traits::ObligationCause::new(ty_span, wfcx.body_def_id, traits::SharedStatic),
                 wfcx.param_env,
                 item_ty,
                 tcx.require_lang_item(LangItem::Sync, Some(ty_span)),
@@ -1269,7 +1235,7 @@ fn check_impl<'tcx>(
                 let mut obligations = traits::wf::trait_obligations(
                     wfcx.infcx,
                     wfcx.param_env,
-                    wfcx.body_id,
+                    wfcx.body_def_id,
                     &trait_pred,
                     ast_trait_ref.path.span,
                     item,
@@ -1285,7 +1251,7 @@ fn check_impl<'tcx>(
                 wfcx.register_obligations(obligations);
             }
             None => {
-                let self_ty = tcx.type_of(item.owner_id);
+                let self_ty = tcx.type_of(item.owner_id).subst_identity();
                 let self_ty = wfcx.normalize(
                     item.span,
                     Some(WellFormedLoc::Ty(item.hir_id().expect_owner().def_id)),
@@ -1330,7 +1296,7 @@ fn check_where_clauses<'tcx>(wfcx: &WfCheckingCtxt<'_, 'tcx>, span: Span, def_id
         match param.kind {
             GenericParamDefKind::Type { .. } => {
                 if is_our_default(param) {
-                    let ty = tcx.type_of(param.def_id);
+                    let ty = tcx.type_of(param.def_id).subst_identity();
                     // Ignore dependent defaults -- that is, where the default of one type
                     // parameter includes another (e.g., `<T, U = T>`). In those cases, we can't
                     // be sure if it will error or not as user might always specify the other.
@@ -1382,7 +1348,7 @@ fn check_where_clauses<'tcx>(wfcx: &WfCheckingCtxt<'_, 'tcx>, span: Span, def_id
             GenericParamDefKind::Type { .. } => {
                 // If the param has a default, ...
                 if is_our_default(param) {
-                    let default_ty = tcx.type_of(param.def_id);
+                    let default_ty = tcx.type_of(param.def_id).subst_identity();
                     // ... and it's not a dependent default, ...
                     if !default_ty.needs_subst() {
                         // ... then substitute it with the default.
@@ -1417,7 +1383,7 @@ fn check_where_clauses<'tcx>(wfcx: &WfCheckingCtxt<'_, 'tcx>, span: Span, def_id
             struct CountParams {
                 params: FxHashSet<u32>,
             }
-            impl<'tcx> ty::visit::TypeVisitor<'tcx> for CountParams {
+            impl<'tcx> ty::visit::TypeVisitor<TyCtxt<'tcx>> for CountParams {
                 type BreakTy = ();
 
                 fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
@@ -1466,7 +1432,7 @@ fn check_where_clauses<'tcx>(wfcx: &WfCheckingCtxt<'_, 'tcx>, span: Span, def_id
             let pred = wfcx.normalize(sp, None, pred);
             let cause = traits::ObligationCause::new(
                 sp,
-                wfcx.body_id,
+                wfcx.body_def_id,
                 traits::ItemObligation(def_id.to_def_id()),
             );
             traits::Obligation::new(tcx, cause, wfcx.param_env, pred)
@@ -1482,12 +1448,11 @@ fn check_where_clauses<'tcx>(wfcx: &WfCheckingCtxt<'_, 'tcx>, span: Span, def_id
         traits::wf::predicate_obligations(
             infcx,
             wfcx.param_env.without_const(),
-            wfcx.body_id,
+            wfcx.body_def_id,
             p,
             sp,
         )
     });
-
     let obligations: Vec<_> = wf_obligations.chain(default_obligations).collect();
     wfcx.register_obligations(obligations);
 }
@@ -1512,7 +1477,7 @@ fn check_fn_or_method<'tcx>(
         |idx| hir_decl.inputs.get(idx).map_or(hir_decl.output.span(), |arg: &hir::Ty<'_>| arg.span);
 
     sig.inputs_and_output =
-        tcx.mk_type_list(sig.inputs_and_output.iter().enumerate().map(|(idx, ty)| {
+        tcx.mk_type_list_from_iter(sig.inputs_and_output.iter().enumerate().map(|(idx, ty)| {
             wfcx.normalize(
                 arg_span(idx),
                 Some(WellFormedLoc::Param {
@@ -1549,7 +1514,7 @@ fn check_fn_or_method<'tcx>(
         // Check that the argument is a tuple
         if let Some(ty) = inputs.next() {
             wfcx.register_bound(
-                ObligationCause::new(span, wfcx.body_id, ObligationCauseCode::RustCall),
+                ObligationCause::new(span, wfcx.body_def_id, ObligationCauseCode::RustCall),
                 wfcx.param_env,
                 *ty,
                 tcx.require_lang_item(hir::LangItem::Tuple, Some(span)),
@@ -1585,7 +1550,7 @@ fn check_return_position_impl_trait_in_trait_bounds<'tcx>(
     {
         for arg in fn_output.walk() {
             if let ty::GenericArgKind::Type(ty) = arg.unpack()
-                && let ty::Alias(ty::Projection, proj) = ty.kind()
+                && let ty::Alias(ty::Opaque, proj) = ty.kind()
                 && tcx.def_kind(proj.def_id) == DefKind::ImplTraitPlaceholder
                 && tcx.impl_trait_in_trait_parent(proj.def_id) == fn_def_id.to_def_id()
             {
@@ -1597,7 +1562,7 @@ fn check_return_position_impl_trait_in_trait_bounds<'tcx>(
                     traits::wf::predicate_obligations(
                         wfcx.infcx,
                         wfcx.param_env,
-                        wfcx.body_id,
+                        wfcx.body_def_id,
                         normalized_bound,
                         bound_span,
                     )
@@ -1616,7 +1581,7 @@ const HELP_FOR_SELF_TYPE: &str = "consider changing to `self`, `&self`, `&mut se
 fn check_method_receiver<'tcx>(
     wfcx: &WfCheckingCtxt<'_, 'tcx>,
     fn_sig: &hir::FnSig<'_>,
-    method: &ty::AssocItem,
+    method: ty::AssocItem,
     self_ty: Ty<'tcx>,
 ) {
     let tcx = wfcx.tcx();
@@ -1627,7 +1592,7 @@ fn check_method_receiver<'tcx>(
 
     let span = fn_sig.decl.inputs[0].span;
 
-    let sig = tcx.fn_sig(method.def_id);
+    let sig = tcx.fn_sig(method.def_id).subst_identity();
     let sig = tcx.liberate_late_bound_regions(method.def_id, sig);
     let sig = wfcx.normalize(span, None, sig);
 
@@ -1697,9 +1662,9 @@ fn receiver_is_valid<'tcx>(
     let infcx = wfcx.infcx;
     let tcx = wfcx.tcx();
     let cause =
-        ObligationCause::new(span, wfcx.body_id, traits::ObligationCauseCode::MethodReceiver);
+        ObligationCause::new(span, wfcx.body_def_id, traits::ObligationCauseCode::MethodReceiver);
 
-    let can_eq_self = |ty| infcx.can_eq(wfcx.param_env, self_ty, ty).is_ok();
+    let can_eq_self = |ty| infcx.can_eq(wfcx.param_env, self_ty, ty);
 
     // `self: Self` is always valid.
     if can_eq_self(receiver_ty) {
@@ -1709,7 +1674,7 @@ fn receiver_is_valid<'tcx>(
         return true;
     }
 
-    let mut autoderef = Autoderef::new(infcx, wfcx.param_env, wfcx.body_id, span, receiver_ty);
+    let mut autoderef = Autoderef::new(infcx, wfcx.param_env, wfcx.body_def_id, span, receiver_ty);
 
     // The `arbitrary_self_types` feature allows raw pointer receivers like `self: *const Self`.
     if arbitrary_self_types_enabled {
@@ -1799,7 +1764,7 @@ fn check_variances_for_type_defn<'tcx>(
     item: &hir::Item<'tcx>,
     hir_generics: &hir::Generics<'_>,
 ) {
-    let ty = tcx.type_of(item.owner_id);
+    let ty = tcx.type_of(item.owner_id).subst_identity();
     if tcx.has_error_field(ty) {
         return;
     }
@@ -1894,8 +1859,7 @@ impl<'tcx> WfCheckingCtxt<'_, 'tcx> {
         let mut span = self.span;
         let empty_env = ty::ParamEnv::empty();
 
-        let def_id = tcx.hir().local_def_id(self.body_id);
-        let predicates_with_span = tcx.predicates_of(def_id).predicates.iter().copied();
+        let predicates_with_span = tcx.predicates_of(self.body_def_id).predicates.iter().copied();
         // Check elaborated bounds.
         let implied_obligations = traits::elaborate_predicates_with_span(tcx, predicates_with_span);
 
@@ -1910,7 +1874,7 @@ impl<'tcx> WfCheckingCtxt<'_, 'tcx> {
             // Match the existing behavior.
             if pred.is_global() && !pred.has_late_bound_vars() {
                 let pred = self.normalize(span, None, pred);
-                let hir_node = tcx.hir().find(self.body_id);
+                let hir_node = tcx.hir().find_by_def_id(self.body_def_id);
 
                 // only use the span of the predicate clause (#90869)
 
@@ -1929,7 +1893,7 @@ impl<'tcx> WfCheckingCtxt<'_, 'tcx> {
 
                 let obligation = traits::Obligation::new(
                     tcx,
-                    traits::ObligationCause::new(span, self.body_id, traits::TrivialBound),
+                    traits::ObligationCause::new(span, self.body_def_id, traits::TrivialBound),
                     empty_env,
                     pred,
                 );
diff --git a/compiler/rustc_hir_analysis/src/check_unused.rs b/compiler/rustc_hir_analysis/src/check_unused.rs
index ebb78213a..f3f5851d8 100644
--- a/compiler/rustc_hir_analysis/src/check_unused.rs
+++ b/compiler/rustc_hir_analysis/src/check_unused.rs
@@ -1,12 +1,8 @@
-use crate::errors::{ExternCrateNotIdiomatic, UnusedExternCrate};
-use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::unord::UnordSet;
-use rustc_hir as hir;
 use rustc_hir::def::DefKind;
-use rustc_hir::def_id::{DefId, LocalDefId};
+use rustc_hir::def_id::LocalDefId;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::lint;
-use rustc_span::{Span, Symbol};
 
 pub fn check_crate(tcx: TyCtxt<'_>) {
     let mut used_trait_imports: UnordSet<LocalDefId> = Default::default();
@@ -29,7 +25,7 @@ pub fn check_crate(tcx: TyCtxt<'_>) {
         if item.span.is_dummy() {
             continue;
         }
-        let hir::ItemKind::Use(path, _) = item.kind else { unreachable!() };
+        let (path, _) = item.expect_use();
         let msg = if let Ok(snippet) = tcx.sess.source_map().span_to_snippet(path.span) {
             format!("unused import: `{}`", snippet)
         } else {
@@ -43,131 +39,4 @@ pub fn check_crate(tcx: TyCtxt<'_>) {
             |lint| lint,
         );
     }
-
-    unused_crates_lint(tcx);
-}
-
-fn unused_crates_lint(tcx: TyCtxt<'_>) {
-    let lint = lint::builtin::UNUSED_EXTERN_CRATES;
-
-    // Collect first the crates that are completely unused. These we
-    // can always suggest removing (no matter which edition we are
-    // in).
-    let unused_extern_crates: FxHashMap<LocalDefId, Span> = tcx
-        .maybe_unused_extern_crates(())
-        .iter()
-        .filter(|&&(def_id, _)| {
-            tcx.extern_mod_stmt_cnum(def_id).map_or(true, |cnum| {
-                !tcx.is_compiler_builtins(cnum)
-                    && !tcx.is_panic_runtime(cnum)
-                    && !tcx.has_global_allocator(cnum)
-                    && !tcx.has_panic_handler(cnum)
-            })
-        })
-        .cloned()
-        .collect();
-
-    // Collect all the extern crates (in a reliable order).
-    let mut crates_to_lint = vec![];
-
-    for id in tcx.hir().items() {
-        if matches!(tcx.def_kind(id.owner_id), DefKind::ExternCrate) {
-            let item = tcx.hir().item(id);
-            if let hir::ItemKind::ExternCrate(orig_name) = item.kind {
-                crates_to_lint.push(ExternCrateToLint {
-                    def_id: item.owner_id.to_def_id(),
-                    span: item.span,
-                    orig_name,
-                    warn_if_unused: !item.ident.as_str().starts_with('_'),
-                });
-            }
-        }
-    }
-
-    let extern_prelude = &tcx.resolutions(()).extern_prelude;
-
-    for extern_crate in &crates_to_lint {
-        let def_id = extern_crate.def_id.expect_local();
-        let item = tcx.hir().expect_item(def_id);
-
-        // If the crate is fully unused, we suggest removing it altogether.
-        // We do this in any edition.
-        if extern_crate.warn_if_unused {
-            if let Some(&span) = unused_extern_crates.get(&def_id) {
-                // Removal suggestion span needs to include attributes (Issue #54400)
-                let id = tcx.hir().local_def_id_to_hir_id(def_id);
-                let span_with_attrs = tcx
-                    .hir()
-                    .attrs(id)
-                    .iter()
-                    .map(|attr| attr.span)
-                    .fold(span, |acc, attr_span| acc.to(attr_span));
-
-                tcx.emit_spanned_lint(lint, id, span, UnusedExternCrate { span: span_with_attrs });
-                continue;
-            }
-        }
-
-        // If we are not in Rust 2018 edition, then we don't make any further
-        // suggestions.
-        if !tcx.sess.rust_2018() {
-            continue;
-        }
-
-        // If the extern crate isn't in the extern prelude,
-        // there is no way it can be written as a `use`.
-        let orig_name = extern_crate.orig_name.unwrap_or(item.ident.name);
-        if !extern_prelude.get(&orig_name).map_or(false, |from_item| !from_item) {
-            continue;
-        }
-
-        // If the extern crate is renamed, then we cannot suggest replacing it with a use as this
-        // would not insert the new name into the prelude, where other imports in the crate may be
-        // expecting it.
-        if extern_crate.orig_name.is_some() {
-            continue;
-        }
-
-        let id = tcx.hir().local_def_id_to_hir_id(def_id);
-        // If the extern crate has any attributes, they may have funky
-        // semantics we can't faithfully represent using `use` (most
-        // notably `#[macro_use]`). Ignore it.
-        if !tcx.hir().attrs(id).is_empty() {
-            continue;
-        }
-
-        let base_replacement = match extern_crate.orig_name {
-            Some(orig_name) => format!("use {} as {};", orig_name, item.ident.name),
-            None => format!("use {};", item.ident.name),
-        };
-        let vis = tcx.sess.source_map().span_to_snippet(item.vis_span).unwrap_or_default();
-        let add_vis = |to| if vis.is_empty() { to } else { format!("{} {}", vis, to) };
-        tcx.emit_spanned_lint(
-            lint,
-            id,
-            extern_crate.span,
-            ExternCrateNotIdiomatic {
-                span: extern_crate.span,
-                msg_code: add_vis("use".to_string()),
-                suggestion_code: add_vis(base_replacement),
-            },
-        );
-    }
-}
-
-struct ExternCrateToLint {
-    /// `DefId` of the extern crate
-    def_id: DefId,
-
-    /// span from the item
-    span: Span,
-
-    /// if `Some`, then this is renamed (`extern crate orig_name as
-    /// crate_name`), and -- perhaps surprisingly -- this stores the
-    /// *original* name (`item.name` will contain the new name)
-    orig_name: Option<Symbol>,
-
-    /// if `false`, the original name started with `_`, so we shouldn't lint
-    /// about it going unused (but we should still emit idiom lints).
-    warn_if_unused: bool,
 }
diff --git a/compiler/rustc_hir_analysis/src/coherence/builtin.rs b/compiler/rustc_hir_analysis/src/coherence/builtin.rs
index 28c040878..ffb68abf9 100644
--- a/compiler/rustc_hir_analysis/src/coherence/builtin.rs
+++ b/compiler/rustc_hir_analysis/src/coherence/builtin.rs
@@ -11,7 +11,7 @@ use rustc_infer::infer::outlives::env::OutlivesEnvironment;
 use rustc_infer::infer::TyCtxtInferExt;
 use rustc_infer::infer::{self, RegionResolutionError};
 use rustc_middle::ty::adjustment::CoerceUnsizedInfo;
-use rustc_middle::ty::{self, suggest_constraining_type_params, Ty, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, suggest_constraining_type_params, Ty, TyCtxt, TypeVisitableExt};
 use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt;
 use rustc_trait_selection::traits::misc::{
     type_allowed_to_implement_copy, CopyImplementationError, InfringingFieldsReason,
@@ -50,13 +50,13 @@ impl<'tcx> Checker<'tcx> {
 
 fn visit_implementation_of_drop(tcx: TyCtxt<'_>, impl_did: LocalDefId) {
     // Destructors only work on local ADT types.
-    match tcx.type_of(impl_did).kind() {
+    match tcx.type_of(impl_did).subst_identity().kind() {
         ty::Adt(def, _) if def.did().is_local() => return,
         ty::Error(_) => return,
         _ => {}
     }
 
-    let ItemKind::Impl(impl_) = tcx.hir().expect_item(impl_did).kind else { bug!("expected Drop impl item") };
+    let impl_ = tcx.hir().expect_item(impl_did).expect_impl();
 
     tcx.sess.emit_err(DropImplOnWrongItem { span: impl_.self_ty.span });
 }
@@ -64,9 +64,7 @@ fn visit_implementation_of_drop(tcx: TyCtxt<'_>, impl_did: LocalDefId) {
 fn visit_implementation_of_copy(tcx: TyCtxt<'_>, impl_did: LocalDefId) {
     debug!("visit_implementation_of_copy: impl_did={:?}", impl_did);
 
-    let impl_hir_id = tcx.hir().local_def_id_to_hir_id(impl_did);
-
-    let self_type = tcx.type_of(impl_did);
+    let self_type = tcx.type_of(impl_did).subst_identity();
     debug!("visit_implementation_of_copy: self_type={:?} (bound)", self_type);
 
     let param_env = tcx.param_env(impl_did);
@@ -80,7 +78,7 @@ fn visit_implementation_of_copy(tcx: TyCtxt<'_>, impl_did: LocalDefId) {
         _ => bug!("expected Copy impl item"),
     };
 
-    let cause = traits::ObligationCause::misc(span, impl_hir_id);
+    let cause = traits::ObligationCause::misc(span, impl_did);
     match type_allowed_to_implement_copy(tcx, param_env, self_type, cause) {
         Ok(()) => {}
         Err(CopyImplementationError::InfrigingFields(fields)) => {
@@ -178,6 +176,7 @@ fn visit_implementation_of_copy(tcx: TyCtxt<'_>, impl_did: LocalDefId) {
                 bounds.iter().map(|(param, constraint, def_id)| {
                     (param.as_str(), constraint.as_str(), *def_id)
                 }),
+                None,
             );
             err.emit();
         }
@@ -203,12 +202,11 @@ fn visit_implementation_of_coerce_unsized(tcx: TyCtxt<'_>, impl_did: LocalDefId)
 fn visit_implementation_of_dispatch_from_dyn(tcx: TyCtxt<'_>, impl_did: LocalDefId) {
     debug!("visit_implementation_of_dispatch_from_dyn: impl_did={:?}", impl_did);
 
-    let impl_hir_id = tcx.hir().local_def_id_to_hir_id(impl_did);
-    let span = tcx.hir().span(impl_hir_id);
+    let span = tcx.def_span(impl_did);
 
     let dispatch_from_dyn_trait = tcx.require_lang_item(LangItem::DispatchFromDyn, Some(span));
 
-    let source = tcx.type_of(impl_did);
+    let source = tcx.type_of(impl_did).subst_identity();
     assert!(!source.has_escaping_bound_vars());
     let target = {
         let trait_ref = tcx.impl_trait_ref(impl_did).unwrap().subst_identity();
@@ -224,7 +222,7 @@ fn visit_implementation_of_dispatch_from_dyn(tcx: TyCtxt<'_>, impl_did: LocalDef
     let create_err = |msg: &str| struct_span_err!(tcx.sess, span, E0378, "{}", msg);
 
     let infcx = tcx.infer_ctxt().build();
-    let cause = ObligationCause::misc(span, impl_hir_id);
+    let cause = ObligationCause::misc(span, impl_did);
 
     use rustc_type_ir::sty::TyKind::*;
     match (source.kind(), target.kind()) {
@@ -372,7 +370,7 @@ pub fn coerce_unsized_info<'tcx>(tcx: TyCtxt<'tcx>, impl_did: DefId) -> CoerceUn
         tcx.sess.fatal(&format!("`CoerceUnsized` implementation {}", err.to_string()));
     });
 
-    let source = tcx.type_of(impl_did);
+    let source = tcx.type_of(impl_did).subst_identity();
     let trait_ref = tcx.impl_trait_ref(impl_did).unwrap().subst_identity();
     assert_eq!(trait_ref.def_id, coerce_unsized_trait);
     let target = trait_ref.substs.type_at(1);
@@ -386,8 +384,7 @@ pub fn coerce_unsized_info<'tcx>(tcx: TyCtxt<'tcx>, impl_did: DefId) -> CoerceUn
     debug!("visit_implementation_of_coerce_unsized: {:?} -> {:?} (free)", source, target);
 
     let infcx = tcx.infer_ctxt().build();
-    let impl_hir_id = tcx.hir().local_def_id_to_hir_id(impl_did);
-    let cause = ObligationCause::misc(span, impl_hir_id);
+    let cause = ObligationCause::misc(span, impl_did);
     let check_mutbl = |mt_a: ty::TypeAndMut<'tcx>,
                        mt_b: ty::TypeAndMut<'tcx>,
                        mk_ptr: &dyn Fn(Ty<'tcx>) -> Ty<'tcx>| {
@@ -440,7 +437,7 @@ pub fn coerce_unsized_info<'tcx>(tcx: TyCtxt<'tcx>, impl_did: DefId) -> CoerceUn
             }
 
             // Here we are considering a case of converting
-            // `S<P0...Pn>` to S<Q0...Qn>`. As an example, let's imagine a struct `Foo<T, U>`,
+            // `S<P0...Pn>` to `S<Q0...Qn>`. As an example, let's imagine a struct `Foo<T, U>`,
             // which acts like a pointer to `U`, but carries along some extra data of type `T`:
             //
             //     struct Foo<T, U> {
@@ -485,7 +482,7 @@ pub fn coerce_unsized_info<'tcx>(tcx: TyCtxt<'tcx>, impl_did: DefId) -> CoerceUn
                 .filter_map(|(i, f)| {
                     let (a, b) = (f.ty(tcx, substs_a), f.ty(tcx, substs_b));
 
-                    if tcx.type_of(f.did).is_phantom_data() {
+                    if tcx.type_of(f.did).subst_identity().is_phantom_data() {
                         // Ignore PhantomData fields
                         return None;
                     }
@@ -575,7 +572,7 @@ pub fn coerce_unsized_info<'tcx>(tcx: TyCtxt<'tcx>, impl_did: DefId) -> CoerceUn
     };
 
     // Register an obligation for `A: Trait<B>`.
-    let cause = traits::ObligationCause::misc(span, impl_hir_id);
+    let cause = traits::ObligationCause::misc(span, impl_did);
     let predicate =
         predicate_for_trait_def(tcx, param_env, cause, trait_def_id, 0, [source, target]);
     let errors = traits::fully_solve_obligation(&infcx, predicate);
diff --git a/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs b/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs
index dfb982409..02f3eeee0 100644
--- a/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs
+++ b/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs
@@ -14,7 +14,6 @@ use rustc_hir::def_id::{CrateNum, DefId, LocalDefId};
 use rustc_middle::ty::fast_reject::{simplify_type, SimplifiedType, TreatParams};
 use rustc_middle::ty::{self, CrateInherentImpls, Ty, TyCtxt};
 use rustc_span::symbol::sym;
-use rustc_span::Span;
 
 /// On-demand query: yields a map containing all types mapped to their inherent impls.
 pub fn crate_inherent_impls(tcx: TyCtxt<'_>, (): ()) -> CrateInherentImpls {
@@ -57,86 +56,76 @@ const ADD_ATTR: &str =
     "alternatively add `#[rustc_allow_incoherent_impl]` to the relevant impl items";
 
 impl<'tcx> InherentCollect<'tcx> {
-    fn check_def_id(&mut self, item: &hir::Item<'_>, self_ty: Ty<'tcx>, def_id: DefId) {
-        let impl_def_id = item.owner_id;
-        if let Some(def_id) = def_id.as_local() {
+    fn check_def_id(&mut self, impl_def_id: LocalDefId, self_ty: Ty<'tcx>, ty_def_id: DefId) {
+        if let Some(ty_def_id) = ty_def_id.as_local() {
             // Add the implementation to the mapping from implementation to base
             // type def ID, if there is a base type for this implementation and
             // the implementation does not have any associated traits.
-            let vec = self.impls_map.inherent_impls.entry(def_id).or_default();
+            let vec = self.impls_map.inherent_impls.entry(ty_def_id).or_default();
             vec.push(impl_def_id.to_def_id());
             return;
         }
 
         if self.tcx.features().rustc_attrs {
-            let hir::ItemKind::Impl(&hir::Impl { items, .. }) = item.kind else {
-                bug!("expected `impl` item: {:?}", item);
-            };
+            let items = self.tcx.associated_item_def_ids(impl_def_id);
 
-            if !self.tcx.has_attr(def_id, sym::rustc_has_incoherent_inherent_impls) {
+            if !self.tcx.has_attr(ty_def_id, sym::rustc_has_incoherent_inherent_impls) {
+                let impl_span = self.tcx.def_span(impl_def_id);
                 struct_span_err!(
                     self.tcx.sess,
-                    item.span,
+                    impl_span,
                     E0390,
                     "cannot define inherent `impl` for a type outside of the crate where the type is defined",
                 )
                 .help(INTO_DEFINING_CRATE)
-                .span_help(item.span, ADD_ATTR_TO_TY)
+                .span_help(impl_span, ADD_ATTR_TO_TY)
                 .emit();
                 return;
             }
 
-            for impl_item in items {
-                if !self
-                    .tcx
-                    .has_attr(impl_item.id.owner_id.to_def_id(), sym::rustc_allow_incoherent_impl)
-                {
+            for &impl_item in items {
+                if !self.tcx.has_attr(impl_item, sym::rustc_allow_incoherent_impl) {
+                    let impl_span = self.tcx.def_span(impl_def_id);
                     struct_span_err!(
                         self.tcx.sess,
-                        item.span,
+                        impl_span,
                         E0390,
                         "cannot define inherent `impl` for a type outside of the crate where the type is defined",
                     )
                     .help(INTO_DEFINING_CRATE)
-                    .span_help(impl_item.span, ADD_ATTR)
+                    .span_help(self.tcx.def_span(impl_item), ADD_ATTR)
                     .emit();
                     return;
                 }
             }
 
             if let Some(simp) = simplify_type(self.tcx, self_ty, TreatParams::AsInfer) {
-                self.impls_map.incoherent_impls.entry(simp).or_default().push(impl_def_id.def_id);
+                self.impls_map.incoherent_impls.entry(simp).or_default().push(impl_def_id);
             } else {
                 bug!("unexpected self type: {:?}", self_ty);
             }
         } else {
+            let impl_span = self.tcx.def_span(impl_def_id);
             struct_span_err!(
                 self.tcx.sess,
-                item.span,
+                impl_span,
                 E0116,
                 "cannot define inherent `impl` for a type outside of the crate \
                               where the type is defined"
             )
-            .span_label(item.span, "impl for type defined outside of crate.")
+            .span_label(impl_span, "impl for type defined outside of crate.")
             .note("define and implement a trait or new type instead")
             .emit();
         }
     }
 
-    fn check_primitive_impl(
-        &mut self,
-        impl_def_id: LocalDefId,
-        ty: Ty<'tcx>,
-        items: &[hir::ImplItemRef],
-        span: Span,
-    ) {
+    fn check_primitive_impl(&mut self, impl_def_id: LocalDefId, ty: Ty<'tcx>) {
+        let items = self.tcx.associated_item_def_ids(impl_def_id);
         if !self.tcx.hir().rustc_coherence_is_core() {
             if self.tcx.features().rustc_attrs {
-                for item in items {
-                    if !self
-                        .tcx
-                        .has_attr(item.id.owner_id.to_def_id(), sym::rustc_allow_incoherent_impl)
-                    {
+                for &impl_item in items {
+                    if !self.tcx.has_attr(impl_item, sym::rustc_allow_incoherent_impl) {
+                        let span = self.tcx.def_span(impl_def_id);
                         struct_span_err!(
                             self.tcx.sess,
                             span,
@@ -144,12 +133,13 @@ impl<'tcx> InherentCollect<'tcx> {
                             "cannot define inherent `impl` for primitive types outside of `core`",
                         )
                         .help(INTO_CORE)
-                        .span_help(item.span, ADD_ATTR)
+                        .span_help(self.tcx.def_span(impl_item), ADD_ATTR)
                         .emit();
                         return;
                     }
                 }
             } else {
+                let span = self.tcx.def_span(impl_def_id);
                 let mut err = struct_span_err!(
                     self.tcx.sess,
                     span,
@@ -177,34 +167,27 @@ impl<'tcx> InherentCollect<'tcx> {
     }
 
     fn check_item(&mut self, id: hir::ItemId) {
-        if !matches!(self.tcx.def_kind(id.owner_id), DefKind::Impl) {
+        if !matches!(self.tcx.def_kind(id.owner_id), DefKind::Impl { of_trait: false }) {
             return;
         }
 
-        let item = self.tcx.hir().item(id);
-        let hir::ItemKind::Impl(hir::Impl { of_trait: None, self_ty: ty, items, .. }) = item.kind else {
-            return;
-        };
-
-        let self_ty = self.tcx.type_of(item.owner_id);
+        let id = id.owner_id.def_id;
+        let item_span = self.tcx.def_span(id);
+        let self_ty = self.tcx.type_of(id).subst_identity();
         match *self_ty.kind() {
-            ty::Adt(def, _) => {
-                self.check_def_id(item, self_ty, def.did());
-            }
-            ty::Foreign(did) => {
-                self.check_def_id(item, self_ty, did);
-            }
+            ty::Adt(def, _) => self.check_def_id(id, self_ty, def.did()),
+            ty::Foreign(did) => self.check_def_id(id, self_ty, did),
             ty::Dynamic(data, ..) if data.principal_def_id().is_some() => {
-                self.check_def_id(item, self_ty, data.principal_def_id().unwrap());
+                self.check_def_id(id, self_ty, data.principal_def_id().unwrap());
             }
             ty::Dynamic(..) => {
                 struct_span_err!(
                     self.tcx.sess,
-                    ty.span,
+                    item_span,
                     E0785,
                     "cannot define inherent `impl` for a dyn auto trait"
                 )
-                .span_label(ty.span, "impl requires at least one non-auto trait")
+                .span_label(item_span, "impl requires at least one non-auto trait")
                 .note("define and implement a new trait or type instead")
                 .emit();
             }
@@ -220,18 +203,16 @@ impl<'tcx> InherentCollect<'tcx> {
             | ty::Ref(..)
             | ty::Never
             | ty::FnPtr(_)
-            | ty::Tuple(..) => {
-                self.check_primitive_impl(item.owner_id.def_id, self_ty, items, ty.span)
-            }
+            | ty::Tuple(..) => self.check_primitive_impl(id, self_ty),
             ty::Alias(..) | ty::Param(_) => {
                 let mut err = struct_span_err!(
                     self.tcx.sess,
-                    ty.span,
+                    item_span,
                     E0118,
                     "no nominal type found for inherent implementation"
                 );
 
-                err.span_label(ty.span, "impl requires a nominal type")
+                err.span_label(item_span, "impl requires a nominal type")
                     .note("either implement a trait on it or create a newtype to wrap it instead");
 
                 err.emit();
@@ -240,10 +221,11 @@ impl<'tcx> InherentCollect<'tcx> {
             | ty::Closure(..)
             | ty::Generator(..)
             | ty::GeneratorWitness(..)
+            | ty::GeneratorWitnessMIR(..)
             | ty::Bound(..)
             | ty::Placeholder(_)
             | ty::Infer(_) => {
-                bug!("unexpected impl self type of impl: {:?} {:?}", item.owner_id, self_ty);
+                bug!("unexpected impl self type of impl: {:?} {:?}", id, self_ty);
             }
             ty::Error(_) => {}
         }
diff --git a/compiler/rustc_hir_analysis/src/coherence/inherent_impls_overlap.rs b/compiler/rustc_hir_analysis/src/coherence/inherent_impls_overlap.rs
index a9331af4e..7bca4edcc 100644
--- a/compiler/rustc_hir_analysis/src/coherence/inherent_impls_overlap.rs
+++ b/compiler/rustc_hir_analysis/src/coherence/inherent_impls_overlap.rs
@@ -27,8 +27,8 @@ impl<'tcx> InherentOverlapChecker<'tcx> {
     /// namespace.
     fn impls_have_common_items(
         &self,
-        impl_items1: &ty::AssocItems<'_>,
-        impl_items2: &ty::AssocItems<'_>,
+        impl_items1: &ty::AssocItems,
+        impl_items2: &ty::AssocItems,
     ) -> bool {
         let mut impl_items1 = &impl_items1;
         let mut impl_items2 = &impl_items2;
@@ -38,10 +38,10 @@ impl<'tcx> InherentOverlapChecker<'tcx> {
             std::mem::swap(&mut impl_items1, &mut impl_items2);
         }
 
-        for item1 in impl_items1.in_definition_order() {
+        for &item1 in impl_items1.in_definition_order() {
             let collision = impl_items2
                 .filter_by_name_unhygienic(item1.name)
-                .any(|item2| self.compare_hygienically(item1, item2));
+                .any(|&item2| self.compare_hygienically(item1, item2));
 
             if collision {
                 return true;
@@ -51,7 +51,7 @@ impl<'tcx> InherentOverlapChecker<'tcx> {
         false
     }
 
-    fn compare_hygienically(&self, item1: &ty::AssocItem, item2: &ty::AssocItem) -> bool {
+    fn compare_hygienically(&self, item1: ty::AssocItem, item2: ty::AssocItem) -> bool {
         // Symbols and namespace match, compare hygienically.
         item1.kind.namespace() == item2.kind.namespace()
             && item1.ident(self.tcx).normalize_to_macros_2_0()
@@ -98,10 +98,10 @@ impl<'tcx> InherentOverlapChecker<'tcx> {
         let impl_items1 = self.tcx.associated_items(impl1);
         let impl_items2 = self.tcx.associated_items(impl2);
 
-        for item1 in impl_items1.in_definition_order() {
+        for &item1 in impl_items1.in_definition_order() {
             let collision = impl_items2
                 .filter_by_name_unhygienic(item1.name)
-                .find(|item2| self.compare_hygienically(item1, item2));
+                .find(|&&item2| self.compare_hygienically(item1, item2));
 
             if let Some(item2) = collision {
                 let name = item1.ident(self.tcx).normalize_to_macros_2_0();
diff --git a/compiler/rustc_hir_analysis/src/coherence/mod.rs b/compiler/rustc_hir_analysis/src/coherence/mod.rs
index d3b5778ba..23490bc09 100644
--- a/compiler/rustc_hir_analysis/src/coherence/mod.rs
+++ b/compiler/rustc_hir_analysis/src/coherence/mod.rs
@@ -8,7 +8,7 @@
 use rustc_errors::{error_code, struct_span_err};
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_middle::ty::query::Providers;
-use rustc_middle::ty::{self, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, TyCtxt, TypeVisitableExt};
 use rustc_span::sym;
 use rustc_trait_selection::traits;
 
@@ -169,7 +169,7 @@ fn check_object_overlap<'tcx>(
         });
 
         for component_def_id in component_def_ids {
-            if !tcx.is_object_safe(component_def_id) {
+            if !tcx.check_is_object_safe(component_def_id) {
                 // Without the 'object_safe_for_dispatch' feature this is an error
                 // which will be reported by wfcheck. Ignore it here.
                 // This is tested by `coherence-impl-trait-for-trait-object-safe.rs`.
diff --git a/compiler/rustc_hir_analysis/src/coherence/orphan.rs b/compiler/rustc_hir_analysis/src/coherence/orphan.rs
index 95b03eb82..1f2de3f21 100644
--- a/compiler/rustc_hir_analysis/src/coherence/orphan.rs
+++ b/compiler/rustc_hir_analysis/src/coherence/orphan.rs
@@ -8,7 +8,8 @@ use rustc_hir as hir;
 use rustc_middle::ty::subst::InternalSubsts;
 use rustc_middle::ty::util::IgnoreRegions;
 use rustc_middle::ty::{
-    self, ImplPolarity, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitor,
+    self, AliasKind, ImplPolarity, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitableExt,
+    TypeVisitor,
 };
 use rustc_session::lint;
 use rustc_span::def_id::{DefId, LocalDefId};
@@ -39,25 +40,27 @@ fn do_orphan_check_impl<'tcx>(
 ) -> Result<(), ErrorGuaranteed> {
     let trait_def_id = trait_ref.def_id;
 
-    let item = tcx.hir().expect_item(def_id);
-    let hir::ItemKind::Impl(impl_) = item.kind else {
-        bug!("{:?} is not an impl: {:?}", def_id, item);
-    };
-    let sp = tcx.def_span(def_id);
-    let tr = impl_.of_trait.as_ref().unwrap();
-
-    match traits::orphan_check(tcx, item.owner_id.to_def_id()) {
+    match traits::orphan_check(tcx, def_id.to_def_id()) {
         Ok(()) => {}
-        Err(err) => emit_orphan_check_error(
-            tcx,
-            sp,
-            item.span,
-            tr.path.span,
-            trait_ref,
-            impl_.self_ty.span,
-            &impl_.generics,
-            err,
-        )?,
+        Err(err) => {
+            let item = tcx.hir().expect_item(def_id);
+            let hir::ItemKind::Impl(impl_) = item.kind else {
+                bug!("{:?} is not an impl: {:?}", def_id, item);
+            };
+            let tr = impl_.of_trait.as_ref().unwrap();
+            let sp = tcx.def_span(def_id);
+
+            emit_orphan_check_error(
+                tcx,
+                sp,
+                item.span,
+                tr.path.span,
+                trait_ref,
+                impl_.self_ty.span,
+                &impl_.generics,
+                err,
+            )?
+        }
     }
 
     // In addition to the above rules, we restrict impls of auto traits
@@ -86,7 +89,7 @@ fn do_orphan_check_impl<'tcx>(
     // struct B { }
     // impl Foo for A { }
     // impl Foo for B { }
-    // impl !Send for (A, B) { }
+    // impl !Foo for (A, B) { }
     // ```
     //
     // This final impl is legal according to the orphan
@@ -99,50 +102,198 @@ fn do_orphan_check_impl<'tcx>(
         tcx.trait_is_auto(trait_def_id)
     );
 
-    if tcx.trait_is_auto(trait_def_id) && !trait_def_id.is_local() {
+    if tcx.trait_is_auto(trait_def_id) {
         let self_ty = trait_ref.self_ty();
-        let opt_self_def_id = match *self_ty.kind() {
-            ty::Adt(self_def, _) => Some(self_def.did()),
-            ty::Foreign(did) => Some(did),
-            _ => None,
-        };
 
-        let msg = match opt_self_def_id {
-            // We only want to permit nominal types, but not *all* nominal types.
-            // They must be local to the current crate, so that people
-            // can't do `unsafe impl Send for Rc<SomethingLocal>` or
-            // `impl !Send for Box<SomethingLocalAndSend>`.
-            Some(self_def_id) => {
-                if self_def_id.is_local() {
-                    None
+        // If the impl is in the same crate as the auto-trait, almost anything
+        // goes.
+        //
+        //     impl MyAuto for Rc<Something> {}  // okay
+        //     impl<T> !MyAuto for *const T {}   // okay
+        //     impl<T> MyAuto for T {}           // okay
+        //
+        // But there is one important exception: implementing for a trait object
+        // is not allowed.
+        //
+        //     impl MyAuto for dyn Trait {}      // NOT OKAY
+        //     impl<T: ?Sized> MyAuto for T {}   // NOT OKAY
+        //
+        // With this restriction, it's guaranteed that an auto-trait is
+        // implemented for a trait object if and only if the auto-trait is one
+        // of the trait object's trait bounds (or a supertrait of a bound). In
+        // other words `dyn Trait + AutoTrait` always implements AutoTrait,
+        // while `dyn Trait` never implements AutoTrait.
+        //
+        // This is necessary in order for autotrait bounds on methods of trait
+        // objects to be sound.
+        //
+        //     auto trait AutoTrait {}
+        //
+        //     trait ObjectSafeTrait {
+        //         fn f(&self) where Self: AutoTrait;
+        //     }
+        //
+        // We can allow f to be called on `dyn ObjectSafeTrait + AutoTrait`.
+        //
+        // If we didn't deny `impl AutoTrait for dyn Trait`, it would be unsound
+        // for the ObjectSafeTrait shown above to be object safe because someone
+        // could take some type implementing ObjectSafeTrait but not AutoTrait,
+        // unsize it to `dyn ObjectSafeTrait`, and call .f() which has no
+        // concrete implementation (issue #50781).
+        enum LocalImpl {
+            Allow,
+            Disallow { problematic_kind: &'static str },
+        }
+
+        // If the auto-trait is from a dependency, it must only be getting
+        // implemented for a nominal type, and specifically one local to the
+        // current crate.
+        //
+        //     impl<T> Sync for MyStruct<T> {}   // okay
+        //
+        //     impl Sync for Rc<MyStruct> {}     // NOT OKAY
+        enum NonlocalImpl {
+            Allow,
+            DisallowBecauseNonlocal,
+            DisallowOther,
+        }
+
+        // Exhaustive match considering that this logic is essential for
+        // soundness.
+        let (local_impl, nonlocal_impl) = match self_ty.kind() {
+            // struct Struct<T>;
+            // impl AutoTrait for Struct<Foo> {}
+            ty::Adt(self_def, _) => (
+                LocalImpl::Allow,
+                if self_def.did().is_local() {
+                    NonlocalImpl::Allow
                 } else {
-                    Some((
-                        format!(
-                            "cross-crate traits with a default impl, like `{}`, \
-                                    can only be implemented for a struct/enum type \
-                                    defined in the current crate",
-                            tcx.def_path_str(trait_def_id)
-                        ),
-                        "can't implement cross-crate trait for type in another crate",
-                    ))
-                }
+                    NonlocalImpl::DisallowBecauseNonlocal
+                },
+            ),
+
+            // extern { type OpaqueType; }
+            // impl AutoTrait for OpaqueType {}
+            ty::Foreign(did) => (
+                LocalImpl::Allow,
+                if did.is_local() {
+                    NonlocalImpl::Allow
+                } else {
+                    NonlocalImpl::DisallowBecauseNonlocal
+                },
+            ),
+
+            // impl AutoTrait for dyn Trait {}
+            ty::Dynamic(..) => (
+                LocalImpl::Disallow { problematic_kind: "trait object" },
+                NonlocalImpl::DisallowOther,
+            ),
+
+            // impl<T> AutoTrait for T {}
+            // impl<T: ?Sized> AutoTrait for T {}
+            ty::Param(..) => (
+                if self_ty.is_sized(tcx, tcx.param_env(def_id)) {
+                    LocalImpl::Allow
+                } else {
+                    LocalImpl::Disallow { problematic_kind: "generic type" }
+                },
+                NonlocalImpl::DisallowOther,
+            ),
+
+            // trait Id { type This: ?Sized; }
+            // impl<T: ?Sized> Id for T {
+            //     type This = T;
+            // }
+            // impl<T: ?Sized> AutoTrait for <T as Id>::This {}
+            ty::Alias(AliasKind::Projection, _) => (
+                LocalImpl::Disallow { problematic_kind: "associated type" },
+                NonlocalImpl::DisallowOther,
+            ),
+
+            // type Opaque = impl Trait;
+            // impl AutoTrait for Opaque {}
+            ty::Alias(AliasKind::Opaque, _) => (
+                LocalImpl::Disallow { problematic_kind: "opaque type" },
+                NonlocalImpl::DisallowOther,
+            ),
+
+            ty::Bool
+            | ty::Char
+            | ty::Int(..)
+            | ty::Uint(..)
+            | ty::Float(..)
+            | ty::Str
+            | ty::Array(..)
+            | ty::Slice(..)
+            | ty::RawPtr(..)
+            | ty::Ref(..)
+            | ty::FnDef(..)
+            | ty::FnPtr(..)
+            | ty::Never
+            | ty::Tuple(..) => (LocalImpl::Allow, NonlocalImpl::DisallowOther),
+
+            ty::Closure(..)
+            | ty::Generator(..)
+            | ty::GeneratorWitness(..)
+            | ty::GeneratorWitnessMIR(..)
+            | ty::Bound(..)
+            | ty::Placeholder(..)
+            | ty::Infer(..) => {
+                let sp = tcx.def_span(def_id);
+                span_bug!(sp, "weird self type for autotrait impl")
             }
-            _ => Some((
-                format!(
-                    "cross-crate traits with a default impl, like `{}`, can \
-                                only be implemented for a struct/enum type, not `{}`",
-                    tcx.def_path_str(trait_def_id),
-                    self_ty
-                ),
-                "can't implement cross-crate trait with a default impl for \
-                        non-struct/enum type",
-            )),
+
+            ty::Error(..) => (LocalImpl::Allow, NonlocalImpl::Allow),
         };
 
-        if let Some((msg, label)) = msg {
-            let reported =
-                struct_span_err!(tcx.sess, sp, E0321, "{}", msg).span_label(sp, label).emit();
-            return Err(reported);
+        if trait_def_id.is_local() {
+            match local_impl {
+                LocalImpl::Allow => {}
+                LocalImpl::Disallow { problematic_kind } => {
+                    let msg = format!(
+                        "traits with a default impl, like `{trait}`, \
+                                cannot be implemented for {problematic_kind} `{self_ty}`",
+                        trait = tcx.def_path_str(trait_def_id),
+                    );
+                    let label = format!(
+                        "a trait object implements `{trait}` if and only if `{trait}` \
+                                is one of the trait object's trait bounds",
+                        trait = tcx.def_path_str(trait_def_id),
+                    );
+                    let sp = tcx.def_span(def_id);
+                    let reported =
+                        struct_span_err!(tcx.sess, sp, E0321, "{}", msg).note(label).emit();
+                    return Err(reported);
+                }
+            }
+        } else {
+            if let Some((msg, label)) = match nonlocal_impl {
+                NonlocalImpl::Allow => None,
+                NonlocalImpl::DisallowBecauseNonlocal => Some((
+                    format!(
+                        "cross-crate traits with a default impl, like `{}`, \
+                                can only be implemented for a struct/enum type \
+                                defined in the current crate",
+                        tcx.def_path_str(trait_def_id)
+                    ),
+                    "can't implement cross-crate trait for type in another crate",
+                )),
+                NonlocalImpl::DisallowOther => Some((
+                    format!(
+                        "cross-crate traits with a default impl, like `{}`, can \
+                                only be implemented for a struct/enum type, not `{}`",
+                        tcx.def_path_str(trait_def_id),
+                        self_ty
+                    ),
+                    "can't implement cross-crate trait with a default impl for \
+                            non-struct/enum type",
+                )),
+            } {
+                let sp = tcx.def_span(def_id);
+                let reported =
+                    struct_span_err!(tcx.sess, sp, E0321, "{}", msg).span_label(sp, label).emit();
+                return Err(reported);
+            }
         }
     }
 
@@ -381,7 +532,7 @@ fn lint_auto_trait_impl<'tcx>(
         }),
         |lint| {
             let item_span = tcx.def_span(self_type_did);
-            let self_descr = tcx.def_kind(self_type_did).descr(self_type_did);
+            let self_descr = tcx.def_descr(self_type_did);
             match arg {
                 ty::util::NotUniqueParam::DuplicateParam(arg) => {
                     lint.note(&format!("`{}` is mentioned multiple times", arg));
@@ -409,7 +560,7 @@ fn fast_reject_auto_impl<'tcx>(tcx: TyCtxt<'tcx>, trait_def_id: DefId, self_ty:
         seen: FxHashSet<DefId>,
     }
 
-    impl<'tcx> TypeVisitor<'tcx> for DisableAutoTraitVisitor<'tcx> {
+    impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for DisableAutoTraitVisitor<'tcx> {
         type BreakTy = ();
         fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
             let tcx = self.tcx;
diff --git a/compiler/rustc_hir_analysis/src/coherence/unsafety.rs b/compiler/rustc_hir_analysis/src/coherence/unsafety.rs
index fe6119dce..c6b161713 100644
--- a/compiler/rustc_hir_analysis/src/coherence/unsafety.rs
+++ b/compiler/rustc_hir_analysis/src/coherence/unsafety.rs
@@ -3,15 +3,13 @@
 
 use rustc_errors::struct_span_err;
 use rustc_hir as hir;
-use rustc_hir::def::DefKind;
 use rustc_hir::Unsafety;
 use rustc_middle::ty::TyCtxt;
 use rustc_span::def_id::LocalDefId;
 
 pub(super) fn check_item(tcx: TyCtxt<'_>, def_id: LocalDefId) {
-    debug_assert!(matches!(tcx.def_kind(def_id), DefKind::Impl));
     let item = tcx.hir().expect_item(def_id);
-    let hir::ItemKind::Impl(impl_) = item.kind else { bug!() };
+    let impl_ = item.expect_impl();
 
     if let Some(trait_ref) = tcx.impl_trait_ref(item.owner_id) {
         let trait_ref = trait_ref.subst_identity();
diff --git a/compiler/rustc_hir_analysis/src/collect.rs b/compiler/rustc_hir_analysis/src/collect.rs
index c17778ce8..604d54caf 100644
--- a/compiler/rustc_hir_analysis/src/collect.rs
+++ b/compiler/rustc_hir_analysis/src/collect.rs
@@ -25,7 +25,7 @@ use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_hir::{GenericParamKind, Node};
-use rustc_infer::infer::TyCtxtInferExt;
+use rustc_infer::infer::{InferCtxt, TyCtxtInferExt};
 use rustc_infer::traits::ObligationCause;
 use rustc_middle::hir::nested_filter;
 use rustc_middle::ty::query::Providers;
@@ -41,8 +41,8 @@ use std::iter;
 
 mod generics_of;
 mod item_bounds;
-mod lifetimes;
 mod predicates_of;
+mod resolve_bound_vars;
 mod type_of;
 
 ///////////////////////////////////////////////////////////////////////////
@@ -53,7 +53,7 @@ fn collect_mod_item_types(tcx: TyCtxt<'_>, module_def_id: LocalDefId) {
 }
 
 pub fn provide(providers: &mut Providers) {
-    lifetimes::provide(providers);
+    resolve_bound_vars::provide(providers);
     *providers = Providers {
         opt_const_param_of: type_of::opt_const_param_of,
         type_of: type_of::type_of,
@@ -458,13 +458,11 @@ impl<'tcx> AstConv<'tcx> for ItemCtxt<'tcx> {
                                         self.tcx.replace_late_bound_regions_uncached(
                                             poly_trait_ref,
                                             |_| {
-                                                self.tcx.mk_region(ty::ReEarlyBound(
-                                                    ty::EarlyBoundRegion {
-                                                        def_id: item_def_id,
-                                                        index: 0,
-                                                        name: Symbol::intern(&lt_name),
-                                                    },
-                                                ))
+                                                self.tcx.mk_re_early_bound(ty::EarlyBoundRegion {
+                                                    def_id: item_def_id,
+                                                    index: 0,
+                                                    name: Symbol::intern(&lt_name),
+                                                })
                                             }
                                         ),
                                     ),
@@ -501,7 +499,7 @@ impl<'tcx> AstConv<'tcx> for ItemCtxt<'tcx> {
                 }
                 _ => {}
             }
-            self.tcx().ty_error_with_guaranteed(err.emit())
+            self.tcx().ty_error(err.emit())
         }
     }
 
@@ -517,6 +515,10 @@ impl<'tcx> AstConv<'tcx> for ItemCtxt<'tcx> {
     fn record_ty(&self, _hir_id: hir::HirId, _ty: Ty<'tcx>, _span: Span) {
         // There's no place to record types from signatures?
     }
+
+    fn infcx(&self) -> Option<&InferCtxt<'tcx>> {
+        None
+    }
 }
 
 /// Synthesize a new lifetime name that doesn't clash with any of the lifetimes already present.
@@ -903,7 +905,7 @@ fn adt_def(tcx: TyCtxt<'_>, def_id: DefId) -> ty::AdtDef<'_> {
         }
         _ => bug!(),
     };
-    tcx.alloc_adt_def(def_id.to_def_id(), kind, variants, repr)
+    tcx.mk_adt_def(def_id.to_def_id(), kind, variants, repr)
 }
 
 fn trait_def(tcx: TyCtxt<'_>, def_id: DefId) -> ty::TraitDef {
@@ -930,9 +932,10 @@ fn trait_def(tcx: TyCtxt<'_>, def_id: DefId) -> ty::TraitDef {
     }
 
     let is_marker = tcx.has_attr(def_id, sym::marker);
+    let rustc_coinductive = tcx.has_attr(def_id, sym::rustc_coinductive);
     let skip_array_during_method_dispatch =
         tcx.has_attr(def_id, sym::rustc_skip_array_during_method_dispatch);
-    let spec_kind = if tcx.has_attr(def_id, sym::rustc_unsafe_specialization_marker) {
+    let specialization_kind = if tcx.has_attr(def_id, sym::rustc_unsafe_specialization_marker) {
         ty::trait_def::TraitSpecializationKind::Marker
     } else if tcx.has_attr(def_id, sym::rustc_specialization_trait) {
         ty::trait_def::TraitSpecializationKind::AlwaysApplicable
@@ -1032,16 +1035,17 @@ fn trait_def(tcx: TyCtxt<'_>, def_id: DefId) -> ty::TraitDef {
             no_dups.then_some(list)
         });
 
-    ty::TraitDef::new(
+    ty::TraitDef {
         def_id,
         unsafety,
         paren_sugar,
-        is_auto,
+        has_auto_impl: is_auto,
         is_marker,
+        is_coinductive: rustc_coinductive || is_auto,
         skip_array_during_method_dispatch,
-        spec_kind,
+        specialization_kind,
         must_implement_one_of,
-    )
+    }
 }
 
 fn are_suggestable_generic_args(generic_args: &[hir::GenericArg<'_>]) -> bool {
@@ -1087,7 +1091,7 @@ pub fn get_infer_ret_ty<'hir>(output: &'hir hir::FnRetTy<'hir>) -> Option<&'hir
 }
 
 #[instrument(level = "debug", skip(tcx))]
-fn fn_sig(tcx: TyCtxt<'_>, def_id: DefId) -> ty::PolyFnSig<'_> {
+fn fn_sig(tcx: TyCtxt<'_>, def_id: DefId) -> ty::EarlyBinder<ty::PolyFnSig<'_>> {
     use rustc_hir::Node::*;
     use rustc_hir::*;
 
@@ -1096,7 +1100,7 @@ fn fn_sig(tcx: TyCtxt<'_>, def_id: DefId) -> ty::PolyFnSig<'_> {
 
     let icx = ItemCtxt::new(tcx, def_id.to_def_id());
 
-    match tcx.hir().get(hir_id) {
+    let output = match tcx.hir().get(hir_id) {
         TraitItem(hir::TraitItem {
             kind: TraitItemKind::Fn(sig, TraitFn::Provided(_)),
             generics,
@@ -1139,8 +1143,8 @@ fn fn_sig(tcx: TyCtxt<'_>, def_id: DefId) -> ty::PolyFnSig<'_> {
         }
 
         Ctor(data) | Variant(hir::Variant { data, .. }) if data.ctor().is_some() => {
-            let ty = tcx.type_of(tcx.hir().get_parent_item(hir_id));
-            let inputs = data.fields().iter().map(|f| tcx.type_of(f.def_id));
+            let ty = tcx.type_of(tcx.hir().get_parent_item(hir_id)).subst_identity();
+            let inputs = data.fields().iter().map(|f| tcx.type_of(f.def_id).subst_identity());
             ty::Binder::dummy(tcx.mk_fn_sig(
                 inputs,
                 ty,
@@ -1169,7 +1173,8 @@ fn fn_sig(tcx: TyCtxt<'_>, def_id: DefId) -> ty::PolyFnSig<'_> {
         x => {
             bug!("unexpected sort of node in fn_sig(): {:?}", x);
         }
-    }
+    };
+    ty::EarlyBinder(output)
 }
 
 fn infer_return_ty_for_fn_sig<'tcx>(
@@ -1194,28 +1199,22 @@ fn infer_return_ty_for_fn_sig<'tcx>(
             visitor.visit_ty(ty);
             let mut diag = bad_placeholder(tcx, visitor.0, "return type");
             let ret_ty = fn_sig.output();
-            if ret_ty.is_suggestable(tcx, false) {
+            if let Some(ret_ty) = ret_ty.make_suggestable(tcx, false) {
                 diag.span_suggestion(
                     ty.span,
                     "replace with the correct return type",
                     ret_ty,
                     Applicability::MachineApplicable,
                 );
-            } else if matches!(ret_ty.kind(), ty::FnDef(..)) {
-                let fn_sig = ret_ty.fn_sig(tcx);
-                if fn_sig
-                    .skip_binder()
-                    .inputs_and_output
-                    .iter()
-                    .all(|t| t.is_suggestable(tcx, false))
-                {
-                    diag.span_suggestion(
-                        ty.span,
-                        "replace with the correct return type",
-                        fn_sig,
-                        Applicability::MachineApplicable,
-                    );
-                }
+            } else if matches!(ret_ty.kind(), ty::FnDef(..))
+                && let Some(fn_sig) = ret_ty.fn_sig(tcx).make_suggestable(tcx, false)
+            {
+                diag.span_suggestion(
+                    ty.span,
+                    "replace with the correct return type",
+                    fn_sig,
+                    Applicability::MachineApplicable,
+                );
             } else if let Some(sugg) = suggest_impl_trait(tcx, ret_ty, ty.span, hir_id, def_id) {
                 diag.span_suggestion(
                     ty.span,
@@ -1248,11 +1247,12 @@ fn infer_return_ty_for_fn_sig<'tcx>(
     }
 }
 
+// FIXME(vincenzopalazzo): remove the hir item when the refactoring is stable
 fn suggest_impl_trait<'tcx>(
     tcx: TyCtxt<'tcx>,
     ret_ty: Ty<'tcx>,
     span: Span,
-    hir_id: hir::HirId,
+    _hir_id: hir::HirId,
     def_id: LocalDefId,
 ) -> Option<String> {
     let format_as_assoc: fn(_, _, _, _, _) -> _ =
@@ -1274,9 +1274,7 @@ fn suggest_impl_trait<'tcx>(
             let trait_name = tcx.item_name(trait_def_id);
             let args_tuple = substs.type_at(1);
             let ty::Tuple(types) = *args_tuple.kind() else { return None; };
-            if !types.is_suggestable(tcx, false) {
-                return None;
-            }
+            let types = types.make_suggestable(tcx, false)?;
             let maybe_ret =
                 if item_ty.is_unit() { String::new() } else { format!(" -> {item_ty}") };
             Some(format!(
@@ -1324,14 +1322,14 @@ fn suggest_impl_trait<'tcx>(
         }
         let ocx = ObligationCtxt::new_in_snapshot(&infcx);
         let item_ty = ocx.normalize(
-            &ObligationCause::misc(span, hir_id),
+            &ObligationCause::misc(span, def_id),
             param_env,
             tcx.mk_projection(assoc_item_def_id, substs),
         );
         // FIXME(compiler-errors): We may benefit from resolving regions here.
         if ocx.select_where_possible().is_empty()
             && let item_ty = infcx.resolve_vars_if_possible(item_ty)
-            && item_ty.is_suggestable(tcx, false)
+            && let Some(item_ty) = item_ty.make_suggestable(tcx, false)
             && let Some(sugg) = formatter(tcx, infcx.resolve_vars_if_possible(substs), trait_def_id, assoc_item_def_id, item_ty)
         {
             return Some(sugg);
@@ -1342,13 +1340,12 @@ fn suggest_impl_trait<'tcx>(
 
 fn impl_trait_ref(tcx: TyCtxt<'_>, def_id: DefId) -> Option<ty::EarlyBinder<ty::TraitRef<'_>>> {
     let icx = ItemCtxt::new(tcx, def_id);
-    let item = tcx.hir().expect_item(def_id.expect_local());
-    let hir::ItemKind::Impl(impl_) = item.kind else { bug!() };
+    let impl_ = tcx.hir().expect_item(def_id.expect_local()).expect_impl();
     impl_
         .of_trait
         .as_ref()
         .map(|ast_trait_ref| {
-            let selfty = tcx.type_of(def_id);
+            let selfty = tcx.type_of(def_id).subst_identity();
             icx.astconv().instantiate_mono_trait_ref(
                 ast_trait_ref,
                 selfty,
diff --git a/compiler/rustc_hir_analysis/src/collect/generics_of.rs b/compiler/rustc_hir_analysis/src/collect/generics_of.rs
index 014ee9fcc..127d4fa90 100644
--- a/compiler/rustc_hir_analysis/src/collect/generics_of.rs
+++ b/compiler/rustc_hir_analysis/src/collect/generics_of.rs
@@ -1,4 +1,4 @@
-use crate::middle::resolve_lifetime as rl;
+use crate::middle::resolve_bound_vars as rbv;
 use hir::{
     intravisit::{self, Visitor},
     GenericParamKind, HirId, Node,
@@ -394,10 +394,16 @@ fn has_late_bound_regions<'tcx>(tcx: TyCtxt<'tcx>, node: Node<'tcx>) -> Option<S
                 return;
             }
 
-            match self.tcx.named_region(lt.hir_id) {
-                Some(rl::Region::Static | rl::Region::EarlyBound(..)) => {}
-                Some(rl::Region::LateBound(debruijn, _, _)) if debruijn < self.outer_index => {}
-                Some(rl::Region::LateBound(..) | rl::Region::Free(..)) | None => {
+            match self.tcx.named_bound_var(lt.hir_id) {
+                Some(rbv::ResolvedArg::StaticLifetime | rbv::ResolvedArg::EarlyBound(..)) => {}
+                Some(rbv::ResolvedArg::LateBound(debruijn, _, _))
+                    if debruijn < self.outer_index => {}
+                Some(
+                    rbv::ResolvedArg::LateBound(..)
+                    | rbv::ResolvedArg::Free(..)
+                    | rbv::ResolvedArg::Error(_),
+                )
+                | None => {
                     self.has_late_bound_regions = Some(lt.ident.span);
                 }
             }
diff --git a/compiler/rustc_hir_analysis/src/collect/item_bounds.rs b/compiler/rustc_hir_analysis/src/collect/item_bounds.rs
index 8d479f1c3..9cf3ff65a 100644
--- a/compiler/rustc_hir_analysis/src/collect/item_bounds.rs
+++ b/compiler/rustc_hir_analysis/src/collect/item_bounds.rs
@@ -103,7 +103,7 @@ pub(super) fn item_bounds(
     tcx: TyCtxt<'_>,
     def_id: DefId,
 ) -> ty::EarlyBinder<&'_ ty::List<ty::Predicate<'_>>> {
-    let bounds = tcx.mk_predicates(
+    let bounds = tcx.mk_predicates_from_iter(
         util::elaborate_predicates(
             tcx,
             tcx.explicit_item_bounds(def_id).iter().map(|&(bound, _span)| bound),
diff --git a/compiler/rustc_hir_analysis/src/collect/predicates_of.rs b/compiler/rustc_hir_analysis/src/collect/predicates_of.rs
index 46b277d98..2badd66e3 100644
--- a/compiler/rustc_hir_analysis/src/collect/predicates_of.rs
+++ b/compiler/rustc_hir_analysis/src/collect/predicates_of.rs
@@ -9,8 +9,8 @@ use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_middle::ty::subst::InternalSubsts;
-use rustc_middle::ty::ToPredicate;
 use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_middle::ty::{GenericPredicates, ToPredicate};
 use rustc_span::symbol::{sym, Ident};
 use rustc_span::{Span, DUMMY_SP};
 
@@ -151,7 +151,8 @@ fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::GenericP
     trace!(?generics);
 
     // Collect the predicates that were written inline by the user on each
-    // type parameter (e.g., `<T: Foo>`).
+    // type parameter (e.g., `<T: Foo>`). Also add `ConstArgHasType` predicates
+    // for each const parameter.
     for param in ast_generics.params {
         match param.kind {
             // We already dealt with early bound lifetimes above.
@@ -175,7 +176,19 @@ fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::GenericP
                 trace!(?predicates);
             }
             GenericParamKind::Const { .. } => {
-                // Bounds on const parameters are currently not possible.
+                let name = param.name.ident().name;
+                let param_const = ty::ParamConst::new(index, name);
+
+                let ct_ty = tcx.type_of(param.def_id.to_def_id()).subst_identity();
+
+                let ct = tcx.mk_const(param_const, ct_ty);
+
+                let predicate = ty::Binder::dummy(ty::PredicateKind::Clause(
+                    ty::Clause::ConstArgHasType(ct, ct_ty),
+                ))
+                .to_predicate(tcx);
+                predicates.insert((predicate, param.span));
+
                 index += 1;
             }
         }
@@ -251,7 +264,7 @@ fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::GenericP
     // in trait checking. See `setup_constraining_predicates`
     // for details.
     if let Node::Item(&Item { kind: ItemKind::Impl { .. }, .. }) = node {
-        let self_ty = tcx.type_of(def_id);
+        let self_ty = tcx.type_of(def_id).subst_identity();
         let trait_ref = tcx.impl_trait_ref(def_id).map(ty::EarlyBinder::subst_identity);
         cgp::setup_constraining_predicates(
             tcx,
@@ -280,15 +293,15 @@ fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::GenericP
             }
 
             let hir::GenericParamKind::Lifetime { .. } = duplicate.kind else { continue };
-            let dup_def = tcx.hir().local_def_id(duplicate.hir_id).to_def_id();
+            let dup_def = duplicate.def_id.to_def_id();
 
             let Some(dup_index) = generics.param_def_id_to_index(tcx, dup_def) else { bug!() };
 
-            let dup_region = tcx.mk_region(ty::ReEarlyBound(ty::EarlyBoundRegion {
+            let dup_region = tcx.mk_re_early_bound(ty::EarlyBoundRegion {
                 def_id: dup_def,
                 index: dup_index,
                 name: duplicate.name.ident().name,
-            }));
+            });
             predicates.push((
                 ty::Binder::dummy(ty::PredicateKind::Clause(ty::Clause::RegionOutlives(
                     ty::OutlivesPredicate(orig_region, dup_region),
@@ -439,7 +452,9 @@ pub(super) fn explicit_predicates_of<'tcx>(
             let hir_id = tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
             let parent_def_id = tcx.hir().get_parent_item(hir_id);
 
-            if tcx.hir().opt_const_param_default_param_def_id(hir_id).is_some() {
+            if let Some(defaulted_param_def_id) =
+                tcx.hir().opt_const_param_default_param_def_id(hir_id)
+            {
                 // In `generics_of` we set the generics' parent to be our parent's parent which means that
                 // we lose out on the predicates of our actual parent if we dont return those predicates here.
                 // (See comment in `generics_of` for more information on why the parent shenanigans is necessary)
@@ -452,7 +467,39 @@ pub(super) fn explicit_predicates_of<'tcx>(
                 //
                 // In the above code we want the anon const to have predicates in its param env for `T: Trait`
                 // and we would be calling `explicit_predicates_of(Foo)` here
-                return tcx.explicit_predicates_of(parent_def_id);
+                let parent_preds = tcx.explicit_predicates_of(parent_def_id);
+
+                // If we dont filter out `ConstArgHasType` predicates then every single defaulted const parameter
+                // will ICE because of #106994. FIXME(generic_const_exprs): remove this when a more general solution
+                // to #106994 is implemented.
+                let filtered_predicates = parent_preds
+                    .predicates
+                    .into_iter()
+                    .filter(|(pred, _)| {
+                        if let ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(ct, _)) =
+                            pred.kind().skip_binder()
+                        {
+                            match ct.kind() {
+                                ty::ConstKind::Param(param_const) => {
+                                    let defaulted_param_idx = tcx
+                                        .generics_of(parent_def_id)
+                                        .param_def_id_to_index[&defaulted_param_def_id.to_def_id()];
+                                    param_const.index < defaulted_param_idx
+                                }
+                                _ => bug!(
+                                    "`ConstArgHasType` in `predicates_of`\
+                                 that isn't a `Param` const"
+                                ),
+                            }
+                        } else {
+                            true
+                        }
+                    })
+                    .cloned();
+                return GenericPredicates {
+                    parent: parent_preds.parent,
+                    predicates: { tcx.arena.alloc_from_iter(filtered_predicates) },
+                };
             }
 
             let parent_def_kind = tcx.def_kind(parent_def_id);
diff --git a/compiler/rustc_hir_analysis/src/collect/lifetimes.rs b/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs
index 359122d4e..65a9052a6 100644
--- a/compiler/rustc_hir_analysis/src/collect/lifetimes.rs
+++ b/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs
@@ -16,67 +16,72 @@ use rustc_hir::intravisit::{self, Visitor};
 use rustc_hir::{GenericArg, GenericParam, GenericParamKind, HirIdMap, LifetimeName, Node};
 use rustc_middle::bug;
 use rustc_middle::hir::nested_filter;
-use rustc_middle::middle::resolve_lifetime::*;
+use rustc_middle::middle::resolve_bound_vars::*;
 use rustc_middle::ty::{self, DefIdTree, TyCtxt, TypeSuperVisitable, TypeVisitor};
+use rustc_session::lint;
 use rustc_span::def_id::DefId;
 use rustc_span::symbol::{sym, Ident};
 use rustc_span::Span;
 use std::fmt;
 
+use crate::errors;
+
 trait RegionExt {
-    fn early(param: &GenericParam<'_>) -> (LocalDefId, Region);
+    fn early(param: &GenericParam<'_>) -> (LocalDefId, ResolvedArg);
 
-    fn late(index: u32, param: &GenericParam<'_>) -> (LocalDefId, Region);
+    fn late(index: u32, param: &GenericParam<'_>) -> (LocalDefId, ResolvedArg);
 
     fn id(&self) -> Option<DefId>;
 
-    fn shifted(self, amount: u32) -> Region;
+    fn shifted(self, amount: u32) -> ResolvedArg;
 }
 
-impl RegionExt for Region {
-    fn early(param: &GenericParam<'_>) -> (LocalDefId, Region) {
-        debug!("Region::early: def_id={:?}", param.def_id);
-        (param.def_id, Region::EarlyBound(param.def_id.to_def_id()))
+impl RegionExt for ResolvedArg {
+    fn early(param: &GenericParam<'_>) -> (LocalDefId, ResolvedArg) {
+        debug!("ResolvedArg::early: def_id={:?}", param.def_id);
+        (param.def_id, ResolvedArg::EarlyBound(param.def_id.to_def_id()))
     }
 
-    fn late(idx: u32, param: &GenericParam<'_>) -> (LocalDefId, Region) {
+    fn late(idx: u32, param: &GenericParam<'_>) -> (LocalDefId, ResolvedArg) {
         let depth = ty::INNERMOST;
         debug!(
-            "Region::late: idx={:?}, param={:?} depth={:?} def_id={:?}",
+            "ResolvedArg::late: idx={:?}, param={:?} depth={:?} def_id={:?}",
             idx, param, depth, param.def_id,
         );
-        (param.def_id, Region::LateBound(depth, idx, param.def_id.to_def_id()))
+        (param.def_id, ResolvedArg::LateBound(depth, idx, param.def_id.to_def_id()))
     }
 
     fn id(&self) -> Option<DefId> {
         match *self {
-            Region::Static => None,
+            ResolvedArg::StaticLifetime | ResolvedArg::Error(_) => None,
 
-            Region::EarlyBound(id) | Region::LateBound(_, _, id) | Region::Free(_, id) => Some(id),
+            ResolvedArg::EarlyBound(id)
+            | ResolvedArg::LateBound(_, _, id)
+            | ResolvedArg::Free(_, id) => Some(id),
         }
     }
 
-    fn shifted(self, amount: u32) -> Region {
+    fn shifted(self, amount: u32) -> ResolvedArg {
         match self {
-            Region::LateBound(debruijn, idx, id) => {
-                Region::LateBound(debruijn.shifted_in(amount), idx, id)
+            ResolvedArg::LateBound(debruijn, idx, id) => {
+                ResolvedArg::LateBound(debruijn.shifted_in(amount), idx, id)
             }
             _ => self,
         }
     }
 }
 
-/// Maps the id of each lifetime reference to the lifetime decl
+/// Maps the id of each bound variable reference to the variable decl
 /// that it corresponds to.
 ///
-/// FIXME. This struct gets converted to a `ResolveLifetimes` for
+/// FIXME. This struct gets converted to a `ResolveBoundVars` for
 /// actual use. It has the same data, but indexed by `LocalDefId`. This
 /// is silly.
 #[derive(Debug, Default)]
-struct NamedRegionMap {
-    // maps from every use of a named (not anonymous) lifetime to a
-    // `Region` describing how that region is bound
-    defs: HirIdMap<Region>,
+struct NamedVarMap {
+    // maps from every use of a named (not anonymous) bound var to a
+    // `ResolvedArg` describing how that variable is bound
+    defs: HirIdMap<ResolvedArg>,
 
     // Maps relevant hir items to the bound vars on them. These include:
     // - function defs
@@ -87,9 +92,9 @@ struct NamedRegionMap {
     late_bound_vars: HirIdMap<Vec<ty::BoundVariableKind>>,
 }
 
-struct LifetimeContext<'a, 'tcx> {
+struct BoundVarContext<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
-    map: &'a mut NamedRegionMap,
+    map: &'a mut NamedVarMap,
     scope: ScopeRef<'a>,
 }
 
@@ -102,7 +107,7 @@ enum Scope<'a> {
     Binder {
         /// We use an IndexMap here because we want these lifetimes in order
         /// for diagnostics.
-        lifetimes: FxIndexMap<LocalDefId, Region>,
+        bound_vars: FxIndexMap<LocalDefId, ResolvedArg>,
 
         scope_type: BinderScopeType,
 
@@ -141,7 +146,7 @@ enum Scope<'a> {
     /// inferred in a function body or potentially error outside one),
     /// for the default choice of lifetime in a trait object type.
     ObjectLifetimeDefault {
-        lifetime: Option<Region>,
+        lifetime: Option<ResolvedArg>,
         s: ScopeRef<'a>,
     },
 
@@ -150,7 +155,7 @@ enum Scope<'a> {
     /// lifetimes encountered when identifying the trait that an associated type
     /// is declared on.
     Supertrait {
-        lifetimes: Vec<ty::BoundVariableKind>,
+        bound_vars: Vec<ty::BoundVariableKind>,
         s: ScopeRef<'a>,
     },
 
@@ -158,6 +163,15 @@ enum Scope<'a> {
         s: ScopeRef<'a>,
     },
 
+    /// Disallows capturing non-lifetime binders from parent scopes.
+    ///
+    /// This is necessary for something like `for<T> [(); { /* references T */ }]:`,
+    /// since we don't do something more correct like replacing any captured
+    /// late-bound vars with early-bound params in the const's own generics.
+    AnonConstBoundary {
+        s: ScopeRef<'a>,
+    },
+
     Root {
         opt_parent_item: Option<LocalDefId>,
     },
@@ -185,9 +199,9 @@ struct TruncatedScopeDebug<'a>(&'a Scope<'a>);
 impl<'a> fmt::Debug for TruncatedScopeDebug<'a> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self.0 {
-            Scope::Binder { lifetimes, scope_type, hir_id, where_bound_origin, s: _ } => f
+            Scope::Binder { bound_vars, scope_type, hir_id, where_bound_origin, s: _ } => f
                 .debug_struct("Binder")
-                .field("lifetimes", lifetimes)
+                .field("bound_vars", bound_vars)
                 .field("scope_type", scope_type)
                 .field("hir_id", hir_id)
                 .field("where_bound_origin", where_bound_origin)
@@ -202,12 +216,13 @@ impl<'a> fmt::Debug for TruncatedScopeDebug<'a> {
                 .field("lifetime", lifetime)
                 .field("s", &"..")
                 .finish(),
-            Scope::Supertrait { lifetimes, s: _ } => f
+            Scope::Supertrait { bound_vars, s: _ } => f
                 .debug_struct("Supertrait")
-                .field("lifetimes", lifetimes)
+                .field("bound_vars", bound_vars)
                 .field("s", &"..")
                 .finish(),
             Scope::TraitRefBoundary { s: _ } => f.debug_struct("TraitRefBoundary").finish(),
+            Scope::AnonConstBoundary { s: _ } => f.debug_struct("AnonConstBoundary").finish(),
             Scope::Root { opt_parent_item } => {
                 f.debug_struct("Root").field("opt_parent_item", &opt_parent_item).finish()
             }
@@ -219,27 +234,27 @@ type ScopeRef<'a> = &'a Scope<'a>;
 
 pub(crate) fn provide(providers: &mut ty::query::Providers) {
     *providers = ty::query::Providers {
-        resolve_lifetimes,
+        resolve_bound_vars,
 
-        named_region_map: |tcx, id| tcx.resolve_lifetimes(id).defs.get(&id),
+        named_variable_map: |tcx, id| tcx.resolve_bound_vars(id).defs.get(&id),
         is_late_bound_map,
         object_lifetime_default,
-        late_bound_vars_map: |tcx, id| tcx.resolve_lifetimes(id).late_bound_vars.get(&id),
+        late_bound_vars_map: |tcx, id| tcx.resolve_bound_vars(id).late_bound_vars.get(&id),
 
         ..*providers
     };
 }
 
-/// Computes the `ResolveLifetimes` map that contains data for an entire `Item`.
+/// Computes the `ResolveBoundVars` map that contains data for an entire `Item`.
 /// You should not read the result of this query directly, but rather use
-/// `named_region_map`, `is_late_bound_map`, etc.
+/// `named_variable_map`, `is_late_bound_map`, etc.
 #[instrument(level = "debug", skip(tcx))]
-fn resolve_lifetimes(tcx: TyCtxt<'_>, local_def_id: hir::OwnerId) -> ResolveLifetimes {
-    let mut named_region_map =
-        NamedRegionMap { defs: Default::default(), late_bound_vars: Default::default() };
-    let mut visitor = LifetimeContext {
+fn resolve_bound_vars(tcx: TyCtxt<'_>, local_def_id: hir::OwnerId) -> ResolveBoundVars {
+    let mut named_variable_map =
+        NamedVarMap { defs: Default::default(), late_bound_vars: Default::default() };
+    let mut visitor = BoundVarContext {
         tcx,
-        map: &mut named_region_map,
+        map: &mut named_variable_map,
         scope: &Scope::Root { opt_parent_item: None },
     };
     match tcx.hir().owner(local_def_id) {
@@ -260,13 +275,13 @@ fn resolve_lifetimes(tcx: TyCtxt<'_>, local_def_id: hir::OwnerId) -> ResolveLife
         hir::OwnerNode::Crate(_) => {}
     }
 
-    let mut rl = ResolveLifetimes::default();
+    let mut rl = ResolveBoundVars::default();
 
-    for (hir_id, v) in named_region_map.defs {
+    for (hir_id, v) in named_variable_map.defs {
         let map = rl.defs.entry(hir_id.owner).or_default();
         map.insert(hir_id.local_id, v);
     }
-    for (hir_id, v) in named_region_map.late_bound_vars {
+    for (hir_id, v) in named_variable_map.late_bound_vars {
         let map = rl.late_bound_vars.entry(hir_id.owner).or_default();
         map.insert(hir_id.local_id, v);
     }
@@ -276,39 +291,53 @@ fn resolve_lifetimes(tcx: TyCtxt<'_>, local_def_id: hir::OwnerId) -> ResolveLife
     rl
 }
 
-fn late_region_as_bound_region(tcx: TyCtxt<'_>, region: &Region) -> ty::BoundVariableKind {
-    match region {
-        Region::LateBound(_, _, def_id) => {
+fn late_arg_as_bound_arg<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    arg: &ResolvedArg,
+    param: &GenericParam<'tcx>,
+) -> ty::BoundVariableKind {
+    match arg {
+        ResolvedArg::LateBound(_, _, def_id) => {
             let name = tcx.hir().name(tcx.hir().local_def_id_to_hir_id(def_id.expect_local()));
-            ty::BoundVariableKind::Region(ty::BrNamed(*def_id, name))
+            match param.kind {
+                GenericParamKind::Lifetime { .. } => {
+                    ty::BoundVariableKind::Region(ty::BrNamed(*def_id, name))
+                }
+                GenericParamKind::Type { .. } => {
+                    ty::BoundVariableKind::Ty(ty::BoundTyKind::Param(*def_id, name))
+                }
+                GenericParamKind::Const { .. } => ty::BoundVariableKind::Const,
+            }
         }
-        _ => bug!("{:?} is not a late region", region),
+        _ => bug!("{:?} is not a late argument", arg),
     }
 }
 
-impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
+impl<'a, 'tcx> BoundVarContext<'a, 'tcx> {
     /// Returns the binders in scope and the type of `Binder` that should be created for a poly trait ref.
     fn poly_trait_ref_binder_info(&mut self) -> (Vec<ty::BoundVariableKind>, BinderScopeType) {
         let mut scope = self.scope;
-        let mut supertrait_lifetimes = vec![];
+        let mut supertrait_bound_vars = vec![];
         loop {
             match scope {
                 Scope::Body { .. } | Scope::Root { .. } => {
                     break (vec![], BinderScopeType::Normal);
                 }
 
-                Scope::Elision { s, .. } | Scope::ObjectLifetimeDefault { s, .. } => {
+                Scope::Elision { s, .. }
+                | Scope::ObjectLifetimeDefault { s, .. }
+                | Scope::AnonConstBoundary { s } => {
                     scope = s;
                 }
 
-                Scope::Supertrait { s, lifetimes } => {
-                    supertrait_lifetimes = lifetimes.clone();
+                Scope::Supertrait { s, bound_vars } => {
+                    supertrait_bound_vars = bound_vars.clone();
                     scope = s;
                 }
 
                 Scope::TraitRefBoundary { .. } => {
                     // We should only see super trait lifetimes if there is a `Binder` above
-                    assert!(supertrait_lifetimes.is_empty());
+                    assert!(supertrait_bound_vars.is_empty());
                     break (vec![], BinderScopeType::Normal);
                 }
 
@@ -316,14 +345,64 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                     // Nested poly trait refs have the binders concatenated
                     let mut full_binders =
                         self.map.late_bound_vars.entry(*hir_id).or_default().clone();
-                    full_binders.extend(supertrait_lifetimes.into_iter());
+                    full_binders.extend(supertrait_bound_vars.into_iter());
                     break (full_binders, BinderScopeType::Concatenating);
                 }
             }
         }
     }
+
+    fn visit_poly_trait_ref_inner(
+        &mut self,
+        trait_ref: &'tcx hir::PolyTraitRef<'tcx>,
+        non_lifetime_binder_allowed: NonLifetimeBinderAllowed,
+    ) {
+        debug!("visit_poly_trait_ref(trait_ref={:?})", trait_ref);
+
+        let (mut binders, scope_type) = self.poly_trait_ref_binder_info();
+
+        let initial_bound_vars = binders.len() as u32;
+        let mut bound_vars: FxIndexMap<LocalDefId, ResolvedArg> = FxIndexMap::default();
+        let binders_iter =
+            trait_ref.bound_generic_params.iter().enumerate().map(|(late_bound_idx, param)| {
+                let pair = ResolvedArg::late(initial_bound_vars + late_bound_idx as u32, param);
+                let r = late_arg_as_bound_arg(self.tcx, &pair.1, param);
+                bound_vars.insert(pair.0, pair.1);
+                r
+            });
+        binders.extend(binders_iter);
+
+        if let NonLifetimeBinderAllowed::Deny(where_) = non_lifetime_binder_allowed {
+            deny_non_region_late_bound(self.tcx, &mut bound_vars, where_);
+        }
+
+        debug!(?binders);
+        self.record_late_bound_vars(trait_ref.trait_ref.hir_ref_id, binders);
+
+        // Always introduce a scope here, even if this is in a where clause and
+        // we introduced the binders around the bounded Ty. In that case, we
+        // just reuse the concatenation functionality also present in nested trait
+        // refs.
+        let scope = Scope::Binder {
+            hir_id: trait_ref.trait_ref.hir_ref_id,
+            bound_vars,
+            s: self.scope,
+            scope_type,
+            where_bound_origin: None,
+        };
+        self.with(scope, |this| {
+            walk_list!(this, visit_generic_param, trait_ref.bound_generic_params);
+            this.visit_trait_ref(&trait_ref.trait_ref);
+        });
+    }
+}
+
+enum NonLifetimeBinderAllowed {
+    Deny(&'static str),
+    Allow,
 }
-impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
+
+impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> {
     type NestedFilter = nested_filter::OnlyBodies;
 
     fn nested_visit_map(&mut self) -> Self::Map {
@@ -386,22 +465,23 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                 }
             }
 
-            let (lifetimes, binders): (FxIndexMap<LocalDefId, Region>, Vec<_>) =
+            let (mut bound_vars, binders): (FxIndexMap<LocalDefId, ResolvedArg>, Vec<_>) =
                 bound_generic_params
                     .iter()
-                    .filter(|param| matches!(param.kind, GenericParamKind::Lifetime { .. }))
                     .enumerate()
                     .map(|(late_bound_idx, param)| {
-                        let pair = Region::late(late_bound_idx as u32, param);
-                        let r = late_region_as_bound_region(self.tcx, &pair.1);
+                        let pair = ResolvedArg::late(late_bound_idx as u32, param);
+                        let r = late_arg_as_bound_arg(self.tcx, &pair.1, param);
                         (pair, r)
                     })
                     .unzip();
 
+            deny_non_region_late_bound(self.tcx, &mut bound_vars, "closures");
+
             self.record_late_bound_vars(e.hir_id, binders);
             let scope = Scope::Binder {
                 hir_id: e.hir_id,
-                lifetimes,
+                bound_vars,
                 s: self.scope,
                 scope_type: BinderScopeType::Normal,
                 where_bound_origin: None,
@@ -461,7 +541,8 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                 // conservatively add all resolved lifetimes. Otherwise we run into problems in
                 // cases like `type Foo<'a> = impl Bar<As = impl Baz + 'a>`.
                 let parent_item = self.tcx.hir().get_parent_item(item.hir_id());
-                let resolved_lifetimes: &ResolveLifetimes = self.tcx.resolve_lifetimes(parent_item);
+                let resolved_lifetimes: &ResolveBoundVars =
+                    self.tcx.resolve_bound_vars(parent_item);
                 // We need to add *all* deps, since opaque tys may want them from *us*
                 for (&owner, defs) in resolved_lifetimes.defs.iter() {
                     defs.iter().for_each(|(&local_id, region)| {
@@ -478,35 +559,33 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                 }
             }
             hir::ItemKind::OpaqueTy(hir::OpaqueTy {
-                origin: hir::OpaqueTyOrigin::FnReturn(_) | hir::OpaqueTyOrigin::AsyncFn(_),
+                origin: hir::OpaqueTyOrigin::FnReturn(parent) | hir::OpaqueTyOrigin::AsyncFn(parent),
                 generics,
                 ..
             }) => {
                 // We want to start our early-bound indices at the end of the parent scope,
                 // not including any parent `impl Trait`s.
-                let mut lifetimes = FxIndexMap::default();
+                let mut bound_vars = FxIndexMap::default();
                 debug!(?generics.params);
                 for param in generics.params {
-                    match param.kind {
-                        GenericParamKind::Lifetime { .. } => {
-                            let (def_id, reg) = Region::early(&param);
-                            lifetimes.insert(def_id, reg);
-                        }
-                        GenericParamKind::Type { .. } | GenericParamKind::Const { .. } => {}
-                    }
+                    let (def_id, reg) = ResolvedArg::early(&param);
+                    bound_vars.insert(def_id, reg);
                 }
 
-                let scope = Scope::Binder {
-                    hir_id: item.hir_id(),
-                    lifetimes,
-                    s: self.scope,
-                    scope_type: BinderScopeType::Normal,
-                    where_bound_origin: None,
-                };
+                let scope = Scope::Root { opt_parent_item: Some(parent) };
                 self.with(scope, |this| {
-                    let scope = Scope::TraitRefBoundary { s: this.scope };
-                    this.with(scope, |this| intravisit::walk_item(this, item))
-                });
+                    let scope = Scope::Binder {
+                        hir_id: item.hir_id(),
+                        bound_vars,
+                        s: this.scope,
+                        scope_type: BinderScopeType::Normal,
+                        where_bound_origin: None,
+                    };
+                    this.with(scope, |this| {
+                        let scope = Scope::TraitRefBoundary { s: this.scope };
+                        this.with(scope, |this| intravisit::walk_item(this, item))
+                    });
+                })
             }
             hir::ItemKind::TyAlias(_, generics)
             | hir::ItemKind::Enum(_, generics)
@@ -516,18 +595,11 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
             | hir::ItemKind::TraitAlias(generics, ..)
             | hir::ItemKind::Impl(&hir::Impl { generics, .. }) => {
                 // These kinds of items have only early-bound lifetime parameters.
-                let lifetimes = generics
-                    .params
-                    .iter()
-                    .filter_map(|param| match param.kind {
-                        GenericParamKind::Lifetime { .. } => Some(Region::early(param)),
-                        GenericParamKind::Type { .. } | GenericParamKind::Const { .. } => None,
-                    })
-                    .collect();
+                let bound_vars = generics.params.iter().map(ResolvedArg::early).collect();
                 self.record_late_bound_vars(item.hir_id(), vec![]);
                 let scope = Scope::Binder {
                     hir_id: item.hir_id(),
-                    lifetimes,
+                    bound_vars,
                     scope_type: BinderScopeType::Normal,
                     s: self.scope,
                     where_bound_origin: None,
@@ -562,21 +634,23 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
     fn visit_ty(&mut self, ty: &'tcx hir::Ty<'tcx>) {
         match ty.kind {
             hir::TyKind::BareFn(c) => {
-                let (lifetimes, binders): (FxIndexMap<LocalDefId, Region>, Vec<_>) = c
+                let (mut bound_vars, binders): (FxIndexMap<LocalDefId, ResolvedArg>, Vec<_>) = c
                     .generic_params
                     .iter()
-                    .filter(|param| matches!(param.kind, GenericParamKind::Lifetime { .. }))
                     .enumerate()
                     .map(|(late_bound_idx, param)| {
-                        let pair = Region::late(late_bound_idx as u32, param);
-                        let r = late_region_as_bound_region(self.tcx, &pair.1);
+                        let pair = ResolvedArg::late(late_bound_idx as u32, param);
+                        let r = late_arg_as_bound_arg(self.tcx, &pair.1, param);
                         (pair, r)
                     })
                     .unzip();
+
+                deny_non_region_late_bound(self.tcx, &mut bound_vars, "function pointer types");
+
                 self.record_late_bound_vars(ty.hir_id, binders);
                 let scope = Scope::Binder {
                     hir_id: ty.hir_id,
-                    lifetimes,
+                    bound_vars,
                     s: self.scope,
                     scope_type: BinderScopeType::Normal,
                     where_bound_origin: None,
@@ -592,7 +666,10 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                 let scope = Scope::TraitRefBoundary { s: self.scope };
                 self.with(scope, |this| {
                     for bound in bounds {
-                        this.visit_poly_trait_ref(bound);
+                        this.visit_poly_trait_ref_inner(
+                            bound,
+                            NonLifetimeBinderAllowed::Deny("trait object types"),
+                        );
                     }
                 });
                 match lifetime.res {
@@ -674,7 +751,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                     // well-supported at the moment, so this doesn't work.
                     // In the future, this should be fixed and this error should be removed.
                     let def = self.map.defs.get(&lifetime.hir_id).cloned();
-                    let Some(Region::LateBound(_, _, def_id)) = def else {
+                    let Some(ResolvedArg::LateBound(_, _, def_id)) = def else {
                         continue
                     };
                     let Some(def_id) = def_id.as_local() else {
@@ -722,18 +799,11 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
             }
             Type(bounds, ty) => {
                 let generics = &trait_item.generics;
-                let lifetimes = generics
-                    .params
-                    .iter()
-                    .filter_map(|param| match param.kind {
-                        GenericParamKind::Lifetime { .. } => Some(Region::early(param)),
-                        GenericParamKind::Type { .. } | GenericParamKind::Const { .. } => None,
-                    })
-                    .collect();
+                let bound_vars = generics.params.iter().map(ResolvedArg::early).collect();
                 self.record_late_bound_vars(trait_item.hir_id(), vec![]);
                 let scope = Scope::Binder {
                     hir_id: trait_item.hir_id(),
-                    lifetimes,
+                    bound_vars,
                     s: self.scope,
                     scope_type: BinderScopeType::Normal,
                     where_bound_origin: None,
@@ -768,18 +838,12 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
             }),
             Type(ty) => {
                 let generics = &impl_item.generics;
-                let lifetimes: FxIndexMap<LocalDefId, Region> = generics
-                    .params
-                    .iter()
-                    .filter_map(|param| match param.kind {
-                        GenericParamKind::Lifetime { .. } => Some(Region::early(param)),
-                        GenericParamKind::Const { .. } | GenericParamKind::Type { .. } => None,
-                    })
-                    .collect();
+                let bound_vars: FxIndexMap<LocalDefId, ResolvedArg> =
+                    generics.params.iter().map(ResolvedArg::early).collect();
                 self.record_late_bound_vars(impl_item.hir_id(), vec![]);
                 let scope = Scope::Binder {
                     hir_id: impl_item.hir_id(),
-                    lifetimes,
+                    bound_vars,
                     s: self.scope,
                     scope_type: BinderScopeType::Normal,
                     where_bound_origin: None,
@@ -803,7 +867,9 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
     #[instrument(level = "debug", skip(self))]
     fn visit_lifetime(&mut self, lifetime_ref: &'tcx hir::Lifetime) {
         match lifetime_ref.res {
-            hir::LifetimeName::Static => self.insert_lifetime(lifetime_ref, Region::Static),
+            hir::LifetimeName::Static => {
+                self.insert_lifetime(lifetime_ref, ResolvedArg::StaticLifetime)
+            }
             hir::LifetimeName::Param(param_def_id) => {
                 self.resolve_lifetime_ref(param_def_id, lifetime_ref)
             }
@@ -814,13 +880,16 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
         }
     }
 
-    fn visit_path(&mut self, path: &hir::Path<'tcx>, _: hir::HirId) {
+    fn visit_path(&mut self, path: &hir::Path<'tcx>, hir_id: hir::HirId) {
         for (i, segment) in path.segments.iter().enumerate() {
             let depth = path.segments.len() - i - 1;
             if let Some(args) = segment.args {
                 self.visit_segment_args(path.res, depth, args);
             }
         }
+        if let Res::Def(DefKind::TyParam | DefKind::ConstParam, param_def_id) = path.res {
+            self.resolve_type_ref(param_def_id.expect_local(), hir_id);
+        }
     }
 
     fn visit_fn(
@@ -829,7 +898,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
         fd: &'tcx hir::FnDecl<'tcx>,
         body_id: hir::BodyId,
         _: Span,
-        _: hir::HirId,
+        _: LocalDefId,
     ) {
         let output = match fd.output {
             hir::FnRetTy::DefaultReturn(_) => None,
@@ -869,24 +938,16 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                         origin,
                         ..
                     }) => {
-                        let lifetimes: FxIndexMap<LocalDefId, Region> =
+                        let (bound_vars, binders): (FxIndexMap<LocalDefId, ResolvedArg>, Vec<_>) =
                             bound_generic_params
                                 .iter()
-                                .filter(|param| {
-                                    matches!(param.kind, GenericParamKind::Lifetime { .. })
-                                })
                                 .enumerate()
                                 .map(|(late_bound_idx, param)| {
-                                        Region::late(late_bound_idx as u32, param)
-                                })
-                                .collect();
-                        let binders: Vec<_> =
-                            lifetimes
-                                .iter()
-                                .map(|(_, region)| {
-                                     late_region_as_bound_region(this.tcx, region)
+                                    let pair = ResolvedArg::late(late_bound_idx as u32, param);
+                                    let r = late_arg_as_bound_arg(this.tcx, &pair.1, param);
+                                    (pair, r)
                                 })
-                                .collect();
+                                .unzip();
                         this.record_late_bound_vars(hir_id, binders.clone());
                         // Even if there are no lifetimes defined here, we still wrap it in a binder
                         // scope. If there happens to be a nested poly trait ref (an error), that
@@ -894,7 +955,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                         // being wrong.
                         let scope = Scope::Binder {
                             hir_id,
-                            lifetimes,
+                            bound_vars,
                             s: this.scope,
                             scope_type: BinderScopeType::Normal,
                             where_bound_origin: Some(origin),
@@ -920,21 +981,23 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                                 if lt.res != hir::LifetimeName::Static {
                                     continue;
                                 }
-                                this.insert_lifetime(lt, Region::Static);
-                                this.tcx
-                                    .sess
-                                    .struct_span_warn(
-                                        lifetime.ident.span,
-                                        &format!(
-                                            "unnecessary lifetime parameter `{}`",
+                                this.insert_lifetime(lt, ResolvedArg::StaticLifetime);
+                                this.tcx.struct_span_lint_hir(
+                                    lint::builtin::UNUSED_LIFETIMES,
+                                    lifetime.hir_id,
+                                    lifetime.ident.span,
+                                    format!(
+                                        "unnecessary lifetime parameter `{}`",
+                                        lifetime.ident
+                                    ),
+                                    |lint| {
+                                        let help = &format!(
+                                            "you can use the `'static` lifetime directly, in place of `{}`",
                                             lifetime.ident,
-                                        ),
-                                    )
-                                    .help(&format!(
-                                        "you can use the `'static` lifetime directly, in place of `{}`",
-                                        lifetime.ident,
-                                    ))
-                                    .emit();
+                                        );
+                                        lint.help(help)
+                                    },
+                                );
                             }
                         }
                     }
@@ -964,7 +1027,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
                 self.record_late_bound_vars(*hir_id, binders);
                 let scope = Scope::Binder {
                     hir_id: *hir_id,
-                    lifetimes: FxIndexMap::default(),
+                    bound_vars: FxIndexMap::default(),
                     s: self.scope,
                     scope_type,
                     where_bound_origin: None,
@@ -978,42 +1041,12 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
     }
 
     fn visit_poly_trait_ref(&mut self, trait_ref: &'tcx hir::PolyTraitRef<'tcx>) {
-        debug!("visit_poly_trait_ref(trait_ref={:?})", trait_ref);
-
-        let (mut binders, scope_type) = self.poly_trait_ref_binder_info();
-
-        let initial_bound_vars = binders.len() as u32;
-        let mut lifetimes: FxIndexMap<LocalDefId, Region> = FxIndexMap::default();
-        let binders_iter = trait_ref
-            .bound_generic_params
-            .iter()
-            .filter(|param| matches!(param.kind, GenericParamKind::Lifetime { .. }))
-            .enumerate()
-            .map(|(late_bound_idx, param)| {
-                let pair = Region::late(initial_bound_vars + late_bound_idx as u32, param);
-                let r = late_region_as_bound_region(self.tcx, &pair.1);
-                lifetimes.insert(pair.0, pair.1);
-                r
-            });
-        binders.extend(binders_iter);
-
-        debug!(?binders);
-        self.record_late_bound_vars(trait_ref.trait_ref.hir_ref_id, binders);
+        self.visit_poly_trait_ref_inner(trait_ref, NonLifetimeBinderAllowed::Allow);
+    }
 
-        // Always introduce a scope here, even if this is in a where clause and
-        // we introduced the binders around the bounded Ty. In that case, we
-        // just reuse the concatenation functionality also present in nested trait
-        // refs.
-        let scope = Scope::Binder {
-            hir_id: trait_ref.trait_ref.hir_ref_id,
-            lifetimes,
-            s: self.scope,
-            scope_type,
-            where_bound_origin: None,
-        };
-        self.with(scope, |this| {
-            walk_list!(this, visit_generic_param, trait_ref.bound_generic_params);
-            this.visit_trait_ref(&trait_ref.trait_ref);
+    fn visit_anon_const(&mut self, c: &'tcx hir::AnonConst) {
+        self.with(Scope::AnonConstBoundary { s: self.scope }, |this| {
+            intravisit::walk_anon_const(this, c);
         });
     }
 }
@@ -1021,55 +1054,63 @@ impl<'a, 'tcx> Visitor<'tcx> for LifetimeContext<'a, 'tcx> {
 fn object_lifetime_default(tcx: TyCtxt<'_>, param_def_id: DefId) -> ObjectLifetimeDefault {
     debug_assert_eq!(tcx.def_kind(param_def_id), DefKind::TyParam);
     let param_def_id = param_def_id.expect_local();
-    let parent_def_id = tcx.local_parent(param_def_id);
-    let generics = tcx.hir().get_generics(parent_def_id).unwrap();
-    let param_hir_id = tcx.local_def_id_to_hir_id(param_def_id);
-    let param = generics.params.iter().find(|p| p.hir_id == param_hir_id).unwrap();
-
-    // Scan the bounds and where-clauses on parameters to extract bounds
-    // of the form `T:'a` so as to determine the `ObjectLifetimeDefault`
-    // for each type parameter.
-    match param.kind {
-        GenericParamKind::Type { .. } => {
-            let mut set = Set1::Empty;
-
-            // Look for `type: ...` where clauses.
-            for bound in generics.bounds_for_param(param_def_id) {
-                // Ignore `for<'a> type: ...` as they can change what
-                // lifetimes mean (although we could "just" handle it).
-                if !bound.bound_generic_params.is_empty() {
-                    continue;
-                }
+    let hir::Node::GenericParam(param) = tcx.hir().get_by_def_id(param_def_id) else {
+        bug!("expected GenericParam for object_lifetime_default");
+    };
+    match param.source {
+        hir::GenericParamSource::Generics => {
+            let parent_def_id = tcx.local_parent(param_def_id);
+            let generics = tcx.hir().get_generics(parent_def_id).unwrap();
+            let param_hir_id = tcx.local_def_id_to_hir_id(param_def_id);
+            let param = generics.params.iter().find(|p| p.hir_id == param_hir_id).unwrap();
+
+            // Scan the bounds and where-clauses on parameters to extract bounds
+            // of the form `T:'a` so as to determine the `ObjectLifetimeDefault`
+            // for each type parameter.
+            match param.kind {
+                GenericParamKind::Type { .. } => {
+                    let mut set = Set1::Empty;
+
+                    // Look for `type: ...` where clauses.
+                    for bound in generics.bounds_for_param(param_def_id) {
+                        // Ignore `for<'a> type: ...` as they can change what
+                        // lifetimes mean (although we could "just" handle it).
+                        if !bound.bound_generic_params.is_empty() {
+                            continue;
+                        }
 
-                for bound in bound.bounds {
-                    if let hir::GenericBound::Outlives(lifetime) = bound {
-                        set.insert(lifetime.res);
+                        for bound in bound.bounds {
+                            if let hir::GenericBound::Outlives(lifetime) = bound {
+                                set.insert(lifetime.res);
+                            }
+                        }
                     }
-                }
-            }
 
-            match set {
-                Set1::Empty => ObjectLifetimeDefault::Empty,
-                Set1::One(hir::LifetimeName::Static) => ObjectLifetimeDefault::Static,
-                Set1::One(hir::LifetimeName::Param(param_def_id)) => {
-                    ObjectLifetimeDefault::Param(param_def_id.to_def_id())
+                    match set {
+                        Set1::Empty => ObjectLifetimeDefault::Empty,
+                        Set1::One(hir::LifetimeName::Static) => ObjectLifetimeDefault::Static,
+                        Set1::One(hir::LifetimeName::Param(param_def_id)) => {
+                            ObjectLifetimeDefault::Param(param_def_id.to_def_id())
+                        }
+                        _ => ObjectLifetimeDefault::Ambiguous,
+                    }
+                }
+                _ => {
+                    bug!("object_lifetime_default_raw must only be called on a type parameter")
                 }
-                _ => ObjectLifetimeDefault::Ambiguous,
             }
         }
-        _ => {
-            bug!("object_lifetime_default_raw must only be called on a type parameter")
-        }
+        hir::GenericParamSource::Binder => ObjectLifetimeDefault::Empty,
     }
 }
 
-impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
+impl<'a, 'tcx> BoundVarContext<'a, 'tcx> {
     fn with<F>(&mut self, wrap_scope: Scope<'_>, f: F)
     where
-        F: for<'b> FnOnce(&mut LifetimeContext<'b, 'tcx>),
+        F: for<'b> FnOnce(&mut BoundVarContext<'b, 'tcx>),
     {
-        let LifetimeContext { tcx, map, .. } = self;
-        let mut this = LifetimeContext { tcx: *tcx, map, scope: &wrap_scope };
+        let BoundVarContext { tcx, map, .. } = self;
+        let mut this = BoundVarContext { tcx: *tcx, map, scope: &wrap_scope };
         let span = debug_span!("scope", scope = ?TruncatedScopeDebug(&this.scope));
         {
             let _enter = span.enter();
@@ -1110,23 +1151,25 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
         generics: &'tcx hir::Generics<'tcx>,
         walk: F,
     ) where
-        F: for<'b, 'c> FnOnce(&'b mut LifetimeContext<'c, 'tcx>),
+        F: for<'b, 'c> FnOnce(&'b mut BoundVarContext<'c, 'tcx>),
     {
         let mut named_late_bound_vars = 0;
-        let lifetimes: FxIndexMap<LocalDefId, Region> = generics
+        let bound_vars: FxIndexMap<LocalDefId, ResolvedArg> = generics
             .params
             .iter()
-            .filter_map(|param| match param.kind {
+            .map(|param| match param.kind {
                 GenericParamKind::Lifetime { .. } => {
                     if self.tcx.is_late_bound(param.hir_id) {
                         let late_bound_idx = named_late_bound_vars;
                         named_late_bound_vars += 1;
-                        Some(Region::late(late_bound_idx, param))
+                        ResolvedArg::late(late_bound_idx, param)
                     } else {
-                        Some(Region::early(param))
+                        ResolvedArg::early(param)
                     }
                 }
-                GenericParamKind::Type { .. } | GenericParamKind::Const { .. } => None,
+                GenericParamKind::Type { .. } | GenericParamKind::Const { .. } => {
+                    ResolvedArg::early(param)
+                }
             })
             .collect();
 
@@ -1139,14 +1182,14 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
             })
             .enumerate()
             .map(|(late_bound_idx, param)| {
-                let pair = Region::late(late_bound_idx as u32, param);
-                late_region_as_bound_region(self.tcx, &pair.1)
+                let pair = ResolvedArg::late(late_bound_idx as u32, param);
+                late_arg_as_bound_arg(self.tcx, &pair.1, param)
             })
             .collect();
         self.record_late_bound_vars(hir_id, binders);
         let scope = Scope::Binder {
             hir_id,
-            lifetimes,
+            bound_vars,
             s: self.scope,
             scope_type: BinderScopeType::Normal,
             where_bound_origin: None,
@@ -1177,15 +1220,15 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                 Scope::Root { opt_parent_item } => {
                     if let Some(parent_item) = opt_parent_item
                         && let parent_generics = self.tcx.generics_of(parent_item)
-                        && parent_generics.param_def_id_to_index.contains_key(&region_def_id.to_def_id())
+                        && parent_generics.param_def_id_to_index(self.tcx, region_def_id.to_def_id()).is_some()
                     {
-                        break Some(Region::EarlyBound(region_def_id.to_def_id()));
+                        break Some(ResolvedArg::EarlyBound(region_def_id.to_def_id()));
                     }
                     break None;
                 }
 
-                Scope::Binder { ref lifetimes, scope_type, s, where_bound_origin, .. } => {
-                    if let Some(&def) = lifetimes.get(&region_def_id) {
+                Scope::Binder { ref bound_vars, scope_type, s, where_bound_origin, .. } => {
+                    if let Some(&def) = bound_vars.get(&region_def_id) {
                         break Some(def.shifted(late_depth));
                     }
                     match scope_type {
@@ -1252,26 +1295,34 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                 Scope::Elision { s, .. }
                 | Scope::ObjectLifetimeDefault { s, .. }
                 | Scope::Supertrait { s, .. }
-                | Scope::TraitRefBoundary { s, .. } => {
+                | Scope::TraitRefBoundary { s, .. }
+                | Scope::AnonConstBoundary { s } => {
                     scope = s;
                 }
             }
         };
 
         if let Some(mut def) = result {
-            if let Region::EarlyBound(..) = def {
+            if let ResolvedArg::EarlyBound(..) = def {
                 // Do not free early-bound regions, only late-bound ones.
             } else if let Some(body_id) = outermost_body {
                 let fn_id = self.tcx.hir().body_owner(body_id);
                 match self.tcx.hir().get(fn_id) {
-                    Node::Item(hir::Item { kind: hir::ItemKind::Fn(..), .. })
+                    Node::Item(hir::Item { owner_id, kind: hir::ItemKind::Fn(..), .. })
                     | Node::TraitItem(hir::TraitItem {
-                        kind: hir::TraitItemKind::Fn(..), ..
+                        owner_id,
+                        kind: hir::TraitItemKind::Fn(..),
+                        ..
                     })
-                    | Node::ImplItem(hir::ImplItem { kind: hir::ImplItemKind::Fn(..), .. })
-                    | Node::Expr(hir::Expr { kind: hir::ExprKind::Closure(..), .. }) => {
-                        let scope = self.tcx.hir().local_def_id(fn_id);
-                        def = Region::Free(scope.to_def_id(), def.id().unwrap());
+                    | Node::ImplItem(hir::ImplItem {
+                        owner_id,
+                        kind: hir::ImplItemKind::Fn(..),
+                        ..
+                    }) => {
+                        def = ResolvedArg::Free(owner_id.to_def_id(), def.id().unwrap());
+                    }
+                    Node::Expr(hir::Expr { kind: hir::ExprKind::Closure(closure), .. }) => {
+                        def = ResolvedArg::Free(closure.def_id.to_def_id(), def.id().unwrap());
                     }
                     _ => {}
                 }
@@ -1310,7 +1361,8 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                 | Scope::Elision { s, .. }
                 | Scope::ObjectLifetimeDefault { s, .. }
                 | Scope::Supertrait { s, .. }
-                | Scope::TraitRefBoundary { s, .. } => {
+                | Scope::TraitRefBoundary { s, .. }
+                | Scope::AnonConstBoundary { s } => {
                     scope = s;
                 }
             }
@@ -1322,6 +1374,87 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
         );
     }
 
+    fn resolve_type_ref(&mut self, param_def_id: LocalDefId, hir_id: hir::HirId) {
+        // Walk up the scope chain, tracking the number of fn scopes
+        // that we pass through, until we find a lifetime with the
+        // given name or we run out of scopes.
+        // search.
+        let mut late_depth = 0;
+        let mut scope = self.scope;
+        let mut crossed_anon_const = false;
+        let result = loop {
+            match *scope {
+                Scope::Body { s, .. } => {
+                    scope = s;
+                }
+
+                Scope::Root { opt_parent_item } => {
+                    if let Some(parent_item) = opt_parent_item
+                        && let parent_generics = self.tcx.generics_of(parent_item)
+                        && parent_generics.param_def_id_to_index(self.tcx, param_def_id.to_def_id()).is_some()
+                    {
+                        break Some(ResolvedArg::EarlyBound(param_def_id.to_def_id()));
+                    }
+                    break None;
+                }
+
+                Scope::Binder { ref bound_vars, scope_type, s, .. } => {
+                    if let Some(&def) = bound_vars.get(&param_def_id) {
+                        break Some(def.shifted(late_depth));
+                    }
+                    match scope_type {
+                        BinderScopeType::Normal => late_depth += 1,
+                        BinderScopeType::Concatenating => {}
+                    }
+                    scope = s;
+                }
+
+                Scope::Elision { s, .. }
+                | Scope::ObjectLifetimeDefault { s, .. }
+                | Scope::Supertrait { s, .. }
+                | Scope::TraitRefBoundary { s, .. } => {
+                    scope = s;
+                }
+
+                Scope::AnonConstBoundary { s } => {
+                    crossed_anon_const = true;
+                    scope = s;
+                }
+            }
+        };
+
+        if let Some(def) = result {
+            if let ResolvedArg::LateBound(..) = def && crossed_anon_const {
+                let use_span = self.tcx.hir().span(hir_id);
+                let def_span = self.tcx.def_span(param_def_id);
+                match self.tcx.def_kind(param_def_id) {
+                    DefKind::ConstParam => {
+                        self.tcx.sess.emit_err(errors::CannotCaptureLateBoundInAnonConst::Const {
+                            use_span,
+                            def_span,
+                        });
+                    }
+                    DefKind::TyParam => {
+                        self.tcx.sess.emit_err(errors::CannotCaptureLateBoundInAnonConst::Type {
+                            use_span,
+                            def_span,
+                        });
+                    }
+                    _ => unreachable!(),
+                }
+                return;
+            }
+
+            self.map.defs.insert(hir_id, def);
+            return;
+        }
+
+        self.tcx.sess.delay_span_bug(
+            self.tcx.hir().span(hir_id),
+            format!("could not resolve {param_def_id:?}"),
+        );
+    }
+
     #[instrument(level = "debug", skip(self))]
     fn visit_segment_args(
         &mut self,
@@ -1390,7 +1523,8 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                         | Scope::Elision { s, .. }
                         | Scope::ObjectLifetimeDefault { s, .. }
                         | Scope::Supertrait { s, .. }
-                        | Scope::TraitRefBoundary { s, .. } => {
+                        | Scope::TraitRefBoundary { s, .. }
+                        | Scope::AnonConstBoundary { s } => {
                             scope = s;
                         }
                     }
@@ -1408,10 +1542,10 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                     if in_body {
                         None
                     } else {
-                        Some(Region::Static)
+                        Some(ResolvedArg::StaticLifetime)
                     }
                 }
-                ObjectLifetimeDefault::Static => Some(Region::Static),
+                ObjectLifetimeDefault::Static => Some(ResolvedArg::StaticLifetime),
                 ObjectLifetimeDefault::Param(param_def_id) => {
                     // This index can be used with `generic_args` since `parent_count == 0`.
                     let index = generics.param_def_id_to_index[&param_def_id] as usize;
@@ -1500,18 +1634,19 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
         // in the trait ref `YY<...>` in `Item: YY<...>`.
         for binding in generic_args.bindings {
             let scope = Scope::ObjectLifetimeDefault {
-                lifetime: if has_lifetime_parameter { None } else { Some(Region::Static) },
+                lifetime: if has_lifetime_parameter {
+                    None
+                } else {
+                    Some(ResolvedArg::StaticLifetime)
+                },
                 s: self.scope,
             };
             if let Some(type_def_id) = type_def_id {
-                let lifetimes = LifetimeContext::supertrait_hrtb_lifetimes(
-                    self.tcx,
-                    type_def_id,
-                    binding.ident,
-                );
+                let bound_vars =
+                    BoundVarContext::supertrait_hrtb_vars(self.tcx, type_def_id, binding.ident);
                 self.with(scope, |this| {
                     let scope = Scope::Supertrait {
-                        lifetimes: lifetimes.unwrap_or_default(),
+                        bound_vars: bound_vars.unwrap_or_default(),
                         s: this.scope,
                     };
                     this.with(scope, |this| this.visit_assoc_type_binding(binding));
@@ -1534,7 +1669,7 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
     /// ```
     /// In this case, if we wanted to the supertrait HRTB lifetimes for `As` on
     /// the starting trait `Bar`, we would return `Some(['b, 'a])`.
-    fn supertrait_hrtb_lifetimes(
+    fn supertrait_hrtb_vars(
         tcx: TyCtxt<'tcx>,
         def_id: DefId,
         assoc_name: Ident,
@@ -1556,7 +1691,7 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
             // See issue #83753. If someone writes an associated type on a non-trait, just treat it as
             // there being no supertrait HRTBs.
             match tcx.def_kind(def_id) {
-                DefKind::Trait | DefKind::TraitAlias | DefKind::Impl => {}
+                DefKind::Trait | DefKind::TraitAlias | DefKind::Impl { .. } => {}
                 _ => break None,
             }
 
@@ -1619,13 +1754,15 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
                     scope = s;
                 }
 
-                Scope::Root { .. } | Scope::Elision { .. } => break Region::Static,
+                Scope::Root { .. } | Scope::Elision { .. } => break ResolvedArg::StaticLifetime,
 
                 Scope::Body { .. } | Scope::ObjectLifetimeDefault { lifetime: None, .. } => return,
 
                 Scope::ObjectLifetimeDefault { lifetime: Some(l), .. } => break l,
 
-                Scope::Supertrait { s, .. } | Scope::TraitRefBoundary { s, .. } => {
+                Scope::Supertrait { s, .. }
+                | Scope::TraitRefBoundary { s, .. }
+                | Scope::AnonConstBoundary { s } => {
                     scope = s;
                 }
             }
@@ -1634,7 +1771,7 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
     }
 
     #[instrument(level = "debug", skip(self))]
-    fn insert_lifetime(&mut self, lifetime_ref: &'tcx hir::Lifetime, def: Region) {
+    fn insert_lifetime(&mut self, lifetime_ref: &'tcx hir::Lifetime, def: ResolvedArg) {
         debug!(span = ?lifetime_ref.ident.span);
         self.map.defs.insert(lifetime_ref.hir_id, def);
     }
@@ -1642,7 +1779,11 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
     /// Sometimes we resolve a lifetime, but later find that it is an
     /// error (esp. around impl trait). In that case, we remove the
     /// entry into `map.defs` so as not to confuse later code.
-    fn uninsert_lifetime_on_error(&mut self, lifetime_ref: &'tcx hir::Lifetime, bad_def: Region) {
+    fn uninsert_lifetime_on_error(
+        &mut self,
+        lifetime_ref: &'tcx hir::Lifetime,
+        bad_def: ResolvedArg,
+    ) {
         let old_value = self.map.defs.remove(&lifetime_ref.hir_id);
         assert_eq!(old_value, Some(bad_def));
     }
@@ -1658,10 +1799,12 @@ impl<'a, 'tcx> LifetimeContext<'a, 'tcx> {
 /// "Constrained" basically means that it appears in any type but
 /// not amongst the inputs to a projection. In other words, `<&'a
 /// T as Trait<''b>>::Foo` does not constrain `'a` or `'b`.
-fn is_late_bound_map(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<&FxIndexSet<LocalDefId>> {
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
-    let decl = tcx.hir().fn_decl_by_hir_id(hir_id)?;
-    let generics = tcx.hir().get_generics(def_id)?;
+fn is_late_bound_map(
+    tcx: TyCtxt<'_>,
+    owner_id: hir::OwnerId,
+) -> Option<&FxIndexSet<hir::ItemLocalId>> {
+    let decl = tcx.hir().fn_decl_by_hir_id(owner_id.into())?;
+    let generics = tcx.hir().get_generics(owner_id.def_id)?;
 
     let mut late_bound = FxIndexSet::default();
 
@@ -1695,24 +1838,22 @@ fn is_late_bound_map(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<&FxIndexSet<
             hir::GenericParamKind::Type { .. } | hir::GenericParamKind::Const { .. } => continue,
         }
 
-        let param_def_id = tcx.hir().local_def_id(param.hir_id);
-
         // appears in the where clauses? early-bound.
-        if appears_in_where_clause.regions.contains(&param_def_id) {
+        if appears_in_where_clause.regions.contains(&param.def_id) {
             continue;
         }
 
         // does not appear in the inputs, but appears in the return type? early-bound.
-        if !constrained_by_input.regions.contains(&param_def_id)
-            && appears_in_output.regions.contains(&param_def_id)
+        if !constrained_by_input.regions.contains(&param.def_id)
+            && appears_in_output.regions.contains(&param.def_id)
         {
             continue;
         }
 
-        debug!("lifetime {:?} with id {:?} is late-bound", param.name.ident(), param.hir_id);
+        debug!("lifetime {:?} with id {:?} is late-bound", param.name.ident(), param.def_id);
 
-        let inserted = late_bound.insert(param_def_id);
-        assert!(inserted, "visited lifetime {:?} twice", param.hir_id);
+        let inserted = late_bound.insert(param.hir_id.local_id);
+        assert!(inserted, "visited lifetime {:?} twice", param.def_id);
     }
 
     debug!(?late_bound);
@@ -1745,7 +1886,7 @@ fn is_late_bound_map(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<&FxIndexSet<
 
     use std::ops::ControlFlow;
     use ty::Ty;
-    impl<'tcx> TypeVisitor<'tcx> for ConstrainedCollectorPostAstConv {
+    impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for ConstrainedCollectorPostAstConv {
         fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<!> {
             match t.kind() {
                 ty::Param(param_ty) => {
@@ -1797,7 +1938,7 @@ fn is_late_bound_map(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<&FxIndexSet<
                     let mut walker = ConstrainedCollectorPostAstConv {
                         arg_is_constrained: vec![false; generics.params.len()].into_boxed_slice(),
                     };
-                    walker.visit_ty(self.tcx.type_of(alias_def));
+                    walker.visit_ty(self.tcx.type_of(alias_def).subst_identity());
 
                     match segments.last() {
                         Some(hir::PathSegment { args: Some(args), .. }) => {
@@ -1865,3 +2006,37 @@ fn is_late_bound_map(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<&FxIndexSet<
         }
     }
 }
+
+pub fn deny_non_region_late_bound(
+    tcx: TyCtxt<'_>,
+    bound_vars: &mut FxIndexMap<LocalDefId, ResolvedArg>,
+    where_: &str,
+) {
+    let mut first = true;
+
+    for (var, arg) in bound_vars {
+        let Node::GenericParam(param) = tcx.hir().get_by_def_id(*var) else {
+            bug!();
+        };
+
+        let what = match param.kind {
+            hir::GenericParamKind::Type { .. } => "type",
+            hir::GenericParamKind::Const { .. } => "const",
+            hir::GenericParamKind::Lifetime { .. } => continue,
+        };
+
+        let mut diag = tcx.sess.struct_span_err(
+            param.span,
+            format!("late-bound {what} parameter not allowed on {where_}"),
+        );
+
+        let guar = if tcx.features().non_lifetime_binders && first {
+            diag.emit()
+        } else {
+            diag.delay_as_bug()
+        };
+
+        first = false;
+        *arg = ResolvedArg::Error(guar);
+    }
+}
diff --git a/compiler/rustc_hir_analysis/src/collect/type_of.rs b/compiler/rustc_hir_analysis/src/collect/type_of.rs
index 5e388a2f2..50073d94e 100644
--- a/compiler/rustc_hir_analysis/src/collect/type_of.rs
+++ b/compiler/rustc_hir_analysis/src/collect/type_of.rs
@@ -8,7 +8,9 @@ use rustc_middle::hir::nested_filter;
 use rustc_middle::ty::print::with_forced_trimmed_paths;
 use rustc_middle::ty::subst::InternalSubsts;
 use rustc_middle::ty::util::IntTypeExt;
-use rustc_middle::ty::{self, DefIdTree, Ty, TyCtxt, TypeFolder, TypeSuperFoldable, TypeVisitable};
+use rustc_middle::ty::{
+    self, DefIdTree, IsSuggestable, Ty, TyCtxt, TypeFolder, TypeSuperFoldable, TypeVisitableExt,
+};
 use rustc_span::symbol::Ident;
 use rustc_span::{Span, DUMMY_SP};
 
@@ -54,15 +56,14 @@ pub(super) fn opt_const_param_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<
             // ty which is a fully resolved projection.
             // For the code example above, this would mean converting Self::Assoc<3>
             // into a ty::Alias(ty::Projection, <Self as Foo>::Assoc<3>)
-            let item_hir_id = tcx
+            let item_def_id = tcx
                 .hir()
-                .parent_iter(hir_id)
-                .filter(|(_, node)| matches!(node, Node::Item(_)))
-                .map(|(id, _)| id)
-                .next()
-                .unwrap();
-            let item_did = tcx.hir().local_def_id(item_hir_id).to_def_id();
-            let item_ctxt = &ItemCtxt::new(tcx, item_did) as &dyn crate::astconv::AstConv<'_>;
+                .parent_owner_iter(hir_id)
+                .find(|(_, node)| matches!(node, OwnerNode::Item(_)))
+                .unwrap()
+                .0
+                .to_def_id();
+            let item_ctxt = &ItemCtxt::new(tcx, item_def_id) as &dyn crate::astconv::AstConv<'_>;
             let ty = item_ctxt.ast_ty_to_ty(hir_ty);
 
             // Iterate through the generics of the projection to find the one that corresponds to
@@ -242,7 +243,7 @@ fn get_path_containing_arg_in_pat<'hir>(
     arg_path
 }
 
-pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> Ty<'_> {
+pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::EarlyBinder<Ty<'_>> {
     let def_id = def_id.expect_local();
     use rustc_hir::*;
 
@@ -250,7 +251,7 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> Ty<'_> {
 
     let icx = ItemCtxt::new(tcx, def_id.to_def_id());
 
-    match tcx.hir().get(hir_id) {
+    let output = match tcx.hir().get(hir_id) {
         Node::TraitItem(item) => match item.kind {
             TraitItemKind::Fn(..) => {
                 let substs = InternalSubsts::identity_for_item(tcx, def_id.to_def_id());
@@ -258,13 +259,8 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> Ty<'_> {
             }
             TraitItemKind::Const(ty, body_id) => body_id
                 .and_then(|body_id| {
-                    if is_suggestable_infer_ty(ty) {
-                        Some(infer_placeholder_type(
-                            tcx, def_id, body_id, ty.span, item.ident, "constant",
-                        ))
-                    } else {
-                        None
-                    }
+                    is_suggestable_infer_ty(ty)
+                        .then(|| infer_placeholder_type(tcx, def_id, body_id, ty.span, item.ident, "constant",))
                 })
                 .unwrap_or_else(|| icx.to_ty(ty)),
             TraitItemKind::Type(_, Some(ty)) => icx.to_ty(ty),
@@ -323,8 +319,8 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> Ty<'_> {
                 ItemKind::Impl(hir::Impl { self_ty, .. }) => {
                     match self_ty.find_self_aliases() {
                         spans if spans.len() > 0 => {
-                            tcx.sess.emit_err(crate::errors::SelfInImplSelf { span: spans.into(), note: (), });
-                            tcx.ty_error()
+                            let guar = tcx.sess.emit_err(crate::errors::SelfInImplSelf { span: spans.into(), note: () });
+                            tcx.ty_error(guar)
                         },
                         _ => icx.to_ty(*self_ty),
                     }
@@ -381,7 +377,7 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> Ty<'_> {
 
         Node::Ctor(def) | Node::Variant(Variant { data: def, .. }) => match def {
             VariantData::Unit(..) | VariantData::Struct(..) => {
-                tcx.type_of(tcx.hir().get_parent_item(hir_id))
+                tcx.type_of(tcx.hir().get_parent_item(hir_id)).subst_identity()
             }
             VariantData::Tuple(..) => {
                 let substs = InternalSubsts::identity_for_item(tcx, def_id.to_def_id());
@@ -398,7 +394,7 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> Ty<'_> {
         Node::AnonConst(_) if let Some(param) = tcx.opt_const_param_of(def_id) => {
             // We defer to `type_of` of the corresponding parameter
             // for generic arguments.
-            tcx.type_of(param)
+            tcx.type_of(param).subst_identity()
         }
 
         Node::AnonConst(_) => {
@@ -450,7 +446,7 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> Ty<'_> {
                     && e.hir_id == hir_id =>
                 {
                     let Some(trait_def_id) = trait_ref.trait_def_id() else {
-                        return tcx.ty_error_with_message(DUMMY_SP, "Could not find trait");
+                        return ty::EarlyBinder(tcx.ty_error_with_message(DUMMY_SP, "Could not find trait"));
                     };
                     let assoc_items = tcx.associated_items(trait_def_id);
                     let assoc_item = assoc_items.find_by_name_and_kind(
@@ -460,7 +456,7 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> Ty<'_> {
                         def_id.to_def_id(),
                     );
                     if let Some(assoc_item) = assoc_item {
-                        tcx.type_of(assoc_item.def_id)
+                        tcx.type_of(assoc_item.def_id).subst_identity()
                     } else {
                         // FIXME(associated_const_equality): add a useful error message here.
                         tcx.ty_error_with_message(
@@ -484,7 +480,7 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> Ty<'_> {
                         }) =>
                 {
                     let Some(trait_def_id) = trait_ref.trait_def_id() else {
-                        return tcx.ty_error_with_message(DUMMY_SP, "Could not find trait");
+                        return ty::EarlyBinder(tcx.ty_error_with_message(DUMMY_SP, "Could not find trait"));
                     };
                     let assoc_items = tcx.associated_items(trait_def_id);
                     let assoc_item = assoc_items.find_by_name_and_kind(
@@ -505,7 +501,7 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> Ty<'_> {
                     if let Some(param)
                         = assoc_item.map(|item| &tcx.generics_of(item.def_id).params[idx]).filter(|param| param.kind.is_ty_or_const())
                     {
-                        tcx.type_of(param.def_id)
+                        tcx.type_of(param.def_id).subst_identity()
                     } else {
                         // FIXME(associated_const_equality): add a useful error message here.
                         tcx.ty_error_with_message(
@@ -519,7 +515,7 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> Ty<'_> {
                     def_id: param_def_id,
                     kind: GenericParamKind::Const { default: Some(ct), .. },
                     ..
-                }) if ct.hir_id == hir_id => tcx.type_of(param_def_id),
+                }) if ct.hir_id == hir_id => tcx.type_of(param_def_id).subst_identity(),
 
                 x => tcx.ty_error_with_message(
                     DUMMY_SP,
@@ -537,7 +533,8 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: DefId) -> Ty<'_> {
         x => {
             bug!("unexpected sort of node in type_of(): {:?}", x);
         }
-    }
+    };
+    ty::EarlyBinder(output)
 }
 
 #[instrument(skip(tcx), level = "debug")]
@@ -602,8 +599,9 @@ fn find_opaque_ty_constraints_for_tait(tcx: TyCtxt<'_>, def_id: LocalDefId) -> T
             // // constant does not contain interior mutability.
             // ```
             let tables = self.tcx.typeck(item_def_id);
-            if let Some(_) = tables.tainted_by_errors {
-                self.found = Some(ty::OpaqueHiddenType { span: DUMMY_SP, ty: self.tcx.ty_error() });
+            if let Some(guar) = tables.tainted_by_errors {
+                self.found =
+                    Some(ty::OpaqueHiddenType { span: DUMMY_SP, ty: self.tcx.ty_error(guar) });
                 return;
             }
             let Some(&typeck_hidden_ty) = tables.concrete_opaque_types.get(&self.def_id) else {
@@ -621,8 +619,8 @@ fn find_opaque_ty_constraints_for_tait(tcx: TyCtxt<'_>, def_id: LocalDefId) -> T
                 debug!(?concrete_type, "found constraint");
                 if let Some(prev) = &mut self.found {
                     if concrete_type.ty != prev.ty && !(concrete_type, prev.ty).references_error() {
-                        prev.report_mismatch(&concrete_type, self.tcx);
-                        prev.ty = self.tcx.ty_error();
+                        let guar = prev.report_mismatch(&concrete_type, self.tcx);
+                        prev.ty = self.tcx.ty_error(guar);
                     }
                 } else {
                     self.found = Some(concrete_type);
@@ -709,7 +707,7 @@ fn find_opaque_ty_constraints_for_tait(tcx: TyCtxt<'_>, def_id: LocalDefId) -> T
                 _ => "item",
             },
         });
-        return tcx.ty_error_with_guaranteed(reported);
+        return tcx.ty_error(reported);
     };
 
     // Only check against typeck if we didn't already error
@@ -817,11 +815,11 @@ fn find_opaque_ty_constraints_for_rpit(
 
     concrete.map(|concrete| concrete.ty).unwrap_or_else(|| {
         let table = tcx.typeck(owner_def_id);
-        if let Some(_) = table.tainted_by_errors {
+        if let Some(guar) = table.tainted_by_errors {
             // Some error in the
             // owner fn prevented us from populating
             // the `concrete_opaque_types` table.
-            tcx.ty_error()
+            tcx.ty_error(guar)
         } else {
             table.concrete_opaque_types.get(&def_id).map(|ty| ty.ty).unwrap_or_else(|| {
                 // We failed to resolve the opaque type or it
@@ -846,35 +844,23 @@ fn infer_placeholder_type<'a>(
 ) -> Ty<'a> {
     // Attempts to make the type nameable by turning FnDefs into FnPtrs.
     struct MakeNameable<'tcx> {
-        success: bool,
         tcx: TyCtxt<'tcx>,
     }
 
-    impl<'tcx> MakeNameable<'tcx> {
-        fn new(tcx: TyCtxt<'tcx>) -> Self {
-            MakeNameable { success: true, tcx }
-        }
-    }
-
-    impl<'tcx> TypeFolder<'tcx> for MakeNameable<'tcx> {
-        fn tcx(&self) -> TyCtxt<'tcx> {
+    impl<'tcx> TypeFolder<TyCtxt<'tcx>> for MakeNameable<'tcx> {
+        fn interner(&self) -> TyCtxt<'tcx> {
             self.tcx
         }
 
         fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
-            if !self.success {
-                return ty;
-            }
-
-            match ty.kind() {
-                ty::FnDef(def_id, _) => self.tcx.mk_fn_ptr(self.tcx.fn_sig(*def_id)),
-                // FIXME: non-capturing closures should also suggest a function pointer
-                ty::Closure(..) | ty::Generator(..) => {
-                    self.success = false;
-                    ty
+            let ty = match *ty.kind() {
+                ty::FnDef(def_id, substs) => {
+                    self.tcx.mk_fn_ptr(self.tcx.fn_sig(def_id).subst(self.tcx, substs))
                 }
-                _ => ty.super_fold_with(self),
-            }
+                _ => ty,
+            };
+
+            ty.super_fold_with(self)
         }
     }
 
@@ -897,15 +883,11 @@ fn infer_placeholder_type<'a>(
                     suggestions.clear();
                 }
 
-                // Suggesting unnameable types won't help.
-                let mut mk_nameable = MakeNameable::new(tcx);
-                let ty = mk_nameable.fold_ty(ty);
-                let sugg_ty = if mk_nameable.success { Some(ty) } else { None };
-                if let Some(sugg_ty) = sugg_ty {
+                if let Some(ty) = ty.make_suggestable(tcx, false) {
                     err.span_suggestion(
                         span,
                         &format!("provide a type for the {item}", item = kind),
-                        format!("{colon} {sugg_ty}"),
+                        format!("{colon} {ty}"),
                         Applicability::MachineApplicable,
                     );
                 } else {
@@ -922,15 +904,12 @@ fn infer_placeholder_type<'a>(
             let mut diag = bad_placeholder(tcx, vec![span], kind);
 
             if !ty.references_error() {
-                let mut mk_nameable = MakeNameable::new(tcx);
-                let ty = mk_nameable.fold_ty(ty);
-                let sugg_ty = if mk_nameable.success { Some(ty) } else { None };
-                if let Some(sugg_ty) = sugg_ty {
+                if let Some(ty) = ty.make_suggestable(tcx, false) {
                     diag.span_suggestion(
                         span,
                         "replace with the correct type",
-                        sugg_ty,
-                        Applicability::MaybeIncorrect,
+                        ty,
+                        Applicability::MachineApplicable,
                     );
                 } else {
                     with_forced_trimmed_paths!(diag.span_note(
diff --git a/compiler/rustc_hir_analysis/src/constrained_generic_params.rs b/compiler/rustc_hir_analysis/src/constrained_generic_params.rs
index 56cc1d8fa..e18b0f082 100644
--- a/compiler/rustc_hir_analysis/src/constrained_generic_params.rs
+++ b/compiler/rustc_hir_analysis/src/constrained_generic_params.rs
@@ -43,7 +43,7 @@ pub fn parameters_for_impl<'tcx>(
 /// of parameters whose values are needed in order to constrain `ty` - these
 /// differ, with the latter being a superset, in the presence of projections.
 pub fn parameters_for<'tcx>(
-    t: &impl TypeVisitable<'tcx>,
+    t: &impl TypeVisitable<TyCtxt<'tcx>>,
     include_nonconstraining: bool,
 ) -> Vec<Parameter> {
     let mut collector = ParameterCollector { parameters: vec![], include_nonconstraining };
@@ -56,7 +56,7 @@ struct ParameterCollector {
     include_nonconstraining: bool,
 }
 
-impl<'tcx> TypeVisitor<'tcx> for ParameterCollector {
+impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for ParameterCollector {
     fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
         match *t.kind() {
             ty::Alias(ty::Projection, ..) if !self.include_nonconstraining => {
diff --git a/compiler/rustc_hir_analysis/src/errors.rs b/compiler/rustc_hir_analysis/src/errors.rs
index 04f5f3f62..3e0692757 100644
--- a/compiler/rustc_hir_analysis/src/errors.rs
+++ b/compiler/rustc_hir_analysis/src/errors.rs
@@ -1,8 +1,11 @@
 //! Errors emitted by `rustc_hir_analysis`.
 
-use rustc_errors::{error_code, Applicability, DiagnosticBuilder, ErrorGuaranteed, Handler};
-use rustc_errors::{IntoDiagnostic, MultiSpan};
-use rustc_macros::{Diagnostic, LintDiagnostic};
+use crate::fluent_generated as fluent;
+use rustc_errors::{
+    error_code, Applicability, DiagnosticBuilder, ErrorGuaranteed, Handler, IntoDiagnostic,
+    MultiSpan,
+};
+use rustc_macros::Diagnostic;
 use rustc_middle::ty::Ty;
 use rustc_span::{symbol::Ident, Span, Symbol};
 
@@ -41,11 +44,11 @@ pub struct LifetimesOrBoundsMismatchOnTrait {
     #[primary_span]
     #[label]
     pub span: Span,
-    #[label(generics_label)]
+    #[label(hir_analysis_generics_label)]
     pub generics_span: Option<Span>,
-    #[label(where_label)]
+    #[label(hir_analysis_where_label)]
     pub where_span: Option<Span>,
-    #[label(bounds_label)]
+    #[label(hir_analysis_bounds_label)]
     pub bounds_span: Vec<Span>,
     pub item_kind: &'static str,
     pub ident: Ident,
@@ -57,7 +60,7 @@ pub struct AsyncTraitImplShouldBeAsync {
     #[primary_span]
     // #[label]
     pub span: Span,
-    #[label(trait_item_label)]
+    #[label(hir_analysis_trait_item_label)]
     pub trait_item_span: Option<Span>,
     pub method_name: Symbol,
 }
@@ -77,7 +80,7 @@ pub struct FieldAlreadyDeclared {
     #[primary_span]
     #[label]
     pub span: Span,
-    #[label(previous_decl_label)]
+    #[label(hir_analysis_previous_decl_label)]
     pub prev_span: Span,
 }
 
@@ -109,7 +112,7 @@ pub struct CopyImplOnNonAdt {
 pub struct TraitObjectDeclaredWithNoTraits {
     #[primary_span]
     pub span: Span,
-    #[label(alias_span)]
+    #[label(hir_analysis_alias_span)]
     pub trait_alias_span: Option<Span>,
 }
 
@@ -145,7 +148,7 @@ pub struct ValueOfAssociatedStructAlreadySpecified {
     #[primary_span]
     #[label]
     pub span: Span,
-    #[label(previous_bound_label)]
+    #[label(hir_analysis_previous_bound_label)]
     pub prev_span: Span,
     pub item_name: Ident,
     pub def_path: String,
@@ -175,7 +178,7 @@ impl<'a> IntoDiagnostic<'a> for MissingTypeParams {
     fn into_diagnostic(self, handler: &'a Handler) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
         let mut err = handler.struct_span_err_with_code(
             self.span,
-            rustc_errors::fluent::hir_analysis_missing_type_params,
+            fluent::hir_analysis_missing_type_params,
             error_code!(E0393),
         );
         err.set_arg("parameterCount", self.missing_type_params.len());
@@ -188,7 +191,7 @@ impl<'a> IntoDiagnostic<'a> for MissingTypeParams {
                 .join(", "),
         );
 
-        err.span_label(self.def_span, rustc_errors::fluent::label);
+        err.span_label(self.def_span, fluent::hir_analysis_label);
 
         let mut suggested = false;
         // Don't suggest setting the type params if there are some already: the order is
@@ -203,7 +206,7 @@ impl<'a> IntoDiagnostic<'a> for MissingTypeParams {
                 // least we can clue them to the correct syntax `Iterator<Type>`.
                 err.span_suggestion(
                     self.span,
-                    rustc_errors::fluent::suggestion,
+                    fluent::hir_analysis_suggestion,
                     format!(
                         "{}<{}>",
                         snippet,
@@ -219,10 +222,10 @@ impl<'a> IntoDiagnostic<'a> for MissingTypeParams {
             }
         }
         if !suggested {
-            err.span_label(self.span, rustc_errors::fluent::no_suggestion_label);
+            err.span_label(self.span, fluent::hir_analysis_no_suggestion_label);
         }
 
-        err.note(rustc_errors::fluent::note);
+        err.note(fluent::hir_analysis_note);
         err
     }
 }
@@ -244,26 +247,6 @@ pub struct SubstsOnOverriddenImpl {
     pub span: Span,
 }
 
-#[derive(LintDiagnostic)]
-#[diag(hir_analysis_unused_extern_crate)]
-pub struct UnusedExternCrate {
-    #[suggestion(applicability = "machine-applicable", code = "")]
-    pub span: Span,
-}
-
-#[derive(LintDiagnostic)]
-#[diag(hir_analysis_extern_crate_not_idiomatic)]
-pub struct ExternCrateNotIdiomatic {
-    #[suggestion(
-        style = "short",
-        applicability = "machine-applicable",
-        code = "{suggestion_code}"
-    )]
-    pub span: Span,
-    pub msg_code: String,
-    pub suggestion_code: String,
-}
-
 #[derive(Diagnostic)]
 #[diag(hir_analysis_const_impl_for_non_const_trait)]
 pub struct ConstImplForNonConstTrait {
@@ -274,7 +257,7 @@ pub struct ConstImplForNonConstTrait {
     pub local_trait_span: Option<Span>,
     #[note]
     pub marking: (),
-    #[note(adding)]
+    #[note(hir_analysis_adding)]
     pub adding: (),
 }
 
@@ -312,3 +295,107 @@ pub struct AutoDerefReachedRecursionLimit<'a> {
     pub suggested_limit: rustc_session::Limit,
     pub crate_name: Symbol,
 }
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_where_clause_on_main, code = "E0646")]
+pub(crate) struct WhereClauseOnMain {
+    #[primary_span]
+    pub span: Span,
+    #[label]
+    pub generics_span: Option<Span>,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_track_caller_on_main)]
+pub(crate) struct TrackCallerOnMain {
+    #[primary_span]
+    #[suggestion(applicability = "maybe-incorrect", code = "")]
+    pub span: Span,
+    #[label(hir_analysis_track_caller_on_main)]
+    pub annotated: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_start_not_track_caller)]
+pub(crate) struct StartTrackCaller {
+    #[primary_span]
+    pub span: Span,
+    #[label]
+    pub start: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_start_not_async, code = "E0752")]
+pub(crate) struct StartAsync {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_start_function_where, code = "E0647")]
+pub(crate) struct StartFunctionWhere {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_start_function_parameters, code = "E0132")]
+pub(crate) struct StartFunctionParameters {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_main_function_return_type_generic, code = "E0131")]
+pub(crate) struct MainFunctionReturnTypeGeneric {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_main_function_async, code = "E0752")]
+pub(crate) struct MainFunctionAsync {
+    #[primary_span]
+    pub span: Span,
+    #[label]
+    pub asyncness: Option<Span>,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_main_function_generic_parameters, code = "E0131")]
+pub(crate) struct MainFunctionGenericParameters {
+    #[primary_span]
+    pub span: Span,
+    #[label]
+    pub label_span: Option<Span>,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_variadic_function_compatible_convention, code = "E0045")]
+pub(crate) struct VariadicFunctionCompatibleConvention<'a> {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    pub conventions: &'a str,
+}
+
+#[derive(Diagnostic)]
+pub(crate) enum CannotCaptureLateBoundInAnonConst {
+    #[diag(hir_analysis_cannot_capture_late_bound_ty_in_anon_const)]
+    Type {
+        #[primary_span]
+        use_span: Span,
+        #[label]
+        def_span: Span,
+    },
+    #[diag(hir_analysis_cannot_capture_late_bound_const_in_anon_const)]
+    Const {
+        #[primary_span]
+        use_span: Span,
+        #[label]
+        def_span: Span,
+    },
+}
diff --git a/compiler/rustc_hir_analysis/src/hir_wf_check.rs b/compiler/rustc_hir_analysis/src/hir_wf_check.rs
index 17dbb126b..e330fcc78 100644
--- a/compiler/rustc_hir_analysis/src/hir_wf_check.rs
+++ b/compiler/rustc_hir_analysis/src/hir_wf_check.rs
@@ -1,11 +1,12 @@
 use crate::collect::ItemCtxt;
 use rustc_hir as hir;
 use rustc_hir::intravisit::{self, Visitor};
-use rustc_hir::{ForeignItem, ForeignItemKind, HirId};
+use rustc_hir::{ForeignItem, ForeignItemKind};
 use rustc_infer::infer::TyCtxtInferExt;
 use rustc_infer::traits::{ObligationCause, WellFormedLoc};
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, Region, TyCtxt, TypeFoldable, TypeFolder};
+use rustc_span::def_id::LocalDefId;
 use rustc_trait_selection::traits;
 
 pub fn provide(providers: &mut Providers) {
@@ -57,7 +58,7 @@ fn diagnostic_hir_wf_check<'tcx>(
         cause: Option<ObligationCause<'tcx>>,
         cause_depth: usize,
         icx: ItemCtxt<'tcx>,
-        hir_id: HirId,
+        def_id: LocalDefId,
         param_env: ty::ParamEnv<'tcx>,
         depth: usize,
     }
@@ -68,7 +69,7 @@ fn diagnostic_hir_wf_check<'tcx>(
             let tcx_ty = self.icx.to_ty(ty).fold_with(&mut EraseAllBoundRegions { tcx: self.tcx });
             let cause = traits::ObligationCause::new(
                 ty.span,
-                self.hir_id,
+                self.def_id,
                 traits::ObligationCauseCode::WellFormed(None),
             );
             let errors = traits::fully_solve_obligation(
@@ -106,7 +107,7 @@ fn diagnostic_hir_wf_check<'tcx>(
         cause: None,
         cause_depth: 0,
         icx,
-        hir_id,
+        def_id,
         param_env: tcx.param_env(def_id.to_def_id()),
         depth: 0,
     };
@@ -188,8 +189,8 @@ struct EraseAllBoundRegions<'tcx> {
 // us an inaccurate span for an error message, but cannot
 // lead to unsoundness (we call `delay_span_bug` at the start
 // of `diagnostic_hir_wf_check`).
-impl<'tcx> TypeFolder<'tcx> for EraseAllBoundRegions<'tcx> {
-    fn tcx<'a>(&'a self) -> TyCtxt<'tcx> {
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for EraseAllBoundRegions<'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
     fn fold_region(&mut self, r: Region<'tcx>) -> Region<'tcx> {
diff --git a/compiler/rustc_hir_analysis/src/impl_wf_check.rs b/compiler/rustc_hir_analysis/src/impl_wf_check.rs
index 4fe893442..82a96f8e6 100644
--- a/compiler/rustc_hir_analysis/src/impl_wf_check.rs
+++ b/compiler/rustc_hir_analysis/src/impl_wf_check.rs
@@ -16,7 +16,7 @@ use rustc_errors::struct_span_err;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::LocalDefId;
 use rustc_middle::ty::query::Providers;
-use rustc_middle::ty::{self, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, TyCtxt, TypeVisitableExt};
 use rustc_span::{Span, Symbol};
 
 mod min_specialization;
@@ -55,7 +55,7 @@ fn check_mod_impl_wf(tcx: TyCtxt<'_>, module_def_id: LocalDefId) {
     let min_specialization = tcx.features().min_specialization;
     let module = tcx.hir_module_items(module_def_id);
     for id in module.items() {
-        if matches!(tcx.def_kind(id.owner_id), DefKind::Impl) {
+        if matches!(tcx.def_kind(id.owner_id), DefKind::Impl { .. }) {
             enforce_impl_params_are_constrained(tcx, id.owner_id.def_id);
             if min_specialization {
                 check_min_specialization(tcx, id.owner_id.def_id);
@@ -70,7 +70,7 @@ pub fn provide(providers: &mut Providers) {
 
 fn enforce_impl_params_are_constrained(tcx: TyCtxt<'_>, impl_def_id: LocalDefId) {
     // Every lifetime used in an associated type must be constrained.
-    let impl_self_ty = tcx.type_of(impl_def_id);
+    let impl_self_ty = tcx.type_of(impl_def_id).subst_identity();
     if impl_self_ty.references_error() {
         // Don't complain about unconstrained type params when self ty isn't known due to errors.
         // (#36836)
@@ -104,7 +104,7 @@ fn enforce_impl_params_are_constrained(tcx: TyCtxt<'_>, impl_def_id: LocalDefId)
             match item.kind {
                 ty::AssocKind::Type => {
                     if item.defaultness(tcx).has_value() {
-                        cgp::parameters_for(&tcx.type_of(def_id), true)
+                        cgp::parameters_for(&tcx.type_of(def_id).subst_identity(), true)
                     } else {
                         Vec::new()
                     }
diff --git a/compiler/rustc_hir_analysis/src/impl_wf_check/min_specialization.rs b/compiler/rustc_hir_analysis/src/impl_wf_check/min_specialization.rs
index bcda26c4c..daa5d1570 100644
--- a/compiler/rustc_hir_analysis/src/impl_wf_check/min_specialization.rs
+++ b/compiler/rustc_hir_analysis/src/impl_wf_check/min_specialization.rs
@@ -76,7 +76,7 @@ use rustc_infer::infer::TyCtxtInferExt;
 use rustc_infer::traits::specialization_graph::Node;
 use rustc_middle::ty::subst::{GenericArg, InternalSubsts, SubstsRef};
 use rustc_middle::ty::trait_def::TraitSpecializationKind;
-use rustc_middle::ty::{self, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, TyCtxt, TypeVisitableExt};
 use rustc_span::Span;
 use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt;
 use rustc_trait_selection::traits::outlives_bounds::InferCtxtExt as _;
@@ -164,7 +164,6 @@ fn get_impl_substs(
     let infcx = &tcx.infer_ctxt().build();
     let ocx = ObligationCtxt::new(infcx);
     let param_env = tcx.param_env(impl1_def_id);
-    let impl1_hir_id = tcx.hir().local_def_id_to_hir_id(impl1_def_id);
 
     let assumed_wf_types =
         ocx.assumed_wf_types(param_env, tcx.def_span(impl1_def_id), impl1_def_id);
@@ -179,7 +178,7 @@ fn get_impl_substs(
         return None;
     }
 
-    let implied_bounds = infcx.implied_bounds_tys(param_env, impl1_hir_id, assumed_wf_types);
+    let implied_bounds = infcx.implied_bounds_tys(param_env, impl1_def_id, assumed_wf_types);
     let outlives_env = OutlivesEnvironment::with_bounds(param_env, Some(infcx), implied_bounds);
     let _ =
         infcx.err_ctxt().check_region_obligations_and_report_errors(impl1_def_id, &outlives_env);
@@ -372,15 +371,9 @@ fn check_predicates<'tcx>(
     // Include the well-formed predicates of the type parameters of the impl.
     for arg in tcx.impl_trait_ref(impl1_def_id).unwrap().subst_identity().substs {
         let infcx = &tcx.infer_ctxt().build();
-        let obligations = wf::obligations(
-            infcx,
-            tcx.param_env(impl1_def_id),
-            tcx.hir().local_def_id_to_hir_id(impl1_def_id),
-            0,
-            arg,
-            span,
-        )
-        .unwrap();
+        let obligations =
+            wf::obligations(infcx, tcx.param_env(impl1_def_id), impl1_def_id, 0, arg, span)
+                .unwrap();
 
         assert!(!obligations.needs_infer());
         impl2_predicates.extend(
@@ -503,6 +496,16 @@ fn check_specialization_on<'tcx>(tcx: TyCtxt<'tcx>, predicate: ty::Predicate<'tc
                 )
                 .emit();
         }
+        ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..)) => {
+            // FIXME(min_specialization), FIXME(const_generics):
+            // It probably isn't right to allow _every_ `ConstArgHasType` but I am somewhat unsure
+            // about the actual rules that would be sound. Can't just always error here because otherwise
+            // std/core doesn't even compile as they have `const N: usize` in some specializing impls.
+            //
+            // While we do not support constructs like `<T, const N: T>` there is probably no risk of
+            // soundness bugs, but when we support generic const parameter types this will need to be
+            // revisited.
+        }
         _ => {
             tcx.sess
                 .struct_span_err(span, &format!("cannot specialize on predicate `{}`", predicate))
@@ -524,6 +527,8 @@ fn trait_predicate_kind<'tcx>(
         ty::PredicateKind::Clause(ty::Clause::RegionOutlives(_))
         | ty::PredicateKind::Clause(ty::Clause::TypeOutlives(_))
         | ty::PredicateKind::Clause(ty::Clause::Projection(_))
+        | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
+        | ty::PredicateKind::AliasEq(..)
         | ty::PredicateKind::WellFormed(_)
         | ty::PredicateKind::Subtype(_)
         | ty::PredicateKind::Coerce(_)
diff --git a/compiler/rustc_hir_analysis/src/lib.rs b/compiler/rustc_hir_analysis/src/lib.rs
index 02548ae89..33c132fd5 100644
--- a/compiler/rustc_hir_analysis/src/lib.rs
+++ b/compiler/rustc_hir_analysis/src/lib.rs
@@ -98,29 +98,31 @@ mod outlives;
 pub mod structured_errors;
 mod variance;
 
-use rustc_errors::{struct_span_err, ErrorGuaranteed};
+use rustc_errors::ErrorGuaranteed;
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
 use rustc_hir as hir;
-use rustc_hir::def_id::DefId;
-use rustc_hir::{Node, CRATE_HIR_ID};
+use rustc_hir::Node;
 use rustc_infer::infer::{InferOk, TyCtxtInferExt};
+use rustc_macros::fluent_messages;
 use rustc_middle::middle;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_middle::util;
 use rustc_session::{config::EntryFnType, parse::feature_err};
+use rustc_span::def_id::{DefId, LocalDefId, CRATE_DEF_ID};
 use rustc_span::{symbol::sym, Span, DUMMY_SP};
 use rustc_target::spec::abi::Abi;
 use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt as _;
 use rustc_trait_selection::traits::{self, ObligationCause, ObligationCauseCode};
 
-use std::iter;
 use std::ops::Not;
 
 use astconv::AstConv;
 use bounds::Bounds;
 
+fluent_messages! { "../locales/en-US.ftl" }
+
 fn require_c_abi_if_c_variadic(tcx: TyCtxt<'_>, decl: &hir::FnDecl<'_>, abi: Abi, span: Span) {
-    const ERROR_HEAD: &str = "C-variadic function must have a compatible calling convention";
     const CONVENTIONS_UNSTABLE: &str = "`C`, `cdecl`, `win64`, `sysv64` or `efiapi`";
     const CONVENTIONS_STABLE: &str = "`C` or `cdecl`";
     const UNSTABLE_EXPLAIN: &str =
@@ -152,8 +154,7 @@ fn require_c_abi_if_c_variadic(tcx: TyCtxt<'_>, decl: &hir::FnDecl<'_>, abi: Abi
         (true, false) => CONVENTIONS_UNSTABLE,
     };
 
-    let mut err = struct_span_err!(tcx.sess, span, E0045, "{}, like {}", ERROR_HEAD, conventions);
-    err.span_label(span, ERROR_HEAD).emit();
+    tcx.sess.emit_err(errors::VariadicFunctionCompatibleConvention { span, conventions });
 }
 
 fn require_same_types<'tcx>(
@@ -182,19 +183,18 @@ fn require_same_types<'tcx>(
 }
 
 fn check_main_fn_ty(tcx: TyCtxt<'_>, main_def_id: DefId) {
-    let main_fnsig = tcx.fn_sig(main_def_id);
+    let main_fnsig = tcx.fn_sig(main_def_id).subst_identity();
     let main_span = tcx.def_span(main_def_id);
 
-    fn main_fn_diagnostics_hir_id(tcx: TyCtxt<'_>, def_id: DefId, sp: Span) -> hir::HirId {
+    fn main_fn_diagnostics_def_id(tcx: TyCtxt<'_>, def_id: DefId, sp: Span) -> LocalDefId {
         if let Some(local_def_id) = def_id.as_local() {
-            let hir_id = tcx.hir().local_def_id_to_hir_id(local_def_id);
-            let hir_type = tcx.type_of(local_def_id);
+            let hir_type = tcx.type_of(local_def_id).subst_identity();
             if !matches!(hir_type.kind(), ty::FnDef(..)) {
                 span_bug!(sp, "main has a non-function type: found `{}`", hir_type);
             }
-            hir_id
+            local_def_id
         } else {
-            CRATE_HIR_ID
+            CRATE_DEF_ID
         }
     }
 
@@ -205,7 +205,7 @@ fn check_main_fn_ty(tcx: TyCtxt<'_>, main_def_id: DefId) {
         let hir_id = tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
         match tcx.hir().find(hir_id) {
             Some(Node::Item(hir::Item { kind: hir::ItemKind::Fn(_, generics, _), .. })) => {
-                generics.params.is_empty().not().then(|| generics.span)
+                generics.params.is_empty().not().then_some(generics.span)
             }
             _ => {
                 span_bug!(tcx.def_span(def_id), "main has a non-function type");
@@ -251,58 +251,35 @@ fn check_main_fn_ty(tcx: TyCtxt<'_>, main_def_id: DefId) {
     }
 
     let mut error = false;
-    let main_diagnostics_hir_id = main_fn_diagnostics_hir_id(tcx, main_def_id, main_span);
+    let main_diagnostics_def_id = main_fn_diagnostics_def_id(tcx, main_def_id, main_span);
     let main_fn_generics = tcx.generics_of(main_def_id);
     let main_fn_predicates = tcx.predicates_of(main_def_id);
     if main_fn_generics.count() != 0 || !main_fnsig.bound_vars().is_empty() {
         let generics_param_span = main_fn_generics_params_span(tcx, main_def_id);
-        let msg = "`main` function is not allowed to have generic \
-            parameters";
-        let mut diag =
-            struct_span_err!(tcx.sess, generics_param_span.unwrap_or(main_span), E0131, "{}", msg);
-        if let Some(generics_param_span) = generics_param_span {
-            let label = "`main` cannot have generic parameters";
-            diag.span_label(generics_param_span, label);
-        }
-        diag.emit();
+        tcx.sess.emit_err(errors::MainFunctionGenericParameters {
+            span: generics_param_span.unwrap_or(main_span),
+            label_span: generics_param_span,
+        });
         error = true;
     } else if !main_fn_predicates.predicates.is_empty() {
         // generics may bring in implicit predicates, so we skip this check if generics is present.
         let generics_where_clauses_span = main_fn_where_clauses_span(tcx, main_def_id);
-        let mut diag = struct_span_err!(
-            tcx.sess,
-            generics_where_clauses_span.unwrap_or(main_span),
-            E0646,
-            "`main` function is not allowed to have a `where` clause"
-        );
-        if let Some(generics_where_clauses_span) = generics_where_clauses_span {
-            diag.span_label(generics_where_clauses_span, "`main` cannot have a `where` clause");
-        }
-        diag.emit();
+        tcx.sess.emit_err(errors::WhereClauseOnMain {
+            span: generics_where_clauses_span.unwrap_or(main_span),
+            generics_span: generics_where_clauses_span,
+        });
         error = true;
     }
 
     let main_asyncness = tcx.asyncness(main_def_id);
     if let hir::IsAsync::Async = main_asyncness {
-        let mut diag = struct_span_err!(
-            tcx.sess,
-            main_span,
-            E0752,
-            "`main` function is not allowed to be `async`"
-        );
         let asyncness_span = main_fn_asyncness_span(tcx, main_def_id);
-        if let Some(asyncness_span) = asyncness_span {
-            diag.span_label(asyncness_span, "`main` function is not allowed to be `async`");
-        }
-        diag.emit();
+        tcx.sess.emit_err(errors::MainFunctionAsync { span: main_span, asyncness: asyncness_span });
         error = true;
     }
 
     for attr in tcx.get_attrs(main_def_id, sym::track_caller) {
-        tcx.sess
-            .struct_span_err(attr.span, "`main` function is not allowed to be `#[track_caller]`")
-            .span_label(main_span, "`main` function is not allowed to be `#[track_caller]`")
-            .emit();
+        tcx.sess.emit_err(errors::TrackCallerOnMain { span: attr.span, annotated: main_span });
         error = true;
     }
 
@@ -315,9 +292,7 @@ fn check_main_fn_ty(tcx: TyCtxt<'_>, main_def_id: DefId) {
         let return_ty = main_fnsig.output();
         let return_ty_span = main_fn_return_type_span(tcx, main_def_id).unwrap_or(main_span);
         if !return_ty.bound_vars().is_empty() {
-            let msg = "`main` function return type is not allowed to have generic \
-                    parameters";
-            struct_span_err!(tcx.sess, return_ty_span, E0131, "{}", msg).emit();
+            tcx.sess.emit_err(errors::MainFunctionReturnTypeGeneric { span: return_ty_span });
             error = true;
         }
         let return_ty = return_ty.skip_binder();
@@ -326,7 +301,7 @@ fn check_main_fn_ty(tcx: TyCtxt<'_>, main_def_id: DefId) {
         let param_env = ty::ParamEnv::empty();
         let cause = traits::ObligationCause::new(
             return_ty_span,
-            main_diagnostics_hir_id,
+            main_diagnostics_def_id,
             ObligationCauseCode::MainFunctionType,
         );
         let ocx = traits::ObligationCtxt::new(&infcx);
@@ -349,14 +324,14 @@ fn check_main_fn_ty(tcx: TyCtxt<'_>, main_def_id: DefId) {
     }
 
     let se_ty = tcx.mk_fn_ptr(expected_return_type.map_bound(|expected_return_type| {
-        tcx.mk_fn_sig(iter::empty(), expected_return_type, false, hir::Unsafety::Normal, Abi::Rust)
+        tcx.mk_fn_sig([], expected_return_type, false, hir::Unsafety::Normal, Abi::Rust)
     }));
 
     require_same_types(
         tcx,
         &ObligationCause::new(
             main_span,
-            main_diagnostics_hir_id,
+            main_diagnostics_def_id,
             ObligationCauseCode::MainFunctionType,
         ),
         se_ty,
@@ -367,63 +342,35 @@ fn check_start_fn_ty(tcx: TyCtxt<'_>, start_def_id: DefId) {
     let start_def_id = start_def_id.expect_local();
     let start_id = tcx.hir().local_def_id_to_hir_id(start_def_id);
     let start_span = tcx.def_span(start_def_id);
-    let start_t = tcx.type_of(start_def_id);
+    let start_t = tcx.type_of(start_def_id).subst_identity();
     match start_t.kind() {
         ty::FnDef(..) => {
             if let Some(Node::Item(it)) = tcx.hir().find(start_id) {
                 if let hir::ItemKind::Fn(sig, generics, _) = &it.kind {
                     let mut error = false;
                     if !generics.params.is_empty() {
-                        struct_span_err!(
-                            tcx.sess,
-                            generics.span,
-                            E0132,
-                            "start function is not allowed to have type parameters"
-                        )
-                        .span_label(generics.span, "start function cannot have type parameters")
-                        .emit();
+                        tcx.sess.emit_err(errors::StartFunctionParameters { span: generics.span });
                         error = true;
                     }
                     if generics.has_where_clause_predicates {
-                        struct_span_err!(
-                            tcx.sess,
-                            generics.where_clause_span,
-                            E0647,
-                            "start function is not allowed to have a `where` clause"
-                        )
-                        .span_label(
-                            generics.where_clause_span,
-                            "start function cannot have a `where` clause",
-                        )
-                        .emit();
+                        tcx.sess.emit_err(errors::StartFunctionWhere {
+                            span: generics.where_clause_span,
+                        });
                         error = true;
                     }
                     if let hir::IsAsync::Async = sig.header.asyncness {
                         let span = tcx.def_span(it.owner_id);
-                        struct_span_err!(
-                            tcx.sess,
-                            span,
-                            E0752,
-                            "`start` is not allowed to be `async`"
-                        )
-                        .span_label(span, "`start` is not allowed to be `async`")
-                        .emit();
+                        tcx.sess.emit_err(errors::StartAsync { span: span });
                         error = true;
                     }
 
                     let attrs = tcx.hir().attrs(start_id);
                     for attr in attrs {
                         if attr.has_name(sym::track_caller) {
-                            tcx.sess
-                                .struct_span_err(
-                                    attr.span,
-                                    "`start` is not allowed to be `#[track_caller]`",
-                                )
-                                .span_label(
-                                    start_span,
-                                    "`start` is not allowed to be `#[track_caller]`",
-                                )
-                                .emit();
+                            tcx.sess.emit_err(errors::StartTrackCaller {
+                                span: attr.span,
+                                start: start_span,
+                            });
                             error = true;
                         }
                     }
@@ -435,7 +382,7 @@ fn check_start_fn_ty(tcx: TyCtxt<'_>, start_def_id: DefId) {
             }
 
             let se_ty = tcx.mk_fn_ptr(ty::Binder::dummy(tcx.mk_fn_sig(
-                [tcx.types.isize, tcx.mk_imm_ptr(tcx.mk_imm_ptr(tcx.types.u8))].iter().cloned(),
+                [tcx.types.isize, tcx.mk_imm_ptr(tcx.mk_imm_ptr(tcx.types.u8))],
                 tcx.types.isize,
                 false,
                 hir::Unsafety::Normal,
@@ -444,9 +391,13 @@ fn check_start_fn_ty(tcx: TyCtxt<'_>, start_def_id: DefId) {
 
             require_same_types(
                 tcx,
-                &ObligationCause::new(start_span, start_id, ObligationCauseCode::StartFunctionType),
+                &ObligationCause::new(
+                    start_span,
+                    start_def_id,
+                    ObligationCauseCode::StartFunctionType,
+                ),
                 se_ty,
-                tcx.mk_fn_ptr(tcx.fn_sig(start_def_id)),
+                tcx.mk_fn_ptr(tcx.fn_sig(start_def_id).subst_identity()),
             );
         }
         _ => {
diff --git a/compiler/rustc_hir_analysis/src/outlives/explicit.rs b/compiler/rustc_hir_analysis/src/outlives/explicit.rs
index 663f1c49d..9ee678597 100644
--- a/compiler/rustc_hir_analysis/src/outlives/explicit.rs
+++ b/compiler/rustc_hir_analysis/src/outlives/explicit.rs
@@ -54,7 +54,9 @@ impl<'tcx> ExplicitPredicatesMap<'tcx> {
 
                     ty::PredicateKind::Clause(ty::Clause::Trait(..))
                     | ty::PredicateKind::Clause(ty::Clause::Projection(..))
+                    | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
                     | ty::PredicateKind::WellFormed(..)
+                    | ty::PredicateKind::AliasEq(..)
                     | ty::PredicateKind::ObjectSafe(..)
                     | ty::PredicateKind::ClosureKind(..)
                     | ty::PredicateKind::Subtype(..)
diff --git a/compiler/rustc_hir_analysis/src/outlives/implicit_infer.rs b/compiler/rustc_hir_analysis/src/outlives/implicit_infer.rs
index 925042436..a8b33c74b 100644
--- a/compiler/rustc_hir_analysis/src/outlives/implicit_infer.rs
+++ b/compiler/rustc_hir_analysis/src/outlives/implicit_infer.rs
@@ -46,7 +46,7 @@ pub(super) fn infer_predicates(
                         // For field of type &'a T (reference) or Adt
                         // (struct/enum/union) there will be outlive
                         // requirements for adt_def.
-                        let field_ty = tcx.type_of(field_def.did);
+                        let field_ty = tcx.type_of(field_def.did).subst_identity();
                         let field_span = tcx.def_span(field_def.did);
                         insert_required_predicates_to_be_wf(
                             tcx,
diff --git a/compiler/rustc_hir_analysis/src/outlives/utils.rs b/compiler/rustc_hir_analysis/src/outlives/utils.rs
index 9459c5f54..c5c5f63a1 100644
--- a/compiler/rustc_hir_analysis/src/outlives/utils.rs
+++ b/compiler/rustc_hir_analysis/src/outlives/utils.rs
@@ -170,6 +170,8 @@ fn is_free_region(region: Region<'_>) -> bool {
         // ignore it. We can't put it on the struct header anyway.
         ty::ReLateBound(..) => false,
 
+        ty::ReError(_) => false,
+
         // These regions don't appear in types from type declarations:
         ty::ReErased | ty::ReVar(..) | ty::RePlaceholder(..) | ty::ReFree(..) => {
             bug!("unexpected region in outlives inference: {:?}", region);
diff --git a/compiler/rustc_hir_analysis/src/structured_errors/missing_cast_for_variadic_arg.rs b/compiler/rustc_hir_analysis/src/structured_errors/missing_cast_for_variadic_arg.rs
index 324df313e..089491bef 100644
--- a/compiler/rustc_hir_analysis/src/structured_errors/missing_cast_for_variadic_arg.rs
+++ b/compiler/rustc_hir_analysis/src/structured_errors/missing_cast_for_variadic_arg.rs
@@ -1,6 +1,6 @@
 use crate::structured_errors::StructuredDiagnostic;
 use rustc_errors::{Applicability, DiagnosticBuilder, DiagnosticId, ErrorGuaranteed};
-use rustc_middle::ty::{Ty, TypeVisitable};
+use rustc_middle::ty::{Ty, TypeVisitableExt};
 use rustc_session::Session;
 use rustc_span::Span;
 
diff --git a/compiler/rustc_hir_analysis/src/structured_errors/sized_unsized_cast.rs b/compiler/rustc_hir_analysis/src/structured_errors/sized_unsized_cast.rs
index bb6088054..3b9fb3678 100644
--- a/compiler/rustc_hir_analysis/src/structured_errors/sized_unsized_cast.rs
+++ b/compiler/rustc_hir_analysis/src/structured_errors/sized_unsized_cast.rs
@@ -1,6 +1,6 @@
 use crate::structured_errors::StructuredDiagnostic;
 use rustc_errors::{DiagnosticBuilder, DiagnosticId, ErrorGuaranteed};
-use rustc_middle::ty::{Ty, TypeVisitable};
+use rustc_middle::ty::{Ty, TypeVisitableExt};
 use rustc_session::Session;
 use rustc_span::Span;
 
diff --git a/compiler/rustc_hir_analysis/src/structured_errors/wrong_number_of_generic_args.rs b/compiler/rustc_hir_analysis/src/structured_errors/wrong_number_of_generic_args.rs
index 9133e6540..cae884ae8 100644
--- a/compiler/rustc_hir_analysis/src/structured_errors/wrong_number_of_generic_args.rs
+++ b/compiler/rustc_hir_analysis/src/structured_errors/wrong_number_of_generic_args.rs
@@ -423,7 +423,7 @@ impl<'a, 'tcx> WrongNumberOfGenericArgs<'a, 'tcx> {
 
     fn get_unbound_associated_types(&self) -> Vec<String> {
         if self.tcx.is_trait(self.def_id) {
-            let items: &AssocItems<'_> = self.tcx.associated_items(self.def_id);
+            let items: &AssocItems = self.tcx.associated_items(self.def_id);
             items
                 .in_definition_order()
                 .filter(|item| item.kind == AssocKind::Type)
@@ -439,7 +439,7 @@ impl<'a, 'tcx> WrongNumberOfGenericArgs<'a, 'tcx> {
 
     fn create_error_message(&self) -> String {
         let def_path = self.tcx.def_path_str(self.def_id);
-        let def_kind = self.tcx.def_kind(self.def_id).descr(self.def_id);
+        let def_kind = self.tcx.def_descr(self.def_id);
         let (quantifier, bound) = self.get_quantifier_and_bound();
         let kind = self.kind();
         let provided_lt_args = self.num_provided_lifetime_args();
@@ -462,7 +462,7 @@ impl<'a, 'tcx> WrongNumberOfGenericArgs<'a, 'tcx> {
 
         if self.gen_args.span_ext().is_some() {
             format!(
-                "this {} takes {}{} {} argument{} but {} {} supplied",
+                "{} takes {}{} {} argument{} but {} {} supplied",
                 def_kind,
                 quantifier,
                 bound,
@@ -990,7 +990,7 @@ impl<'a, 'tcx> WrongNumberOfGenericArgs<'a, 'tcx> {
         };
 
         let msg = {
-            let def_kind = self.tcx.def_kind(self.def_id).descr(self.def_id);
+            let def_kind = self.tcx.def_descr(self.def_id);
             let (quantifier, bound) = self.get_quantifier_and_bound();
 
             let params = if bound == 0 {
diff --git a/compiler/rustc_hir_analysis/src/variance/constraints.rs b/compiler/rustc_hir_analysis/src/variance/constraints.rs
index 5e4d82b6f..408bec71e 100644
--- a/compiler/rustc_hir_analysis/src/variance/constraints.rs
+++ b/compiler/rustc_hir_analysis/src/variance/constraints.rs
@@ -101,7 +101,7 @@ impl<'a, 'tcx> ConstraintContext<'a, 'tcx> {
 
         let inferred_start = self.terms_cx.inferred_starts[&def_id];
         let current_item = &CurrentItem { inferred_start };
-        match tcx.type_of(def_id).kind() {
+        match tcx.type_of(def_id).subst_identity().kind() {
             ty::Adt(def, _) => {
                 // Not entirely obvious: constraints on structs/enums do not
                 // affect the variance of their type parameters. See discussion
@@ -112,14 +112,18 @@ impl<'a, 'tcx> ConstraintContext<'a, 'tcx> {
                 for field in def.all_fields() {
                     self.add_constraints_from_ty(
                         current_item,
-                        tcx.type_of(field.did),
+                        tcx.type_of(field.did).subst_identity(),
                         self.covariant,
                     );
                 }
             }
 
             ty::FnDef(..) => {
-                self.add_constraints_from_sig(current_item, tcx.fn_sig(def_id), self.covariant);
+                self.add_constraints_from_sig(
+                    current_item,
+                    tcx.fn_sig(def_id).subst_identity(),
+                    self.covariant,
+                );
             }
 
             ty::Error(_) => {}
@@ -221,8 +225,7 @@ impl<'a, 'tcx> ConstraintContext<'a, 'tcx> {
             }
 
             ty::Ref(region, ty, mutbl) => {
-                let contra = self.contravariant(variance);
-                self.add_constraints_from_region(current, region, contra);
+                self.add_constraints_from_region(current, region, variance);
                 self.add_constraints_from_mt(current, &ty::TypeAndMut { ty, mutbl }, variance);
             }
 
@@ -254,9 +257,8 @@ impl<'a, 'tcx> ConstraintContext<'a, 'tcx> {
             }
 
             ty::Dynamic(data, r, _) => {
-                // The type `Foo<T+'a>` is contravariant w/r/t `'a`:
-                let contra = self.contravariant(variance);
-                self.add_constraints_from_region(current, r, contra);
+                // The type `dyn Trait<T> +'a` is covariant w/r/t `'a`:
+                self.add_constraints_from_region(current, r, variance);
 
                 if let Some(poly_trait_ref) = data.principal() {
                     self.add_constraints_from_invariant_substs(
@@ -291,12 +293,12 @@ impl<'a, 'tcx> ConstraintContext<'a, 'tcx> {
                 // types, where we use Error as the Self type
             }
 
-            ty::Placeholder(..) | ty::GeneratorWitness(..) | ty::Bound(..) | ty::Infer(..) => {
-                bug!(
-                    "unexpected type encountered in \
-                      variance inference: {}",
-                    ty
-                );
+            ty::Placeholder(..)
+            | ty::GeneratorWitness(..)
+            | ty::GeneratorWitnessMIR(..)
+            | ty::Bound(..)
+            | ty::Infer(..) => {
+                bug!("unexpected type encountered in variance inference: {}", ty);
             }
         }
     }
@@ -407,6 +409,8 @@ impl<'a, 'tcx> ConstraintContext<'a, 'tcx> {
                 // way early-bound regions do, so we skip them here.
             }
 
+            ty::ReError(_) => {}
+
             ty::ReFree(..) | ty::ReVar(..) | ty::RePlaceholder(..) | ty::ReErased => {
                 // We don't expect to see anything but 'static or bound
                 // regions when visiting member types or method types.
diff --git a/compiler/rustc_hir_analysis/src/variance/mod.rs b/compiler/rustc_hir_analysis/src/variance/mod.rs
index 079070be2..5d5c8ca60 100644
--- a/compiler/rustc_hir_analysis/src/variance/mod.rs
+++ b/compiler/rustc_hir_analysis/src/variance/mod.rs
@@ -99,7 +99,7 @@ fn variance_of_opaque(tcx: TyCtxt<'_>, item_def_id: LocalDefId) -> &[ty::Varianc
         }
     }
 
-    impl<'tcx> ty::TypeVisitor<'tcx> for OpaqueTypeLifetimeCollector<'tcx> {
+    impl<'tcx> ty::TypeVisitor<TyCtxt<'tcx>> for OpaqueTypeLifetimeCollector<'tcx> {
         #[instrument(level = "trace", skip(self), ret)]
         fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
             if let ty::RegionKind::ReEarlyBound(ebr) = r.kind() {
diff --git a/compiler/rustc_hir_analysis/src/variance/solve.rs b/compiler/rustc_hir_analysis/src/variance/solve.rs
index a17edb598..c27c176e3 100644
--- a/compiler/rustc_hir_analysis/src/variance/solve.rs
+++ b/compiler/rustc_hir_analysis/src/variance/solve.rs
@@ -103,7 +103,7 @@ impl<'a, 'tcx> SolveContext<'a, 'tcx> {
                 self.enforce_const_invariance(generics, variances);
 
                 // Functions are permitted to have unused generic parameters: make those invariant.
-                if let ty::FnDef(..) = tcx.type_of(def_id).kind() {
+                if let ty::FnDef(..) = tcx.type_of(def_id).subst_identity().kind() {
                     for variance in variances.iter_mut() {
                         if *variance == ty::Bivariant {
                             *variance = ty::Invariant;
diff --git a/compiler/rustc_hir_pretty/src/lib.rs b/compiler/rustc_hir_pretty/src/lib.rs
index f74c551a4..c021fca71 100644
--- a/compiler/rustc_hir_pretty/src/lib.rs
+++ b/compiler/rustc_hir_pretty/src/lib.rs
@@ -9,9 +9,7 @@ use rustc_ast_pretty::pp::{self, Breaks};
 use rustc_ast_pretty::pprust::{Comments, PrintState};
 use rustc_hir as hir;
 use rustc_hir::LifetimeParamKind;
-use rustc_hir::{
-    BindingAnnotation, ByRef, GenericArg, GenericParam, GenericParamKind, Mutability, Node, Term,
-};
+use rustc_hir::{BindingAnnotation, ByRef, GenericArg, GenericParam, GenericParamKind, Node, Term};
 use rustc_hir::{GenericBound, PatKind, RangeEnd, TraitBoundModifier};
 use rustc_span::source_map::SourceMap;
 use rustc_span::symbol::{kw, Ident, IdentPrinter, Symbol};
@@ -360,7 +358,7 @@ impl<'a> State<'a> {
                 self.print_anon_const(e);
                 self.word(")");
             }
-            hir::TyKind::Err => {
+            hir::TyKind::Err(_) => {
                 self.popen();
                 self.word("/*ERROR*/");
                 self.pclose();
@@ -1561,7 +1559,7 @@ impl<'a> State<'a> {
                 self.word_space("yield");
                 self.print_expr_maybe_paren(expr, parser::PREC_JUMP);
             }
-            hir::ExprKind::Err => {
+            hir::ExprKind::Err(_) => {
                 self.popen();
                 self.word("/*ERROR*/");
                 self.pclose();
@@ -1746,7 +1744,7 @@ impl<'a> State<'a> {
                 if by_ref == ByRef::Yes {
                     self.word_nbsp("ref");
                 }
-                if mutbl == Mutability::Mut {
+                if mutbl.is_mut() {
                     self.word_nbsp("mut");
                 }
                 self.print_ident(ident);
diff --git a/compiler/rustc_error_messages/locales/en-US/hir_typeck.ftl b/compiler/rustc_hir_typeck/locales/en-US.ftl
index ca72b7faa..adfcbc36a 100644
--- a/compiler/rustc_error_messages/locales/en-US/hir_typeck.ftl
+++ b/compiler/rustc_hir_typeck/locales/en-US.ftl
@@ -1,17 +1,26 @@
-hir_typeck_fru_note = this expression may have been misinterpreted as a `..` range expression
-hir_typeck_fru_expr = this expression does not end in a comma...
-hir_typeck_fru_expr2 = ... so this is interpreted as a `..` range expression, instead of functional record update syntax
-hir_typeck_fru_suggestion =
-    to set the remaining fields{$expr ->
-        [NONE]{""}
-        *[other] {" "}from `{$expr}`
-    }, separate the last named field with a comma
-
 hir_typeck_field_multiply_specified_in_initializer =
     field `{$ident}` specified more than once
     .label = used more than once
     .previous_use_label = first use of `{$ident}`
 
+hir_typeck_copy_impl_on_type_with_dtor =
+    the trait `Copy` may not be implemented for this type; the type has a destructor
+    .label = `Copy` not allowed on types with destructors
+
+hir_typeck_multiple_relaxed_default_bounds =
+    type parameter has more than one relaxed default bound, only one is supported
+
+hir_typeck_copy_impl_on_non_adt =
+    the trait `Copy` may not be implemented for this type
+    .label = type is not a structure or enumeration
+
+hir_typeck_trait_object_declared_with_no_traits =
+    at least one trait is required for an object type
+    .alias_span = this alias does not contain a trait
+
+hir_typeck_functional_record_update_on_non_struct =
+    functional record update syntax requires a struct
+
 hir_typeck_return_stmt_outside_of_fn_body =
     return statement outside of function body
     .encl_body_label = the return is part of this body...
@@ -26,9 +35,6 @@ hir_typeck_struct_expr_non_exhaustive =
 hir_typeck_method_call_on_unknown_type =
     the type of this value must be known to call a method on a raw pointer on it
 
-hir_typeck_functional_record_update_on_non_struct =
-    functional record update syntax requires a struct
-
 hir_typeck_address_of_temporary_taken = cannot take address of a temporary
     .label = temporary value
 
@@ -44,9 +50,6 @@ hir_typeck_missing_parentheses_in_range = can't call method `{$method_name}` on
 
 hir_typeck_add_missing_parentheses_in_range = you must surround the range in parentheses to call its `{$func_name}` function
 
-hir_typeck_op_trait_generic_params =
-    `{$method_name}` must not have any generic parameters
-
 hir_typeck_lang_start_incorrect_number_params = incorrect number of parameters for the `start` lang item
 hir_typeck_lang_start_incorrect_number_params_note_expected_count = the `start` lang item should have four parameters, but found {$found_param_count}
 
@@ -57,3 +60,20 @@ hir_typeck_lang_start_incorrect_param = parameter {$param_num} of the `start` la
 
 hir_typeck_lang_start_incorrect_ret_ty = the return type of the `start` lang item is incorrect
     .suggestion = change the type from `{$found_ty}` to `{$expected_ty}`
+
+hir_typeck_help_set_edition_cargo = set `edition = "{$edition}"` in `Cargo.toml`
+hir_typeck_help_set_edition_standalone = pass `--edition {$edition}` to `rustc`
+hir_typeck_note_edition_guide = for more on editions, read https://doc.rust-lang.org/edition-guide
+
+hir_typeck_convert_to_str = try converting the passed type into a `&str`
+
+hir_typeck_op_trait_generic_params = `{$method_name}` must not have any generic parameters
+
+hir_typeck_fru_note = this expression may have been misinterpreted as a `..` range expression
+hir_typeck_fru_expr = this expression does not end in a comma...
+hir_typeck_fru_expr2 = ... so this is interpreted as a `..` range expression, instead of functional record update syntax
+hir_typeck_fru_suggestion =
+    to set the remaining fields{$expr ->
+        [NONE]{""}
+        *[other] {" "}from `{$expr}`
+    }, separate the last named field with a comma
diff --git a/compiler/rustc_hir_typeck/src/_match.rs b/compiler/rustc_hir_typeck/src/_match.rs
index b6f19d3cc..e19ef2ff3 100644
--- a/compiler/rustc_hir_typeck/src/_match.rs
+++ b/compiler/rustc_hir_typeck/src/_match.rs
@@ -41,7 +41,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // #55810: Type check patterns first so we get types for all bindings.
         let scrut_span = scrut.span.find_ancestor_inside(expr.span).unwrap_or(scrut.span);
         for arm in arms {
-            self.check_pat_top(&arm.pat, scrutinee_ty, Some(scrut_span), true);
+            self.check_pat_top(&arm.pat, scrutinee_ty, Some(scrut_span), Some(scrut));
         }
 
         // Now typecheck the blocks.
@@ -188,8 +188,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let hir = self.tcx.hir();
 
         // First, check that we're actually in the tail of a function.
-        let hir::Node::Expr(hir::Expr { kind: hir::ExprKind::Block(block, _), .. }) =
-            hir.get(self.body_id) else { return; };
+        let Some(body_id) = hir.maybe_body_owned_by(self.body_id) else { return; };
+        let body = hir.body(body_id);
+        let hir::ExprKind::Block(block, _) = body.value.kind else { return; };
         let Some(hir::Stmt { kind: hir::StmtKind::Semi(last_expr), .. })
             = block.innermost_block().stmts.last() else {  return; };
         if last_expr.hir_id != expr.hir_id {
@@ -198,7 +199,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         // Next, make sure that we have no type expectation.
         let Some(ret) = hir
-            .find_by_def_id(self.body_id.owner.def_id)
+            .find_by_def_id(self.body_id)
             .and_then(|owner| owner.fn_decl())
             .map(|decl| decl.output.span()) else { return; };
         let Expectation::IsLast(stmt) = expectation else {
diff --git a/compiler/rustc_hir_typeck/src/callee.rs b/compiler/rustc_hir_typeck/src/callee.rs
index b617821fb..6a0d5c011 100644
--- a/compiler/rustc_hir_typeck/src/callee.rs
+++ b/compiler/rustc_hir_typeck/src/callee.rs
@@ -21,7 +21,7 @@ use rustc_middle::ty::adjustment::{
     Adjust, Adjustment, AllowTwoPhase, AutoBorrow, AutoBorrowMutability,
 };
 use rustc_middle::ty::SubstsRef;
-use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitableExt};
 use rustc_span::def_id::LocalDefId;
 use rustc_span::symbol::{sym, Ident};
 use rustc_span::Span;
@@ -156,7 +156,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 // fnmut vs fnonce. If so, we have to defer further processing.
                 if self.closure_kind(substs).is_none() {
                     let closure_sig = substs.as_closure().sig();
-                    let closure_sig = self.replace_bound_vars_with_fresh_vars(
+                    let closure_sig = self.instantiate_binder_with_fresh_vars(
                         call_expr.span,
                         infer::FnCall,
                         closure_sig,
@@ -232,7 +232,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             let Some(trait_def_id) = opt_trait_def_id else { continue };
 
             let opt_input_type = opt_arg_exprs.map(|arg_exprs| {
-                self.tcx.mk_tup(arg_exprs.iter().map(|e| {
+                self.tcx.mk_tup_from_iter(arg_exprs.iter().map(|e| {
                     self.next_ty_var(TypeVariableOrigin {
                         kind: TypeVariableOriginKind::TypeInference,
                         span: e.span,
@@ -247,6 +247,15 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 adjusted_ty,
                 opt_input_type.as_ref().map(slice::from_ref),
             ) {
+                // Check for `self` receiver on the method, otherwise we can't use this as a `Fn*` trait.
+                if !self.tcx.associated_item(ok.value.def_id).fn_has_self_parameter {
+                    self.tcx.sess.delay_span_bug(
+                        call_expr.span,
+                        "input to overloaded call fn is not a self receiver",
+                    );
+                    return None;
+                }
+
                 let method = self.register_infer_ok_obligations(ok);
                 let mut autoref = None;
                 if borrow {
@@ -257,7 +266,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         // caused an error elsewhere.
                         self.tcx
                             .sess
-                            .delay_span_bug(call_expr.span, "input to call/call_mut is not a ref?");
+                            .delay_span_bug(call_expr.span, "input to call/call_mut is not a ref");
                         return None;
                     };
 
@@ -271,6 +280,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         target: method.sig.inputs()[0],
                     });
                 }
+
                 return Some((autoref, method));
             }
         }
@@ -367,7 +377,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     ) -> Ty<'tcx> {
         let (fn_sig, def_id) = match *callee_ty.kind() {
             ty::FnDef(def_id, subst) => {
-                let fn_sig = self.tcx.bound_fn_sig(def_id).subst(self.tcx, subst);
+                let fn_sig = self.tcx.fn_sig(def_id).subst(self.tcx, subst);
 
                 // Unit testing: function items annotated with
                 // `#[rustc_evaluate_where_clauses]` trigger special output
@@ -428,7 +438,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
                 let err = self.report_invalid_callee(call_expr, callee_expr, callee_ty, arg_exprs);
 
-                return self.tcx.ty_error_with_guaranteed(err);
+                return self.tcx.ty_error(err);
             }
         };
 
@@ -437,7 +447,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // renormalize the associated types at this point, since they
         // previously appeared within a `Binder<>` and hence would not
         // have been normalized before.
-        let fn_sig = self.replace_bound_vars_with_fresh_vars(call_expr.span, infer::FnCall, fn_sig);
+        let fn_sig = self.instantiate_binder_with_fresh_vars(call_expr.span, infer::FnCall, fn_sig);
         let fn_sig = self.normalize(call_expr.span, fn_sig);
 
         // Call the generic checker.
@@ -661,7 +671,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 && !self.type_is_sized_modulo_regions(self.param_env, output_ty, callee_expr.span)
             {
                 let descr = match maybe_def {
-                    DefIdOrName::DefId(def_id) => self.tcx.def_kind(def_id).descr(def_id),
+                    DefIdOrName::DefId(def_id) => self.tcx.def_descr(def_id),
                     DefIdOrName::Name(name) => name,
                 };
                 err.span_label(
@@ -823,7 +833,7 @@ impl<'a, 'tcx> DeferredCallResolution<'tcx> {
                 );
                 err.help(
                     "make sure the `fn`/`fn_mut`/`fn_once` lang items are defined \
-                     and have associated `call`/`call_mut`/`call_once` functions",
+                     and have correctly defined `call`/`call_mut`/`call_once` methods",
                 );
                 err.emit();
             }
diff --git a/compiler/rustc_hir_typeck/src/cast.rs b/compiler/rustc_hir_typeck/src/cast.rs
index 712f9b87a..316c2a7ee 100644
--- a/compiler/rustc_hir_typeck/src/cast.rs
+++ b/compiler/rustc_hir_typeck/src/cast.rs
@@ -41,7 +41,7 @@ use rustc_middle::mir::Mutability;
 use rustc_middle::ty::adjustment::AllowTwoPhase;
 use rustc_middle::ty::cast::{CastKind, CastTy};
 use rustc_middle::ty::error::TypeError;
-use rustc_middle::ty::{self, Ty, TypeAndMut, TypeVisitable, VariantDef};
+use rustc_middle::ty::{self, Ty, TypeAndMut, TypeVisitableExt, VariantDef};
 use rustc_session::lint;
 use rustc_session::Session;
 use rustc_span::def_id::{DefId, LOCAL_CRATE};
@@ -130,6 +130,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             | ty::Float(_)
             | ty::Array(..)
             | ty::GeneratorWitness(..)
+            | ty::GeneratorWitnessMIR(..)
             | ty::RawPtr(_)
             | ty::Ref(..)
             | ty::FnDef(..)
diff --git a/compiler/rustc_hir_typeck/src/check.rs b/compiler/rustc_hir_typeck/src/check.rs
index 57feefbca..bf8259ff7 100644
--- a/compiler/rustc_hir_typeck/src/check.rs
+++ b/compiler/rustc_hir_typeck/src/check.rs
@@ -43,7 +43,7 @@ pub(super) fn check_fn<'a, 'tcx>(
     let ret_ty =
         fcx.register_infer_ok_obligations(fcx.infcx.replace_opaque_types_with_inference_vars(
             declared_ret_ty,
-            body.value.hir_id,
+            fn_def_id,
             decl.output.span(),
             fcx.param_env,
         ));
@@ -74,15 +74,13 @@ pub(super) fn check_fn<'a, 'tcx>(
 
     // C-variadic fns also have a `VaList` input that's not listed in `fn_sig`
     // (as it's created inside the body itself, not passed in from outside).
-    let maybe_va_list = if fn_sig.c_variadic {
+    let maybe_va_list = fn_sig.c_variadic.then(|| {
         let span = body.params.last().unwrap().span;
         let va_list_did = tcx.require_lang_item(LangItem::VaList, Some(span));
         let region = fcx.next_region_var(RegionVariableOrigin::MiscVariable(span));
 
-        Some(tcx.bound_type_of(va_list_did).subst(tcx, &[region.into()]))
-    } else {
-        None
-    };
+        tcx.type_of(va_list_did).subst(tcx, &[region.into()])
+    });
 
     // Add formal parameters.
     let inputs_hir = hir.fn_decl_by_hir_id(fn_id).map(|decl| &decl.inputs);
@@ -90,7 +88,7 @@ pub(super) fn check_fn<'a, 'tcx>(
     for (idx, (param_ty, param)) in inputs_fn.chain(maybe_va_list).zip(body.params).enumerate() {
         // Check the pattern.
         let ty_span = try { inputs_hir?.get(idx)?.span };
-        fcx.check_pat_top(&param.pat, param_ty, ty_span, false);
+        fcx.check_pat_top(&param.pat, param_ty, ty_span, None);
 
         // Check that argument is Sized.
         // The check for a non-trivial pattern is a hack to avoid duplicate warnings
@@ -130,7 +128,12 @@ pub(super) fn check_fn<'a, 'tcx>(
     let gen_ty = if let (Some(_), Some(gen_kind)) = (can_be_generator, body.generator_kind) {
         let interior = fcx
             .next_ty_var(TypeVariableOrigin { kind: TypeVariableOriginKind::MiscVariable, span });
-        fcx.deferred_generator_interiors.borrow_mut().push((body.id(), interior, gen_kind));
+        fcx.deferred_generator_interiors.borrow_mut().push((
+            fn_def_id,
+            body.id(),
+            interior,
+            gen_kind,
+        ));
 
         let (resume_ty, yield_ty) = fcx.resume_yield_tys.unwrap();
         Some(GeneratorTypes {
@@ -167,12 +170,12 @@ pub(super) fn check_fn<'a, 'tcx>(
 
     // Check that a function marked as `#[panic_handler]` has signature `fn(&PanicInfo) -> !`
     if let Some(panic_impl_did) = tcx.lang_items().panic_impl()
-        && panic_impl_did == hir.local_def_id(fn_id).to_def_id()
+        && panic_impl_did == fn_def_id.to_def_id()
     {
         check_panic_info_fn(tcx, panic_impl_did.expect_local(), fn_sig, decl, declared_ret_ty);
     }
 
-    if let Some(lang_start_defid) = tcx.lang_items().start_fn() && lang_start_defid == hir.local_def_id(fn_id).to_def_id() {
+    if let Some(lang_start_defid) = tcx.lang_items().start_fn() && lang_start_defid == fn_def_id.to_def_id() {
         check_lang_start_fn(tcx, fn_sig, decl, fn_def_id);
     }
 
@@ -259,11 +262,9 @@ fn check_lang_start_fn<'tcx>(
         // for example `start`'s generic should be a type parameter
         let generics = tcx.generics_of(def_id);
         let fn_generic = generics.param_at(0, tcx);
-        let generic_tykind =
-            ty::Param(ty::ParamTy { index: fn_generic.index, name: fn_generic.name });
-        let generic_ty = tcx.mk_ty(generic_tykind);
+        let generic_ty = tcx.mk_ty_param(fn_generic.index, fn_generic.name);
         let expected_fn_sig =
-            tcx.mk_fn_sig([].iter(), &generic_ty, false, hir::Unsafety::Normal, Abi::Rust);
+            tcx.mk_fn_sig([], generic_ty, false, hir::Unsafety::Normal, Abi::Rust);
         let expected_ty = tcx.mk_fn_ptr(Binder::dummy(expected_fn_sig));
 
         // we emit the same error to suggest changing the arg no matter what's wrong with the arg
diff --git a/compiler/rustc_hir_typeck/src/closure.rs b/compiler/rustc_hir_typeck/src/closure.rs
index 12a2abfa7..d84fabb78 100644
--- a/compiler/rustc_hir_typeck/src/closure.rs
+++ b/compiler/rustc_hir_typeck/src/closure.rs
@@ -3,8 +3,8 @@
 use super::{check_fn, Expectation, FnCtxt, GeneratorTypes};
 
 use hir::def::DefKind;
+use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
-use rustc_hir::def_id::LocalDefId;
 use rustc_hir::lang_items::LangItem;
 use rustc_hir_analysis::astconv::AstConv;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
@@ -12,9 +12,11 @@ use rustc_infer::infer::LateBoundRegionConversionTime;
 use rustc_infer::infer::{InferOk, InferResult};
 use rustc_macros::{TypeFoldable, TypeVisitable};
 use rustc_middle::ty::subst::InternalSubsts;
-use rustc_middle::ty::visit::TypeVisitable;
-use rustc_middle::ty::{self, Ty, TypeSuperVisitable, TypeVisitor};
+use rustc_middle::ty::visit::{TypeVisitable, TypeVisitableExt};
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeSuperVisitable, TypeVisitor};
+use rustc_span::def_id::LocalDefId;
 use rustc_span::source_map::Span;
+use rustc_span::sym;
 use rustc_target::spec::abi::Abi;
 use rustc_trait_selection::traits;
 use rustc_trait_selection::traits::error_reporting::ArgKind;
@@ -80,7 +82,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         debug!(?bound_sig, ?liberated_sig);
 
-        let mut fcx = FnCtxt::new(self, self.param_env.without_const(), body.value.hir_id);
+        let mut fcx = FnCtxt::new(self, self.param_env.without_const(), closure.def_id);
         let generator_types = check_fn(
             &mut fcx,
             liberated_sig,
@@ -125,7 +127,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // the `closures` table.
         let sig = bound_sig.map_bound(|sig| {
             self.tcx.mk_fn_sig(
-                iter::once(self.tcx.intern_tup(sig.inputs())),
+                [self.tcx.mk_tup(sig.inputs())],
                 sig.output(),
                 sig.c_variadic,
                 sig.unsafety,
@@ -231,7 +233,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 struct MentionsTy<'tcx> {
                     expected_ty: Ty<'tcx>,
                 }
-                impl<'tcx> TypeVisitor<'tcx> for MentionsTy<'tcx> {
+                impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for MentionsTy<'tcx> {
                     type BreakTy = ();
 
                     fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
@@ -288,21 +290,20 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let trait_def_id = projection.trait_def_id(tcx);
 
         let is_fn = tcx.is_fn_trait(trait_def_id);
-        let gen_trait = tcx.require_lang_item(LangItem::Generator, cause_span);
-        let is_gen = gen_trait == trait_def_id;
+
+        let gen_trait = tcx.lang_items().gen_trait();
+        let is_gen = gen_trait == Some(trait_def_id);
+
         if !is_fn && !is_gen {
             debug!("not fn or generator");
             return None;
         }
 
-        if is_gen {
-            // Check that we deduce the signature from the `<_ as std::ops::Generator>::Return`
-            // associated item and not yield.
-            let return_assoc_item = self.tcx.associated_item_def_ids(gen_trait)[1];
-            if return_assoc_item != projection.projection_def_id() {
-                debug!("not return assoc item of generator");
-                return None;
-            }
+        // Check that we deduce the signature from the `<_ as std::ops::Generator>::Return`
+        // associated item and not yield.
+        if is_gen && self.tcx.associated_item(projection.projection_def_id()).name != sym::Return {
+            debug!("not `Return` assoc item of `Generator`");
+            return None;
         }
 
         let input_tys = if is_fn {
@@ -326,7 +327,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         debug!(?ret_param_ty);
 
         let sig = projection.rebind(self.tcx.mk_fn_sig(
-            input_tys.iter(),
+            input_tys,
             ret_param_ty,
             false,
             hir::Unsafety::Normal,
@@ -488,17 +489,18 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             };
         let expected_span =
             expected_sig.cause_span.unwrap_or_else(|| self.tcx.def_span(expr_def_id));
-        self.report_arg_count_mismatch(
-            expected_span,
-            closure_span,
-            expected_args,
-            found_args,
-            true,
-            closure_arg_span,
-        )
-        .emit();
-
-        let error_sig = self.error_sig_of_closure(decl);
+        let guar = self
+            .report_arg_count_mismatch(
+                expected_span,
+                closure_span,
+                expected_args,
+                found_args,
+                true,
+                closure_arg_span,
+            )
+            .emit();
+
+        let error_sig = self.error_sig_of_closure(decl, guar);
 
         self.closure_sigs(expr_def_id, body, error_sig)
     }
@@ -544,7 +546,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             )
             .map(|(hir_ty, &supplied_ty)| {
                 // Instantiate (this part of..) S to S', i.e., with fresh variables.
-                self.replace_bound_vars_with_fresh_vars(
+                self.instantiate_binder_with_fresh_vars(
                     hir_ty.span,
                     LateBoundRegionConversionTime::FnCall,
                     // (*) binder moved to here
@@ -561,12 +563,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             ) {
                 // Check that E' = S'.
                 let cause = self.misc(hir_ty.span);
-                let InferOk { value: (), obligations } =
-                    self.at(&cause, self.param_env).eq(*expected_ty, supplied_ty)?;
+                let InferOk { value: (), obligations } = self
+                    .at(&cause, self.param_env)
+                    .define_opaque_types(true)
+                    .eq(*expected_ty, supplied_ty)?;
                 all_obligations.extend(obligations);
             }
 
-            let supplied_output_ty = self.replace_bound_vars_with_fresh_vars(
+            let supplied_output_ty = self.instantiate_binder_with_fresh_vars(
                 decl.output.span(),
                 LateBoundRegionConversionTime::FnCall,
                 supplied_sig.output(),
@@ -574,6 +578,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             let cause = &self.misc(decl.output.span());
             let InferOk { value: (), obligations } = self
                 .at(cause, self.param_env)
+                .define_opaque_types(true)
                 .eq(expected_sigs.liberated_sig.output(), supplied_output_ty)?;
             all_obligations.extend(obligations);
 
@@ -620,8 +625,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 // function.
                 Some(hir::GeneratorKind::Async(hir::AsyncGeneratorKind::Fn)) => {
                     debug!("closure is async fn body");
-                    self.deduce_future_output_from_obligations(expr_def_id, body.id().hir_id)
-                        .unwrap_or_else(|| {
+                    let def_id = self.tcx.hir().body_owner_def_id(body.id());
+                    self.deduce_future_output_from_obligations(expr_def_id, def_id).unwrap_or_else(
+                        || {
                             // AFAIK, deducing the future output
                             // always succeeds *except* in error cases
                             // like #65159. I'd like to return Error
@@ -630,7 +636,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             // *have* reported an
                             // error. --nikomatsakis
                             astconv.ty_infer(None, decl.output.span())
-                        })
+                        },
+                    )
                 }
 
                 _ => astconv.ty_infer(None, decl.output.span()),
@@ -665,7 +672,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     fn deduce_future_output_from_obligations(
         &self,
         expr_def_id: LocalDefId,
-        body_id: hir::HirId,
+        body_def_id: LocalDefId,
     ) -> Option<Ty<'tcx>> {
         let ret_coercion = self.ret_coercion.as_ref().unwrap_or_else(|| {
             span_bug!(self.tcx.def_span(expr_def_id), "async fn generator outside of a fn")
@@ -725,7 +732,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let InferOk { value: output_ty, obligations } = self
             .replace_opaque_types_with_inference_vars(
                 output_ty,
-                body_id,
+                body_def_id,
                 self.tcx.def_span(expr_def_id),
                 self.param_env,
             );
@@ -787,13 +794,18 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     /// Converts the types that the user supplied, in case that doing
     /// so should yield an error, but returns back a signature where
     /// all parameters are of type `TyErr`.
-    fn error_sig_of_closure(&self, decl: &hir::FnDecl<'_>) -> ty::PolyFnSig<'tcx> {
+    fn error_sig_of_closure(
+        &self,
+        decl: &hir::FnDecl<'_>,
+        guar: ErrorGuaranteed,
+    ) -> ty::PolyFnSig<'tcx> {
         let astconv: &dyn AstConv<'_> = self;
+        let err_ty = self.tcx.ty_error(guar);
 
         let supplied_arguments = decl.inputs.iter().map(|a| {
             // Convert the types that the user supplied (if any), but ignore them.
             astconv.ast_ty_to_ty(a);
-            self.tcx.ty_error()
+            err_ty
         });
 
         if let hir::FnRetTy::Return(ref output) = decl.output {
@@ -802,7 +814,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let result = ty::Binder::dummy(self.tcx.mk_fn_sig(
             supplied_arguments,
-            self.tcx.ty_error(),
+            err_ty,
             decl.c_variadic,
             hir::Unsafety::Normal,
             Abi::RustCall,
diff --git a/compiler/rustc_hir_typeck/src/coercion.rs b/compiler/rustc_hir_typeck/src/coercion.rs
index bbf7b81a2..00b86890b 100644
--- a/compiler/rustc_hir_typeck/src/coercion.rs
+++ b/compiler/rustc_hir_typeck/src/coercion.rs
@@ -54,7 +54,7 @@ use rustc_middle::ty::adjustment::{
 use rustc_middle::ty::error::TypeError;
 use rustc_middle::ty::relate::RelateResult;
 use rustc_middle::ty::subst::SubstsRef;
-use rustc_middle::ty::visit::TypeVisitable;
+use rustc_middle::ty::visit::TypeVisitableExt;
 use rustc_middle::ty::{self, Ty, TypeAndMut};
 use rustc_session::parse::feature_err;
 use rustc_span::symbol::sym;
@@ -143,11 +143,11 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
     fn unify(&self, a: Ty<'tcx>, b: Ty<'tcx>) -> InferResult<'tcx, Ty<'tcx>> {
         debug!("unify(a: {:?}, b: {:?}, use_lub: {})", a, b, self.use_lub);
         self.commit_if_ok(|_| {
+            let at = self.at(&self.cause, self.fcx.param_env).define_opaque_types(true);
             if self.use_lub {
-                self.at(&self.cause, self.fcx.param_env).lub(b, a)
+                at.lub(b, a)
             } else {
-                self.at(&self.cause, self.fcx.param_env)
-                    .sup(b, a)
+                at.sup(b, a)
                     .map(|InferOk { value: (), obligations }| InferOk { value: a, obligations })
             }
         })
@@ -170,12 +170,14 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
         debug!("Coerce.tys({:?} => {:?})", a, b);
 
         // Just ignore error types.
-        if a.references_error() || b.references_error() {
+        if let Err(guar) = (a, b).error_reported() {
             // Best-effort try to unify these types -- we're already on the error path,
             // so this will have the side-effect of making sure we have no ambiguities
             // due to `[type error]` and `_` not coercing together.
-            let _ = self.commit_if_ok(|_| self.at(&self.cause, self.param_env).eq(a, b));
-            return success(vec![], self.fcx.tcx.ty_error(), vec![]);
+            let _ = self.commit_if_ok(|_| {
+                self.at(&self.cause, self.param_env).define_opaque_types(true).eq(a, b)
+            });
+            return success(vec![], self.fcx.tcx.ty_error(guar), vec![]);
         }
 
         // Coercing from `!` to any type is allowed:
@@ -765,7 +767,7 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
             self.cause.clone(),
             self.param_env,
             ty::Binder::dummy(
-                self.tcx.at(self.cause.span).mk_trait_ref(hir::LangItem::PointerSized, [a]),
+                self.tcx.at(self.cause.span).mk_trait_ref(hir::LangItem::PointerLike, [a]),
             ),
         ));
 
@@ -995,7 +997,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let (adjustments, _) = self.register_infer_ok_obligations(ok);
         self.apply_adjustments(expr, adjustments);
-        Ok(if expr_ty.references_error() { self.tcx.ty_error() } else { target })
+        Ok(if let Err(guar) = expr_ty.error_reported() { self.tcx.ty_error(guar) } else { target })
     }
 
     /// Same as `try_coerce()`, but without side-effects.
@@ -1046,7 +1048,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     self.param_env,
                 )
                 .may_apply()
-                .then(|| deref_ty)
+                .then_some(deref_ty)
         })
     }
 
@@ -1432,8 +1434,8 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
 
         // If we see any error types, just propagate that error
         // upwards.
-        if expression_ty.references_error() || self.merged_ty().references_error() {
-            self.final_ty = Some(fcx.tcx.ty_error());
+        if let Err(guar) = (expression_ty, self.merged_ty()).error_reported() {
+            self.final_ty = Some(fcx.tcx.ty_error(guar));
             return;
         }
 
@@ -1484,6 +1486,8 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
             // Another example is `break` with no argument expression.
             assert!(expression_ty.is_unit(), "if let hack without unit type");
             fcx.at(cause, fcx.param_env)
+                // needed for tests/ui/type-alias-impl-trait/issue-65679-inst-opaque-ty-from-val-twice.rs
+                .define_opaque_types(true)
                 .eq_exp(label_expression_as_expected, expression_ty, self.merged_ty())
                 .map(|infer_ok| {
                     fcx.register_infer_ok_obligations(infer_ok);
@@ -1613,12 +1617,14 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
                 if visitor.ret_exprs.len() > 0 && let Some(expr) = expression {
                     self.note_unreachable_loop_return(&mut err, &expr, &visitor.ret_exprs);
                 }
+
                 let reported = err.emit_unless(unsized_return);
 
-                self.final_ty = Some(fcx.tcx.ty_error_with_guaranteed(reported));
+                self.final_ty = Some(fcx.tcx.ty_error(reported));
             }
         }
     }
+
     fn note_unreachable_loop_return(
         &self,
         err: &mut Diagnostic,
@@ -1821,7 +1827,7 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
                         .trait_ref()
                         .and_then(|t| t.trait_def_id())
                         .map_or(false, |def_id| {
-                            fcx.tcx.object_safety_violations(def_id).is_empty()
+                            fcx.tcx.check_is_object_safe(def_id)
                         })
                 })
             }
diff --git a/compiler/rustc_hir_typeck/src/demand.rs b/compiler/rustc_hir_typeck/src/demand.rs
index f4c4d4310..7ba57b3b7 100644
--- a/compiler/rustc_hir_typeck/src/demand.rs
+++ b/compiler/rustc_hir_typeck/src/demand.rs
@@ -1,9 +1,11 @@
 use crate::FnCtxt;
 use rustc_ast::util::parser::PREC_POSTFIX;
+use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::MultiSpan;
 use rustc_errors::{Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed};
 use rustc_hir as hir;
 use rustc_hir::def::CtorKind;
+use rustc_hir::intravisit::Visitor;
 use rustc_hir::lang_items::LangItem;
 use rustc_hir::{is_range_literal, Node};
 use rustc_infer::infer::InferOk;
@@ -11,11 +13,14 @@ use rustc_middle::lint::in_external_macro;
 use rustc_middle::middle::stability::EvalResult;
 use rustc_middle::ty::adjustment::AllowTwoPhase;
 use rustc_middle::ty::error::{ExpectedFound, TypeError};
-use rustc_middle::ty::print::with_no_trimmed_paths;
-use rustc_middle::ty::{self, Article, AssocItem, Ty, TypeAndMut};
+use rustc_middle::ty::fold::{BottomUpFolder, TypeFolder};
+use rustc_middle::ty::print::{with_forced_trimmed_paths, with_no_trimmed_paths};
+use rustc_middle::ty::relate::TypeRelation;
+use rustc_middle::ty::{self, Article, AssocItem, Ty, TypeAndMut, TypeVisitableExt};
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::{BytePos, Span};
 use rustc_trait_selection::infer::InferCtxtExt as _;
+use rustc_trait_selection::traits::error_reporting::method_chain::CollectAllMismatches;
 use rustc_trait_selection::traits::ObligationCause;
 
 use super::method::probe;
@@ -40,7 +45,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         self.annotate_alternative_method_deref(err, expr, error);
 
         // Use `||` to give these suggestions a precedence
-        let _ = self.suggest_missing_parentheses(err, expr)
+        let suggested = self.suggest_missing_parentheses(err, expr)
             || self.suggest_remove_last_method_call(err, expr, expected)
             || self.suggest_associated_const(err, expr, expected)
             || self.suggest_deref_ref_or_into(err, expr, expected, expr_ty, expected_ty_expr)
@@ -54,7 +59,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             || self.suggest_copied_or_cloned(err, expr, expr_ty, expected)
             || self.suggest_clone_for_ref(err, expr, expr_ty, expected)
             || self.suggest_into(err, expr, expr_ty, expected)
-            || self.suggest_floating_point_literal(err, expr, expected);
+            || self.suggest_floating_point_literal(err, expr, expected)
+            || self.suggest_null_ptr_for_literal_zero_given_to_ptr_arg(err, expr, expected)
+            || self.note_result_coercion(err, expr, expected, expr_ty);
+        if !suggested {
+            self.point_at_expr_source_of_inferred_type(err, expr, expr_ty, expected, expr.span);
+        }
     }
 
     pub fn emit_coerce_suggestions(
@@ -73,7 +83,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         self.annotate_expected_due_to_let_ty(err, expr, error);
         self.emit_type_mismatch_suggestions(err, expr, expr_ty, expected, expected_ty_expr, error);
         self.note_type_is_not_clone(err, expected, expr_ty, expr);
-        self.note_need_for_fn_pointer(err, expected, expr_ty);
         self.note_internal_mutation_in_method(err, expr, expected, expr_ty);
         self.check_for_range_as_method_call(err, expr, expr_ty, expected);
         self.check_for_binding_assigned_block_without_tail_expression(err, expr, expr_ty, expected);
@@ -104,7 +113,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         expected: Ty<'tcx>,
         actual: Ty<'tcx>,
     ) -> Option<DiagnosticBuilder<'tcx, ErrorGuaranteed>> {
-        match self.at(cause, self.param_env).sup(expected, actual) {
+        match self.at(cause, self.param_env).define_opaque_types(true).sup(expected, actual) {
             Ok(InferOk { obligations, value: () }) => {
                 self.register_predicates(obligations);
                 None
@@ -134,7 +143,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         expected: Ty<'tcx>,
         actual: Ty<'tcx>,
     ) -> Option<DiagnosticBuilder<'tcx, ErrorGuaranteed>> {
-        match self.at(cause, self.param_env).eq(expected, actual) {
+        match self.at(cause, self.param_env).define_opaque_types(true).eq(expected, actual) {
             Ok(InferOk { obligations, value: () }) => {
                 self.register_predicates(obligations);
                 None
@@ -208,6 +217,223 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         (expected, Some(err))
     }
 
+    pub fn point_at_expr_source_of_inferred_type(
+        &self,
+        err: &mut Diagnostic,
+        expr: &hir::Expr<'_>,
+        found: Ty<'tcx>,
+        expected: Ty<'tcx>,
+        mismatch_span: Span,
+    ) -> bool {
+        let map = self.tcx.hir();
+
+        let hir::ExprKind::Path(hir::QPath::Resolved(None, p)) = expr.kind else { return false; };
+        let [hir::PathSegment { ident, args: None, .. }] = p.segments else { return false; };
+        let hir::def::Res::Local(hir_id) = p.res else { return false; };
+        let Some(hir::Node::Pat(pat)) = map.find(hir_id) else { return false; };
+        let Some(hir::Node::Local(hir::Local {
+            ty: None,
+            init: Some(init),
+            ..
+        })) = map.find_parent(pat.hir_id) else { return false; };
+        let Some(ty) = self.node_ty_opt(init.hir_id) else { return false; };
+        if ty.is_closure() || init.span.overlaps(expr.span) || pat.span.from_expansion() {
+            return false;
+        }
+
+        // Locate all the usages of the relevant binding.
+        struct FindExprs<'hir> {
+            hir_id: hir::HirId,
+            uses: Vec<&'hir hir::Expr<'hir>>,
+        }
+        impl<'v> Visitor<'v> for FindExprs<'v> {
+            fn visit_expr(&mut self, ex: &'v hir::Expr<'v>) {
+                if let hir::ExprKind::Path(hir::QPath::Resolved(None, path)) = ex.kind
+                    && let hir::def::Res::Local(hir_id) = path.res
+                    && hir_id == self.hir_id
+                {
+                    self.uses.push(ex);
+                }
+                hir::intravisit::walk_expr(self, ex);
+            }
+        }
+
+        let mut expr_finder = FindExprs { hir_id, uses: vec![] };
+        let id = map.get_parent_item(hir_id);
+        let hir_id: hir::HirId = id.into();
+
+        let Some(node) = map.find(hir_id) else { return false; };
+        let Some(body_id) = node.body_id() else { return false; };
+        let body = map.body(body_id);
+        expr_finder.visit_expr(body.value);
+        // Hack to make equality checks on types with inference variables and regions useful.
+        let mut eraser = BottomUpFolder {
+            tcx: self.tcx,
+            lt_op: |_| self.tcx.lifetimes.re_erased,
+            ct_op: |c| c,
+            ty_op: |t| match *t.kind() {
+                ty::Infer(ty::TyVar(_)) => self.tcx.mk_ty_var(ty::TyVid::from_u32(0)),
+                ty::Infer(ty::IntVar(_)) => self.tcx.mk_int_var(ty::IntVid { index: 0 }),
+                ty::Infer(ty::FloatVar(_)) => self.tcx.mk_float_var(ty::FloatVid { index: 0 }),
+                _ => t,
+            },
+        };
+        let mut prev = eraser.fold_ty(ty);
+        let mut prev_span: Option<Span> = None;
+
+        for binding in expr_finder.uses {
+            // In every expression where the binding is referenced, we will look at that
+            // expression's type and see if it is where the incorrect found type was fully
+            // "materialized" and point at it. We will also try to provide a suggestion there.
+            if let Some(hir::Node::Expr(expr)
+            | hir::Node::Stmt(hir::Stmt {
+                kind: hir::StmtKind::Expr(expr) | hir::StmtKind::Semi(expr),
+                ..
+            })) = &map.find_parent(binding.hir_id)
+                && let hir::ExprKind::MethodCall(segment, rcvr, args, _span) = expr.kind
+                && rcvr.hir_id == binding.hir_id
+                && let Some(def_id) = self.typeck_results.borrow().type_dependent_def_id(expr.hir_id)
+            {
+                // We special case methods, because they can influence inference through the
+                // call's arguments and we can provide a more explicit span.
+                let sig = self.tcx.fn_sig(def_id).subst_identity();
+                let def_self_ty = sig.input(0).skip_binder();
+                let param_tys = sig.inputs().skip_binder().iter().skip(1);
+                // If there's an arity mismatch, pointing out the call as the source of an inference
+                // can be misleading, so we skip it.
+                if param_tys.len() != args.len() {
+                    continue;
+                }
+                let rcvr_ty = self.node_ty(rcvr.hir_id);
+                // Get the evaluated type *after* calling the method call, so that the influence
+                // of the arguments can be reflected in the receiver type. The receiver
+                // expression has the type *before* theis analysis is done.
+                let ty = match self.lookup_probe_for_diagnostic(
+                    segment.ident,
+                    rcvr_ty,
+                    expr,
+                    probe::ProbeScope::TraitsInScope,
+                    None,
+                ) {
+                    Ok(pick) => eraser.fold_ty(pick.self_ty),
+                    Err(_) => rcvr_ty,
+                };
+                // Remove one layer of references to account for `&mut self` and
+                // `&self`, so that we can compare it against the binding.
+                let (ty, def_self_ty) = match (ty.kind(), def_self_ty.kind()) {
+                    (ty::Ref(_, ty, a), ty::Ref(_, self_ty, b)) if a == b => (*ty, *self_ty),
+                    _ => (ty, def_self_ty),
+                };
+                let mut param_args = FxHashMap::default();
+                let mut param_expected = FxHashMap::default();
+                let mut param_found = FxHashMap::default();
+                if self.can_eq(self.param_env, ty, found) {
+                    // We only point at the first place where the found type was inferred.
+                    for (param_ty, arg) in param_tys.zip(args) {
+                        if def_self_ty.contains(*param_ty) && let ty::Param(_) = param_ty.kind() {
+                            // We found an argument that references a type parameter in `Self`,
+                            // so we assume that this is the argument that caused the found
+                            // type, which we know already because of `can_eq` above was first
+                            // inferred in this method call.
+                            let arg_ty = self.node_ty(arg.hir_id);
+                            if !arg.span.overlaps(mismatch_span) {
+                                err.span_label(
+                                    arg.span,
+                                    &format!(
+                                        "this is of type `{arg_ty}`, which causes `{ident}` to be \
+                                        inferred as `{ty}`",
+                                    ),
+                                );
+                            }
+                            param_args.insert(param_ty, (arg, arg_ty));
+                        }
+                    }
+                }
+
+                // Here we find, for a type param `T`, the type that `T` is in the current
+                // method call *and* in the original expected type. That way, we can see if we
+                // can give any structured suggestion for the function argument.
+                let mut c = CollectAllMismatches {
+                    infcx: &self.infcx,
+                    param_env: self.param_env,
+                    errors: vec![],
+                };
+                let _ = c.relate(def_self_ty, ty);
+                for error in c.errors {
+                    if let TypeError::Sorts(error) = error {
+                        param_found.insert(error.expected, error.found);
+                    }
+                }
+                c.errors = vec![];
+                let _ = c.relate(def_self_ty, expected);
+                for error in c.errors {
+                    if let TypeError::Sorts(error) = error {
+                        param_expected.insert(error.expected, error.found);
+                    }
+                }
+                for (param, (arg, arg_ty)) in param_args.iter() {
+                    let Some(expected) = param_expected.get(param) else { continue; };
+                    let Some(found) = param_found.get(param) else { continue; };
+                    if !self.can_eq(self.param_env, *arg_ty, *found) { continue; }
+                    self.emit_coerce_suggestions(err, arg, *found, *expected, None, None);
+                }
+
+                let ty = eraser.fold_ty(ty);
+                if ty.references_error() {
+                    break;
+                }
+                if ty != prev
+                    && param_args.is_empty()
+                    && self.can_eq(self.param_env, ty, found)
+                {
+                    // We only point at the first place where the found type was inferred.
+                    if !segment.ident.span.overlaps(mismatch_span) {
+                    err.span_label(
+                        segment.ident.span,
+                        with_forced_trimmed_paths!(format!(
+                            "here the type of `{ident}` is inferred to be `{ty}`",
+                        )),
+                    );}
+                    break;
+                } else if !param_args.is_empty() {
+                    break;
+                }
+                prev = ty;
+            } else {
+                let ty = eraser.fold_ty(self.node_ty(binding.hir_id));
+                if ty.references_error() {
+                    break;
+                }
+                if ty != prev
+                    && let Some(span) = prev_span
+                    && self.can_eq(self.param_env, ty, found)
+                {
+                    // We only point at the first place where the found type was inferred.
+                    // We use the *previous* span because if the type is known *here* it means
+                    // it was *evaluated earlier*. We don't do this for method calls because we
+                    // evaluate the method's self type eagerly, but not in any other case.
+                    if !span.overlaps(mismatch_span) {
+                        err.span_label(
+                            span,
+                            with_forced_trimmed_paths!(format!(
+                                "here the type of `{ident}` is inferred to be `{ty}`",
+                            )),
+                        );
+                    }
+                    break;
+                }
+                prev = ty;
+            }
+            if binding.hir_id == expr.hir_id {
+                // Do not look at expressions that come after the expression we were originally
+                // evaluating and had a type error.
+                break;
+            }
+            prev_span = Some(binding.span);
+        }
+        true
+    }
+
     fn annotate_expected_due_to_let_ty(
         &self,
         err: &mut Diagnostic,
@@ -379,6 +605,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 let substs = ty::InternalSubsts::for_item(self.tcx, m.def_id, |param, _| {
                     self.var_for_def(deref.span, param)
                 });
+                let mutability =
+                    match self.tcx.fn_sig(m.def_id).skip_binder().input(0).skip_binder().kind() {
+                        ty::Ref(_, _, hir::Mutability::Mut) => "&mut ",
+                        ty::Ref(_, _, _) => "&",
+                        _ => "",
+                    };
                 vec![
                     (
                         deref.span.until(base.span),
@@ -387,11 +619,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             with_no_trimmed_paths!(
                                 self.tcx.def_path_str_with_substs(m.def_id, substs,)
                             ),
-                            match self.tcx.fn_sig(m.def_id).input(0).skip_binder().kind() {
-                                ty::Ref(_, _, hir::Mutability::Mut) => "&mut ",
-                                ty::Ref(_, _, _) => "&",
-                                _ => "",
-                            },
+                            mutability,
                         ),
                     ),
                     match &args[..] {
@@ -479,6 +707,56 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         );
     }
 
+    pub(crate) fn note_result_coercion(
+        &self,
+        err: &mut Diagnostic,
+        expr: &hir::Expr<'tcx>,
+        expected: Ty<'tcx>,
+        found: Ty<'tcx>,
+    ) -> bool {
+        let ty::Adt(e, substs_e) = expected.kind() else { return false; };
+        let ty::Adt(f, substs_f) = found.kind() else { return false; };
+        if e.did() != f.did() {
+            return false;
+        }
+        if Some(e.did()) != self.tcx.get_diagnostic_item(sym::Result) {
+            return false;
+        }
+        let map = self.tcx.hir();
+        if let Some(hir::Node::Expr(expr)) = map.find_parent(expr.hir_id)
+            && let hir::ExprKind::Ret(_) = expr.kind
+        {
+            // `return foo;`
+        } else if map.get_return_block(expr.hir_id).is_some() {
+            // Function's tail expression.
+        } else {
+            return false;
+        }
+        let e = substs_e.type_at(1);
+        let f = substs_f.type_at(1);
+        if self
+            .infcx
+            .type_implements_trait(
+                self.tcx.get_diagnostic_item(sym::Into).unwrap(),
+                [f, e],
+                self.param_env,
+            )
+            .must_apply_modulo_regions()
+        {
+            err.multipart_suggestion(
+                "use `?` to coerce and return an appropriate `Err`, and wrap the resulting value \
+                 in `Ok` so the expression remains of type `Result`",
+                vec![
+                    (expr.span.shrink_to_lo(), "Ok(".to_string()),
+                    (expr.span.shrink_to_hi(), "?)".to_string()),
+                ],
+                Applicability::MaybeIncorrect,
+            );
+            return true;
+        }
+        false
+    }
+
     /// If the expected type is an enum (Issue #55250) with any variants whose
     /// sole field is of the found type, suggest such variants. (Issue #42764)
     fn suggest_compatible_variants(
@@ -491,7 +769,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         if let ty::Adt(expected_adt, substs) = expected.kind() {
             if let hir::ExprKind::Field(base, ident) = expr.kind {
                 let base_ty = self.typeck_results.borrow().expr_ty(base);
-                if self.can_eq(self.param_env, base_ty, expected).is_ok()
+                if self.can_eq(self.param_env, base_ty, expected)
                     && let Some(base_span) = base.span.find_ancestor_inside(expr.span)
                 {
                     err.span_suggestion_verbose(
@@ -762,7 +1040,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         match method.kind {
             ty::AssocKind::Fn => {
                 method.fn_has_self_parameter
-                    && self.tcx.fn_sig(method.def_id).inputs().skip_binder().len() == 1
+                    && self.tcx.fn_sig(method.def_id).skip_binder().inputs().skip_binder().len()
+                        == 1
             }
             _ => false,
         }
@@ -990,10 +1269,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 // ```
                 let ref_ty = match mutability {
                     hir::Mutability::Mut => {
-                        self.tcx.mk_mut_ref(self.tcx.mk_region(ty::ReStatic), checked_ty)
+                        self.tcx.mk_mut_ref(self.tcx.lifetimes.re_static, checked_ty)
                     }
                     hir::Mutability::Not => {
-                        self.tcx.mk_imm_ref(self.tcx.mk_region(ty::ReStatic), checked_ty)
+                        self.tcx.mk_imm_ref(self.tcx.lifetimes.re_static, checked_ty)
                     }
                 };
                 if self.can_coerce(ref_ty, expected) {
@@ -1015,6 +1294,22 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             sugg_sp = receiver.span;
                         }
                     }
+
+                    if let hir::ExprKind::Unary(hir::UnOp::Deref, ref inner) = expr.kind
+                        && let Some(1) = self.deref_steps(expected, checked_ty) {
+                        // We have `*&T`, check if what was expected was `&T`.
+                        // If so, we may want to suggest removing a `*`.
+                        sugg_sp = sugg_sp.with_hi(inner.span.lo());
+                        return Some((
+                            sugg_sp,
+                            "consider removing deref here".to_string(),
+                            "".to_string(),
+                            Applicability::MachineApplicable,
+                            true,
+                            false,
+                        ));
+                    }
+
                     if let Ok(src) = sm.span_to_snippet(sugg_sp) {
                         let needs_parens = match expr.kind {
                             // parenthesize if needed (Issue #46756)
@@ -1067,7 +1362,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 hir::ExprKind::AddrOf(hir::BorrowKind::Ref, _, ref expr),
                 _,
                 &ty::Ref(_, checked, _),
-            ) if self.can_sub(self.param_env, checked, expected).is_ok() => {
+            ) if self.can_sub(self.param_env, checked, expected) => {
                 // We have `&T`, check if what was expected was `T`. If so,
                 // we may want to suggest removing a `&`.
                 if sm.is_imported(expr.span) {
@@ -1713,7 +2008,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         };
         let hir::StmtKind::Semi(tail_expr) = stmt.kind else { return; };
         let Some(ty) = self.node_ty_opt(tail_expr.hir_id) else { return; };
-        if self.can_eq(self.param_env, expected_ty, ty).is_ok() {
+        if self.can_eq(self.param_env, expected_ty, ty) {
             err.span_suggestion_short(
                 stmt.span.with_lo(tail_expr.span.hi()),
                 "remove this semicolon",
@@ -1742,7 +2037,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                                    args: &[hir::Expr<'_>],
                                    kind: CallableKind| {
             let arg_idx = args.iter().position(|a| a.hir_id == expr.hir_id).unwrap();
-            let fn_ty = self.tcx.bound_type_of(def_id).0;
+            let fn_ty = self.tcx.type_of(def_id).skip_binder();
             if !fn_ty.is_fn() {
                 return;
             }
diff --git a/compiler/rustc_hir_typeck/src/errors.rs b/compiler/rustc_hir_typeck/src/errors.rs
index 5b4fd5e4a..3eee2278d 100644
--- a/compiler/rustc_hir_typeck/src/errors.rs
+++ b/compiler/rustc_hir_typeck/src/errors.rs
@@ -1,8 +1,13 @@
 //! Errors emitted by `rustc_hir_typeck`.
+use crate::fluent_generated as fluent;
 use rustc_errors::{AddToDiagnostic, Applicability, Diagnostic, MultiSpan, SubdiagnosticMessage};
 use rustc_macros::{Diagnostic, Subdiagnostic};
 use rustc_middle::ty::Ty;
-use rustc_span::{symbol::Ident, Span};
+use rustc_span::{
+    edition::{Edition, LATEST_STABLE_EDITION},
+    symbol::Ident,
+    Span,
+};
 
 #[derive(Diagnostic)]
 #[diag(hir_typeck_field_multiply_specified_in_initializer, code = "E0062")]
@@ -10,7 +15,7 @@ pub struct FieldMultiplySpecifiedInInitializer {
     #[primary_span]
     #[label]
     pub span: Span,
-    #[label(previous_use_label)]
+    #[label(hir_typeck_previous_use_label)]
     pub prev_span: Span,
     pub ident: Ident,
 }
@@ -20,9 +25,9 @@ pub struct FieldMultiplySpecifiedInInitializer {
 pub struct ReturnStmtOutsideOfFnBody {
     #[primary_span]
     pub span: Span,
-    #[label(encl_body_label)]
+    #[label(hir_typeck_encl_body_label)]
     pub encl_body_span: Option<Span>,
-    #[label(encl_fn_label)]
+    #[label(hir_typeck_encl_fn_label)]
     pub encl_fn_span: Option<Span>,
 }
 
@@ -153,20 +158,17 @@ impl AddToDiagnostic for TypeMismatchFruTypo {
 
         // Only explain that `a ..b` is a range if it's split up
         if self.expr_span.between(self.fru_span).is_empty() {
-            diag.span_note(
-                self.expr_span.to(self.fru_span),
-                rustc_errors::fluent::hir_typeck_fru_note,
-            );
+            diag.span_note(self.expr_span.to(self.fru_span), fluent::hir_typeck_fru_note);
         } else {
             let mut multispan: MultiSpan = vec![self.expr_span, self.fru_span].into();
-            multispan.push_span_label(self.expr_span, rustc_errors::fluent::hir_typeck_fru_expr);
-            multispan.push_span_label(self.fru_span, rustc_errors::fluent::hir_typeck_fru_expr2);
-            diag.span_note(multispan, rustc_errors::fluent::hir_typeck_fru_note);
+            multispan.push_span_label(self.expr_span, fluent::hir_typeck_fru_expr);
+            multispan.push_span_label(self.fru_span, fluent::hir_typeck_fru_expr2);
+            diag.span_note(multispan, fluent::hir_typeck_fru_note);
         }
 
         diag.span_suggestion(
             self.expr_span.shrink_to_hi(),
-            rustc_errors::fluent::hir_typeck_fru_suggestion,
+            fluent::hir_typeck_fru_suggestion,
             ", ",
             Applicability::MaybeIncorrect,
         );
@@ -205,3 +207,24 @@ pub struct LangStartIncorrectRetTy<'tcx> {
     pub expected_ty: Ty<'tcx>,
     pub found_ty: Ty<'tcx>,
 }
+
+#[derive(Subdiagnostic)]
+pub enum HelpUseLatestEdition {
+    #[help(hir_typeck_help_set_edition_cargo)]
+    #[note(hir_typeck_note_edition_guide)]
+    Cargo { edition: Edition },
+    #[help(hir_typeck_help_set_edition_standalone)]
+    #[note(hir_typeck_note_edition_guide)]
+    Standalone { edition: Edition },
+}
+
+impl HelpUseLatestEdition {
+    pub fn new() -> Self {
+        let edition = LATEST_STABLE_EDITION;
+        if std::env::var_os("CARGO").is_some() {
+            Self::Cargo { edition }
+        } else {
+            Self::Standalone { edition }
+        }
+    }
+}
diff --git a/compiler/rustc_hir_typeck/src/expr.rs b/compiler/rustc_hir_typeck/src/expr.rs
index bc7474cdf..7fc4ccb04 100644
--- a/compiler/rustc_hir_typeck/src/expr.rs
+++ b/compiler/rustc_hir_typeck/src/expr.rs
@@ -8,7 +8,7 @@ use crate::coercion::DynamicCoerceMany;
 use crate::errors::TypeMismatchFruTypo;
 use crate::errors::{AddressOfTemporaryTaken, ReturnStmtOutsideOfFnBody, StructExprNonExhaustive};
 use crate::errors::{
-    FieldMultiplySpecifiedInInitializer, FunctionalRecordUpdateOnNonStruct,
+    FieldMultiplySpecifiedInInitializer, FunctionalRecordUpdateOnNonStruct, HelpUseLatestEdition,
     YieldExprOutsideOfGenerator,
 };
 use crate::fatally_break_rust;
@@ -23,8 +23,8 @@ use rustc_ast as ast;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_errors::{
-    pluralize, struct_span_err, Applicability, Diagnostic, DiagnosticBuilder, DiagnosticId,
-    ErrorGuaranteed, StashKey,
+    pluralize, struct_span_err, AddToDiagnostic, Applicability, Diagnostic, DiagnosticBuilder,
+    DiagnosticId, ErrorGuaranteed, StashKey,
 };
 use rustc_hir as hir;
 use rustc_hir::def::{CtorKind, DefKind, Res};
@@ -42,11 +42,11 @@ use rustc_middle::middle::stability;
 use rustc_middle::ty::adjustment::{Adjust, Adjustment, AllowTwoPhase};
 use rustc_middle::ty::error::TypeError::FieldMisMatch;
 use rustc_middle::ty::subst::SubstsRef;
-use rustc_middle::ty::{self, AdtKind, Ty, TypeVisitable};
+use rustc_middle::ty::{self, AdtKind, Ty, TypeVisitableExt};
 use rustc_session::errors::ExprParenthesesNeeded;
 use rustc_session::parse::feature_err;
+use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::hygiene::DesugaringKind;
-use rustc_span::lev_distance::find_best_match_for_name;
 use rustc_span::source_map::{Span, Spanned};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_target::spec::abi::Abi::RustIntrinsic;
@@ -88,7 +88,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 return if let [Adjustment { kind: Adjust::NeverToAny, target }] = &adjustments[..] {
                     target.to_owned()
                 } else {
-                    self.tcx().ty_error_with_guaranteed(reported)
+                    self.tcx().ty_error(reported)
                 };
             }
 
@@ -313,7 +313,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     tcx.types.never
                 } else {
                     // There was an error; make type-check fail.
-                    tcx.ty_error()
+                    tcx.ty_error_misc()
                 }
             }
             ExprKind::Ret(ref expr_opt) => self.check_expr_return(expr_opt.as_deref(), expr),
@@ -354,7 +354,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             ExprKind::Field(base, field) => self.check_field(expr, &base, field, expected),
             ExprKind::Index(base, idx) => self.check_expr_index(base, idx, expr),
             ExprKind::Yield(value, ref src) => self.check_expr_yield(value, expr, src),
-            hir::ExprKind::Err => tcx.ty_error(),
+            hir::ExprKind::Err(guar) => tcx.ty_error(guar),
         }
     }
 
@@ -402,7 +402,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         {
                             err.subdiagnostic(ExprParenthesesNeeded::surrounding(*sp));
                         }
-                        oprnd_t = tcx.ty_error_with_guaranteed(err.emit());
+                        oprnd_t = tcx.ty_error(err.emit());
                     }
                 }
                 hir::UnOp::Not => {
@@ -452,7 +452,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let tm = ty::TypeAndMut { ty, mutbl };
         match kind {
-            _ if tm.ty.references_error() => self.tcx.ty_error(),
+            _ if tm.ty.references_error() => self.tcx.ty_error_misc(),
             hir::BorrowKind::Raw => {
                 self.check_named_place_expr(oprnd);
                 self.tcx.mk_ptr(tm)
@@ -531,18 +531,20 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 let e =
                     self.tcx.sess.delay_span_bug(qpath.span(), "`Res::Err` but no error emitted");
                 self.set_tainted_by_errors(e);
-                tcx.ty_error_with_guaranteed(e)
+                tcx.ty_error(e)
             }
             Res::Def(DefKind::Variant, _) => {
                 let e = report_unexpected_variant_res(tcx, res, qpath, expr.span, "E0533", "value");
-                tcx.ty_error_with_guaranteed(e)
+                tcx.ty_error(e)
             }
             _ => self.instantiate_value_path(segs, opt_ty, res, expr.span, expr.hir_id).0,
         };
 
         if let ty::FnDef(did, ..) = *ty.kind() {
             let fn_sig = ty.fn_sig(tcx);
-            if tcx.fn_sig(did).abi() == RustIntrinsic && tcx.item_name(did) == sym::transmute {
+            if tcx.fn_sig(did).skip_binder().abi() == RustIntrinsic
+                && tcx.item_name(did) == sym::transmute
+            {
                 let from = fn_sig.inputs().skip_binder()[0];
                 let to = fn_sig.output().skip_binder();
                 // We defer the transmute to the end of typeck, once all inference vars have
@@ -566,7 +568,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     // placeholder lifetimes with probing, we just replace higher lifetimes
                     // with fresh vars.
                     let span = args.get(i).map(|a| a.span).unwrap_or(expr.span);
-                    let input = self.replace_bound_vars_with_fresh_vars(
+                    let input = self.instantiate_binder_with_fresh_vars(
                         span,
                         infer::LateBoundRegionConversionTime::FnCall,
                         fn_sig.input(i),
@@ -584,7 +586,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // Also, as we just want to check sizedness, instead of introducing
             // placeholder lifetimes with probing, we just replace higher lifetimes
             // with fresh vars.
-            let output = self.replace_bound_vars_with_fresh_vars(
+            let output = self.instantiate_binder_with_fresh_vars(
                 expr.span,
                 infer::LateBoundRegionConversionTime::FnCall,
                 fn_sig.output(),
@@ -632,7 +634,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 // If the loop context is not a `loop { }`, then break with
                 // a value is illegal, and `opt_coerce_to` will be `None`.
                 // Just set expectation to error in that case.
-                let coerce_to = opt_coerce_to.unwrap_or_else(|| tcx.ty_error());
+                let coerce_to = opt_coerce_to.unwrap_or_else(|| tcx.ty_error_misc());
 
                 // Recurse without `enclosing_breakables` borrowed.
                 e_ty = self.check_expr_with_hint(e, coerce_to);
@@ -852,7 +854,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // Point any obligations that were registered due to opaque type
             // inference at the return expression.
             self.select_obligations_where_possible(|errors| {
-                self.point_at_return_for_opaque_ty_error(errors, span, return_expr_ty);
+                self.point_at_return_for_opaque_ty_error(errors, span, return_expr_ty, return_expr.span);
             });
         }
     }
@@ -862,9 +864,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         errors: &mut Vec<traits::FulfillmentError<'tcx>>,
         span: Span,
         return_expr_ty: Ty<'tcx>,
+        return_span: Span,
     ) {
         // Don't point at the whole block if it's empty
-        if span == self.tcx.hir().span(self.body_id) {
+        if span == return_span {
             return;
         }
         for err in errors {
@@ -1030,7 +1033,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
 
         let result_ty = coerce.complete(self);
-        if cond_ty.references_error() { self.tcx.ty_error() } else { result_ty }
+        if let Err(guar) = cond_ty.error_reported() { self.tcx.ty_error(guar) } else { result_ty }
     }
 
     /// Type check assignment expression `expr` of form `lhs = rhs`.
@@ -1106,7 +1109,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // If the assignment expression itself is ill-formed, don't
             // bother emitting another error
             let reported = err.emit_unless(lhs_ty.references_error() || rhs_ty.references_error());
-            return self.tcx.ty_error_with_guaranteed(reported);
+            return self.tcx.ty_error(reported);
         }
 
         let lhs_ty = self.check_expr_with_needs(&lhs, Needs::MutPlace);
@@ -1152,8 +1155,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         self.require_type_is_sized(lhs_ty, lhs.span, traits::AssignmentLhsSized);
 
-        if lhs_ty.references_error() || rhs_ty.references_error() {
-            self.tcx.ty_error()
+        if let Err(guar) = (lhs_ty, rhs_ty).error_reported() {
+            self.tcx.ty_error(guar)
         } else {
             self.tcx.mk_unit()
         }
@@ -1271,8 +1274,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let t_expr = self.resolve_vars_if_possible(t_expr);
 
         // Eagerly check for some obvious errors.
-        if t_expr.references_error() || t_cast.references_error() {
-            self.tcx.ty_error()
+        if let Err(guar) = (t_expr, t_cast).error_reported() {
+            self.tcx.ty_error(guar)
         } else {
             // Defer other checks until we're done type checking.
             let mut deferred_cast_checks = self.deferred_cast_checks.borrow_mut();
@@ -1293,7 +1296,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     deferred_cast_checks.push(cast_check);
                     t_cast
                 }
-                Err(_) => self.tcx.ty_error(),
+                Err(guar) => self.tcx.ty_error(guar),
             }
         }
     }
@@ -1374,7 +1377,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let body = self.tcx.hir().body(anon_const.body);
 
         // Create a new function context.
-        let fcx = FnCtxt::new(self, self.param_env.with_const(), body.value.hir_id);
+        let def_id = anon_const.def_id;
+        let fcx = FnCtxt::new(self, self.param_env.with_const(), def_id);
         crate::GatherLocalsVisitor::new(&fcx).visit_body(body);
 
         let ty = fcx.check_expr_with_expectation(&body.value, expected);
@@ -1392,7 +1396,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     ) -> Ty<'tcx> {
         let tcx = self.tcx;
         let count = self.array_length_to_const(count);
-        if let Some(count) = count.try_eval_usize(tcx, self.param_env) {
+        if let Some(count) = count.try_eval_target_usize(tcx, self.param_env) {
             self.suggest_array_len(expr, count);
         }
 
@@ -1419,13 +1423,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             }
         };
 
-        if element_ty.references_error() {
-            return tcx.ty_error();
+        if let Err(guar) = element_ty.error_reported() {
+            return tcx.ty_error(guar);
         }
 
         self.check_repeat_element_needs_copy_bound(element, count, element_ty);
 
-        tcx.mk_ty(ty::Array(t, count))
+        tcx.mk_array_with_const_len(t, count)
     }
 
     fn check_repeat_element_needs_copy_bound(
@@ -1459,7 +1463,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         // If the length is 0, we don't create any elements, so we don't copy any. If the length is 1, we
         // don't copy that one element, we move it. Only check for Copy if the length is larger.
-        if count.try_eval_usize(tcx, self.param_env).map_or(true, |len| len > 1) {
+        if count.try_eval_target_usize(tcx, self.param_env).map_or(true, |len| len > 1) {
             let lang_item = self.tcx.require_lang_item(LangItem::Copy, None);
             let code = traits::ObligationCauseCode::RepeatElementCopy { is_const_fn };
             self.require_type_meets(element_ty, element.span, code, lang_item);
@@ -1488,9 +1492,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             }
             _ => self.check_expr_with_expectation(&e, NoExpectation),
         });
-        let tuple = self.tcx.mk_tup(elt_ts_iter);
-        if tuple.references_error() {
-            self.tcx.ty_error()
+        let tuple = self.tcx.mk_tup_from_iter(elt_ts_iter);
+        if let Err(guar) = tuple.error_reported() {
+            self.tcx.ty_error(guar)
         } else {
             self.require_type_is_sized(tuple, expr.span, traits::TupleInitializerSized);
             tuple
@@ -1506,9 +1510,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         base_expr: &'tcx Option<&'tcx hir::Expr<'tcx>>,
     ) -> Ty<'tcx> {
         // Find the relevant variant
-        let Some((variant, adt_ty)) = self.check_struct_path(qpath, expr.hir_id) else {
-            self.check_struct_fields_on_error(fields, base_expr);
-            return self.tcx.ty_error();
+        let (variant, adt_ty) = match self.check_struct_path(qpath, expr.hir_id) {
+            Ok(data) => data,
+            Err(guar) => {
+                self.check_struct_fields_on_error(fields, base_expr);
+                return self.tcx.ty_error(guar);
+            }
         };
 
         // Prohibit struct expressions when non-exhaustive flag is set.
@@ -1590,12 +1597,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 self.field_ty(field.span, v_field, substs)
             } else {
                 error_happened = true;
-                if let Some(prev_span) = seen_fields.get(&ident) {
+                let guar = if let Some(prev_span) = seen_fields.get(&ident) {
                     tcx.sess.emit_err(FieldMultiplySpecifiedInInitializer {
                         span: field.ident.span,
                         prev_span: *prev_span,
                         ident,
-                    });
+                    })
                 } else {
                     self.report_unknown_field(
                         adt_ty,
@@ -1604,10 +1611,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         ast_fields,
                         adt.variant_descr(),
                         expr_span,
-                    );
-                }
+                    )
+                };
 
-                tcx.ty_error()
+                tcx.ty_error(guar)
             };
 
             // Make sure to give a type to the field even if there's
@@ -1990,14 +1997,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         skip_fields: &[hir::ExprField<'_>],
         kind_name: &str,
         expr_span: Span,
-    ) {
+    ) -> ErrorGuaranteed {
         if variant.is_recovered() {
-            self.set_tainted_by_errors(
-                self.tcx
-                    .sess
-                    .delay_span_bug(expr_span, "parser recovered but no error was emitted"),
-            );
-            return;
+            let guar = self
+                .tcx
+                .sess
+                .delay_span_bug(expr_span, "parser recovered but no error was emitted");
+            self.set_tainted_by_errors(guar);
+            return guar;
         }
         let mut err = self.err_ctxt().type_error_struct_with_diag(
             field.ident.span,
@@ -2111,7 +2118,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 };
             }
         }
-        err.emit();
+        err.emit()
     }
 
     // Return a hint about the closest match in field names
@@ -2151,13 +2158,18 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         variant: &'tcx ty::VariantDef,
         access_span: Span,
     ) -> Vec<Symbol> {
+        let body_owner_hir_id = self.tcx.hir().local_def_id_to_hir_id(self.body_id);
         variant
             .fields
             .iter()
             .filter(|field| {
                 let def_scope = self
                     .tcx
-                    .adjust_ident_and_get_scope(field.ident(self.tcx), variant.def_id, self.body_id)
+                    .adjust_ident_and_get_scope(
+                        field.ident(self.tcx),
+                        variant.def_id,
+                        body_owner_hir_id,
+                    )
                     .1;
                 field.vis.is_accessible_from(def_scope, self.tcx)
                     && !matches!(
@@ -2199,8 +2211,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             match deref_base_ty.kind() {
                 ty::Adt(base_def, substs) if !base_def.is_enum() => {
                     debug!("struct named {:?}", deref_base_ty);
+                    let body_hir_id = self.tcx.hir().local_def_id_to_hir_id(self.body_id);
                     let (ident, def_scope) =
-                        self.tcx.adjust_ident_and_get_scope(field, base_def.did(), self.body_id);
+                        self.tcx.adjust_ident_and_get_scope(field, base_def.did(), body_hir_id);
                     let fields = &base_def.non_enum_variant().fields;
                     if let Some(index) = fields
                         .iter()
@@ -2246,11 +2259,18 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // (#90483) apply adjustments to avoid ExprUseVisitor from
             // creating erroneous projection.
             self.apply_adjustments(base, adjustments);
-            self.ban_private_field_access(expr, base_ty, field, did, expected.only_has_type(self));
-            return self.tcx().ty_error();
+            let guar = self.ban_private_field_access(
+                expr,
+                base_ty,
+                field,
+                did,
+                expected.only_has_type(self),
+            );
+            return self.tcx().ty_error(guar);
         }
 
-        if field.name == kw::Empty {
+        let guar = if field.name == kw::Empty {
+            self.tcx.sess.delay_span_bug(field.span, "field name with no name")
         } else if self.method_exists(
             field,
             base_ty,
@@ -2258,9 +2278,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             true,
             expected.only_has_type(self),
         ) {
-            self.ban_take_value_of_method(expr, base_ty, field);
+            self.ban_take_value_of_method(expr, base_ty, field)
         } else if !base_ty.is_primitive_ty() {
-            self.ban_nonexisting_field(field, base, expr, base_ty);
+            self.ban_nonexisting_field(field, base, expr, base_ty)
         } else {
             let field_name = field.to_string();
             let mut err = type_error_struct!(
@@ -2329,10 +2349,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     );
                 }
             }
-            err.emit();
-        }
+            err.emit()
+        };
 
-        self.tcx().ty_error()
+        self.tcx().ty_error(guar)
     }
 
     fn suggest_await_on_field_access(
@@ -2378,7 +2398,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         base: &'tcx hir::Expr<'tcx>,
         expr: &'tcx hir::Expr<'tcx>,
         base_ty: Ty<'tcx>,
-    ) {
+    ) -> ErrorGuaranteed {
         debug!(
             "ban_nonexisting_field: field={:?}, base={:?}, expr={:?}, base_ty={:?}",
             ident, base, expr, base_ty
@@ -2423,10 +2443,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // We know by construction that `<expr>.await` is either on Rust 2015
             // or results in `ExprKind::Await`. Suggest switching the edition to 2018.
             err.note("to `.await` a `Future`, switch to Rust 2018 or later");
-            err.help_use_latest_edition();
+            HelpUseLatestEdition::new().add_to_diagnostic(&mut err);
         }
 
-        err.emit();
+        err.emit()
     }
 
     fn ban_private_field_access(
@@ -2436,9 +2456,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         field: Ident,
         base_did: DefId,
         return_ty: Option<Ty<'tcx>>,
-    ) {
+    ) -> ErrorGuaranteed {
         let struct_path = self.tcx().def_path_str(base_did);
-        let kind_name = self.tcx().def_kind(base_did).descr(base_did);
+        let kind_name = self.tcx().def_descr(base_did);
         let mut err = struct_span_err!(
             self.tcx().sess,
             field.span,
@@ -2459,10 +2479,15 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 None,
             );
         }
-        err.emit();
+        err.emit()
     }
 
-    fn ban_take_value_of_method(&self, expr: &hir::Expr<'tcx>, expr_t: Ty<'tcx>, field: Ident) {
+    fn ban_take_value_of_method(
+        &self,
+        expr: &hir::Expr<'tcx>,
+        expr_t: Ty<'tcx>,
+        field: Ident,
+    ) -> ErrorGuaranteed {
         let mut err = type_error_struct!(
             self.tcx().sess,
             field.span,
@@ -2534,11 +2559,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             err.help("methods are immutable and cannot be assigned to");
         }
 
-        err.emit();
+        err.emit()
     }
 
     fn point_at_param_definition(&self, err: &mut Diagnostic, param: ty::ParamTy) {
-        let generics = self.tcx.generics_of(self.body_id.owner.to_def_id());
+        let generics = self.tcx.generics_of(self.body_id);
         let generic_param = generics.type_param(&param, self.tcx);
         if let ty::GenericParamDefKind::Type { synthetic: true, .. } = generic_param.kind {
             return;
@@ -2592,7 +2617,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         len: ty::Const<'tcx>,
     ) {
         if let (Some(len), Ok(user_index)) =
-            (len.try_eval_usize(self.tcx, self.param_env), field.as_str().parse::<u64>())
+            (len.try_eval_target_usize(self.tcx, self.param_env), field.as_str().parse::<u64>())
             && let Ok(base) = self.tcx.sess.source_map().span_to_snippet(base.span)
         {
             let help = "instead of using tuple indexing, use array indexing";
@@ -2819,7 +2844,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         }
                     }
                     let reported = err.emit();
-                    self.tcx.ty_error_with_guaranteed(reported)
+                    self.tcx.ty_error(reported)
                 }
             }
         }
diff --git a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
index c8cda0dc9..b9a058d6b 100644
--- a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
+++ b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
@@ -301,7 +301,7 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
             hir::ExprKind::Continue(..)
             | hir::ExprKind::Lit(..)
             | hir::ExprKind::ConstBlock(..)
-            | hir::ExprKind::Err => {}
+            | hir::ExprKind::Err(_) => {}
 
             hir::ExprKind::Loop(blk, ..) => {
                 self.walk_block(blk);
diff --git a/compiler/rustc_hir_typeck/src/fallback.rs b/compiler/rustc_hir_typeck/src/fallback.rs
index dde879780..b7ae621c6 100644
--- a/compiler/rustc_hir_typeck/src/fallback.rs
+++ b/compiler/rustc_hir_typeck/src/fallback.rs
@@ -104,7 +104,7 @@ impl<'tcx> FnCtxt<'_, 'tcx> {
         // type, `?T` is not considered unsolved, but `?I` is. The
         // same is true for float variables.)
         let fallback = match ty.kind() {
-            _ if let Some(e) = self.tainted_by_errors() => self.tcx.ty_error_with_guaranteed(e),
+            _ if let Some(e) = self.tainted_by_errors() => self.tcx.ty_error(e),
             ty::Infer(ty::IntVar(_)) => self.tcx.types.i32,
             ty::Infer(ty::FloatVar(_)) => self.tcx.types.f64,
             _ => match diverging_fallback.get(&ty) {
@@ -196,8 +196,6 @@ impl<'tcx> FnCtxt<'_, 'tcx> {
     ) -> FxHashMap<Ty<'tcx>, Ty<'tcx>> {
         debug!("calculate_diverging_fallback({:?})", unsolved_variables);
 
-        let relationships = self.fulfillment_cx.borrow_mut().relationships().clone();
-
         // Construct a coercion graph where an edge `A -> B` indicates
         // a type variable is that is coerced
         let coercion_graph = self.create_coercion_graph();
@@ -281,9 +279,7 @@ impl<'tcx> FnCtxt<'_, 'tcx> {
             roots_reachable_from_non_diverging,
         );
 
-        debug!("inherited: {:#?}", self.inh.fulfillment_cx.borrow_mut().pending_obligations());
         debug!("obligations: {:#?}", self.fulfillment_cx.borrow_mut().pending_obligations());
-        debug!("relationships: {:#?}", relationships);
 
         // For each diverging variable, figure out whether it can
         // reach a member of N. If so, it falls back to `()`. Else
@@ -297,16 +293,16 @@ impl<'tcx> FnCtxt<'_, 'tcx> {
                 .depth_first_search(root_vid)
                 .any(|n| roots_reachable_from_non_diverging.visited(n));
 
-            let mut relationship = ty::FoundRelationships { self_in_trait: false, output: false };
+            let mut found_infer_var_info = ty::InferVarInfo { self_in_trait: false, output: false };
 
-            for (vid, rel) in relationships.iter() {
-                if self.root_var(*vid) == root_vid {
-                    relationship.self_in_trait |= rel.self_in_trait;
-                    relationship.output |= rel.output;
+            for (vid, info) in self.inh.infer_var_info.borrow().iter() {
+                if self.infcx.root_var(*vid) == root_vid {
+                    found_infer_var_info.self_in_trait |= info.self_in_trait;
+                    found_infer_var_info.output |= info.output;
                 }
             }
 
-            if relationship.self_in_trait && relationship.output {
+            if found_infer_var_info.self_in_trait && found_infer_var_info.output {
                 // This case falls back to () to ensure that the code pattern in
                 // tests/ui/never_type/fallback-closure-ret.rs continues to
                 // compile when never_type_fallback is enabled.
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
index 6ed8adb47..60e55c7b0 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
@@ -23,16 +23,16 @@ use rustc_infer::infer::InferResult;
 use rustc_middle::ty::adjustment::{Adjust, Adjustment, AutoBorrow, AutoBorrowMutability};
 use rustc_middle::ty::error::TypeError;
 use rustc_middle::ty::fold::TypeFoldable;
-use rustc_middle::ty::visit::TypeVisitable;
+use rustc_middle::ty::visit::{TypeVisitable, TypeVisitableExt};
 use rustc_middle::ty::{
-    self, AdtKind, CanonicalUserType, DefIdTree, GenericParamDefKind, Ty, UserType,
+    self, AdtKind, CanonicalUserType, DefIdTree, GenericParamDefKind, Ty, TyCtxt, UserType,
 };
 use rustc_middle::ty::{GenericArgKind, SubstsRef, UserSelfTy, UserSubsts};
 use rustc_session::lint;
 use rustc_span::def_id::LocalDefId;
 use rustc_span::hygiene::DesugaringKind;
 use rustc_span::symbol::{kw, sym, Ident};
-use rustc_span::{Span, DUMMY_SP};
+use rustc_span::Span;
 use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt as _;
 use rustc_trait_selection::traits::{self, NormalizeExt, ObligationCauseCode, ObligationCtxt};
 
@@ -315,7 +315,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
     pub(in super::super) fn normalize<T>(&self, span: Span, value: T) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         self.register_infer_ok_obligations(
             self.at(&self.misc(span), self.param_env).normalize(value),
@@ -443,7 +443,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     // sufficiently enforced with erased regions. =)
     fn can_contain_user_lifetime_bounds<T>(t: T) -> bool
     where
-        T: TypeVisitable<'tcx>,
+        T: TypeVisitable<TyCtxt<'tcx>>,
     {
         t.has_free_regions() || t.has_projections() || t.has_infer_types()
     }
@@ -451,11 +451,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     pub fn node_ty(&self, id: hir::HirId) -> Ty<'tcx> {
         match self.typeck_results.borrow().node_types().get(id) {
             Some(&t) => t,
-            None if let Some(e) = self.tainted_by_errors() => self.tcx.ty_error_with_guaranteed(e),
+            None if let Some(e) = self.tainted_by_errors() => self.tcx.ty_error(e),
             None => {
                 bug!(
-                    "no type for node {}: {} in fcx {}",
-                    id,
+                    "no type for node {} in fcx {}",
                     self.tcx.hir().node_to_string(id),
                     self.tag()
                 );
@@ -466,7 +465,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     pub fn node_ty_opt(&self, id: hir::HirId) -> Option<Ty<'tcx>> {
         match self.typeck_results.borrow().node_types().get(id) {
             Some(&t) => Some(t),
-            None if let Some(e) = self.tainted_by_errors() => Some(self.tcx.ty_error_with_guaranteed(e)),
+            None if let Some(e) = self.tainted_by_errors() => Some(self.tcx.ty_error(e)),
             None => None,
         }
     }
@@ -517,16 +516,72 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     }
 
     pub(in super::super) fn resolve_generator_interiors(&self, def_id: DefId) {
+        if self.tcx.sess.opts.unstable_opts.drop_tracking_mir {
+            self.save_generator_interior_predicates(def_id);
+            return;
+        }
+
+        self.select_obligations_where_possible(|_| {});
+
         let mut generators = self.deferred_generator_interiors.borrow_mut();
-        for (body_id, interior, kind) in generators.drain(..) {
-            self.select_obligations_where_possible(|_| {});
+        for (_, body_id, interior, kind) in generators.drain(..) {
             crate::generator_interior::resolve_interior(self, def_id, body_id, interior, kind);
+            self.select_obligations_where_possible(|_| {});
+        }
+    }
+
+    /// Unify the inference variables corresponding to generator witnesses, and save all the
+    /// predicates that were stalled on those inference variables.
+    ///
+    /// This process allows to conservatively save all predicates that do depend on the generator
+    /// interior types, for later processing by `check_generator_obligations`.
+    ///
+    /// We must not attempt to select obligations after this method has run, or risk query cycle
+    /// ICE.
+    #[instrument(level = "debug", skip(self))]
+    fn save_generator_interior_predicates(&self, def_id: DefId) {
+        // Try selecting all obligations that are not blocked on inference variables.
+        // Once we start unifying generator witnesses, trying to select obligations on them will
+        // trigger query cycle ICEs, as doing so requires MIR.
+        self.select_obligations_where_possible(|_| {});
+
+        let generators = std::mem::take(&mut *self.deferred_generator_interiors.borrow_mut());
+        debug!(?generators);
+
+        for &(expr_def_id, body_id, interior, _) in generators.iter() {
+            debug!(?expr_def_id);
+
+            // Create the `GeneratorWitness` type that we will unify with `interior`.
+            let substs = ty::InternalSubsts::identity_for_item(
+                self.tcx,
+                self.tcx.typeck_root_def_id(expr_def_id.to_def_id()),
+            );
+            let witness = self.tcx.mk_generator_witness_mir(expr_def_id.to_def_id(), substs);
+
+            // Unify `interior` with `witness` and collect all the resulting obligations.
+            let span = self.tcx.hir().body(body_id).value.span;
+            let ok = self
+                .at(&self.misc(span), self.param_env)
+                .eq(interior, witness)
+                .expect("Failed to unify generator interior type");
+            let mut obligations = ok.obligations;
+
+            // Also collect the obligations that were unstalled by this unification.
+            obligations
+                .extend(self.fulfillment_cx.borrow_mut().drain_unstalled_obligations(&self.infcx));
+
+            let obligations = obligations.into_iter().map(|o| (o.predicate, o.cause)).collect();
+            debug!(?obligations);
+            self.typeck_results
+                .borrow_mut()
+                .generator_interior_predicates
+                .insert(expr_def_id, obligations);
         }
     }
 
     #[instrument(skip(self), level = "debug")]
-    pub(in super::super) fn select_all_obligations_or_error(&self) {
-        let mut errors = self.fulfillment_cx.borrow_mut().select_all_or_error(&self);
+    pub(in super::super) fn report_ambiguity_errors(&self) {
+        let mut errors = self.fulfillment_cx.borrow_mut().collect_remaining_errors();
 
         if !errors.is_empty() {
             self.adjust_fulfillment_errors_for_expr_obligation(&mut errors);
@@ -608,12 +663,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
                 ty::PredicateKind::Clause(ty::Clause::Trait(..))
                 | ty::PredicateKind::Clause(ty::Clause::Projection(..))
+                | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
                 | ty::PredicateKind::Subtype(..)
                 | ty::PredicateKind::Coerce(..)
                 | ty::PredicateKind::Clause(ty::Clause::RegionOutlives(..))
                 | ty::PredicateKind::Clause(ty::Clause::TypeOutlives(..))
                 | ty::PredicateKind::WellFormed(..)
                 | ty::PredicateKind::ObjectSafe(..)
+                | ty::PredicateKind::AliasEq(..)
                 | ty::PredicateKind::ConstEvaluatable(..)
                 | ty::PredicateKind::ConstEquate(..)
                 // N.B., this predicate is created by breaking down a
@@ -644,7 +701,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     }
 
     pub(in super::super) fn err_args(&self, len: usize) -> Vec<Ty<'tcx>> {
-        vec![self.tcx.ty_error(); len]
+        let ty_error = self.tcx.ty_error_misc();
+        vec![ty_error; len]
     }
 
     /// Unifies the output type with the expected type early, for more coercions
@@ -680,7 +738,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 if let ty::subst::GenericArgKind::Type(ty) = ty.unpack()
                     && let ty::Alias(ty::Opaque, ty::AliasTy { def_id, .. }) = *ty.kind()
                     && let Some(def_id) = def_id.as_local()
-                    && self.opaque_type_origin(def_id, DUMMY_SP).is_some() {
+                    && self.opaque_type_origin(def_id).is_some() {
                     return None;
                 }
             }
@@ -720,9 +778,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let def_kind = self.tcx.def_kind(def_id);
 
         let item_ty = if let DefKind::Variant = def_kind {
-            self.tcx.bound_type_of(self.tcx.parent(def_id))
+            self.tcx.type_of(self.tcx.parent(def_id))
         } else {
-            self.tcx.bound_type_of(def_id)
+            self.tcx.type_of(def_id)
         };
         let substs = self.fresh_substs_for_item(span, def_id);
         let ty = item_ty.subst(self.tcx, substs);
@@ -866,6 +924,22 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 kind: hir::ImplItemKind::Fn(ref sig, ..),
                 ..
             }) => Some((&sig.decl, ident, false)),
+            Node::Expr(&hir::Expr {
+                hir_id,
+                kind: hir::ExprKind::Closure(..),
+                ..
+            }) if let Some(Node::Expr(&hir::Expr {
+                hir_id,
+                kind: hir::ExprKind::Call(..),
+                ..
+            })) = self.tcx.hir().find_parent(hir_id) &&
+            let Some(Node::Item(&hir::Item {
+                ident,
+                kind: hir::ItemKind::Fn(ref sig, ..),
+                ..
+            })) = self.tcx.hir().find_parent(hir_id) => {
+                Some((&sig.decl, ident, ident.name != sym::main))
+            },
             _ => None,
         }
     }
@@ -926,43 +1000,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
     }
 
-    pub(in super::super) fn note_need_for_fn_pointer(
-        &self,
-        err: &mut Diagnostic,
-        expected: Ty<'tcx>,
-        found: Ty<'tcx>,
-    ) {
-        let (sig, did, substs) = match (&expected.kind(), &found.kind()) {
-            (ty::FnDef(did1, substs1), ty::FnDef(did2, substs2)) => {
-                let sig1 = self.tcx.bound_fn_sig(*did1).subst(self.tcx, substs1);
-                let sig2 = self.tcx.bound_fn_sig(*did2).subst(self.tcx, substs2);
-                if sig1 != sig2 {
-                    return;
-                }
-                err.note(
-                    "different `fn` items always have unique types, even if their signatures are \
-                     the same",
-                );
-                (sig1, *did1, substs1)
-            }
-            (ty::FnDef(did, substs), ty::FnPtr(sig2)) => {
-                let sig1 = self.tcx.bound_fn_sig(*did).subst(self.tcx, substs);
-                if sig1 != *sig2 {
-                    return;
-                }
-                (sig1, *did, substs)
-            }
-            _ => return,
-        };
-        err.help(&format!("change the expected type to be function pointer `{}`", sig));
-        err.help(&format!(
-            "if the expected type is due to type inference, cast the expected `fn` to a function \
-             pointer: `{} as {}`",
-            self.tcx.def_path_str_with_substs(did, substs),
-            sig
-        ));
-    }
-
     // Instantiates the given path, which must refer to an item with the given
     // number of type parameters and type.
     #[instrument(skip(self, span), level = "debug")]
@@ -1095,7 +1132,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             .unwrap_or(false);
 
         let (res, self_ctor_substs) = if let Res::SelfCtor(impl_def_id) = res {
-            let ty = self.handle_raw_ty(span, tcx.at(span).type_of(impl_def_id));
+            let ty = self.handle_raw_ty(span, tcx.at(span).type_of(impl_def_id).subst_identity());
             match ty.normalized.ty_adt_def() {
                 Some(adt_def) if adt_def.has_ctor() => {
                     let (ctor_kind, ctor_def_id) = adt_def.non_enum_variant().ctor.unwrap();
@@ -1125,7 +1162,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         }
                     }
                     let reported = err.emit();
-                    return (tcx.ty_error_with_guaranteed(reported), res);
+                    return (tcx.ty_error(reported), res);
                 }
             }
         } else {
@@ -1191,7 +1228,15 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     }
                     (GenericParamDefKind::Const { .. }, GenericArg::Infer(inf)) => {
                         let tcx = self.fcx.tcx();
-                        self.fcx.ct_infer(tcx.type_of(param.def_id), Some(param), inf.span).into()
+                        self.fcx
+                            .ct_infer(
+                                tcx.type_of(param.def_id)
+                                    .no_bound_vars()
+                                    .expect("const parameter types cannot be generic"),
+                                Some(param),
+                                inf.span,
+                            )
+                            .into()
                     }
                     _ => unreachable!(),
                 }
@@ -1213,7 +1258,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             // If we have a default, then we it doesn't matter that we're not
                             // inferring the type arguments: we provide the default where any
                             // is missing.
-                            tcx.bound_type_of(param.def_id).subst(tcx, substs.unwrap()).into()
+                            tcx.type_of(param.def_id).subst(tcx, substs.unwrap()).into()
                         } else {
                             // If no type arguments were provided, we have to infer them.
                             // This case also occurs as a result of some malformed input, e.g.
@@ -1261,7 +1306,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         // Substitute the values for the type parameters into the type of
         // the referenced item.
-        let ty = tcx.bound_type_of(def_id);
+        let ty = tcx.type_of(def_id);
         assert!(!substs.has_escaping_bound_vars());
         assert!(!ty.0.has_escaping_bound_vars());
         let ty_substituted = self.normalize(span, ty.subst(tcx, substs));
@@ -1272,7 +1317,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // type parameters, which we can infer by unifying the provided `Self`
             // with the substituted impl type.
             // This also occurs for an enum variant on a type alias.
-            let impl_ty = self.normalize(span, tcx.bound_type_of(impl_def_id).subst(tcx, substs));
+            let impl_ty = self.normalize(span, tcx.type_of(impl_def_id).subst(tcx, substs));
             let self_ty = self.normalize(span, self_ty);
             match self.at(&self.misc(span), self.param_env).eq(impl_ty, self_ty) {
                 Ok(ok) => self.register_infer_ok_obligations(ok),
@@ -1373,7 +1418,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     .emit_inference_failure_err((**self).body_id, sp, ty.into(), E0282, true)
                     .emit()
             });
-            let err = self.tcx.ty_error_with_guaranteed(e);
+            let err = self.tcx.ty_error(e);
             self.demand_suptype(sp, err, ty);
             err
         }
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs
new file mode 100644
index 000000000..b09886fe3
--- /dev/null
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs
@@ -0,0 +1,864 @@
+use crate::FnCtxt;
+use rustc_hir as hir;
+use rustc_hir::def::Res;
+use rustc_hir::def_id::DefId;
+use rustc_infer::traits::ObligationCauseCode;
+use rustc_middle::ty::{
+    self, DefIdTree, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitor,
+};
+use rustc_span::{self, Span};
+use rustc_trait_selection::traits;
+
+use std::ops::ControlFlow;
+
+impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
+    pub fn adjust_fulfillment_error_for_expr_obligation(
+        &self,
+        error: &mut traits::FulfillmentError<'tcx>,
+    ) -> bool {
+        let (traits::ExprItemObligation(def_id, hir_id, idx) | traits::ExprBindingObligation(def_id, _, hir_id, idx))
+            = *error.obligation.cause.code().peel_derives() else { return false; };
+        let hir = self.tcx.hir();
+        let hir::Node::Expr(expr) = hir.get(hir_id) else { return false; };
+
+        let Some(unsubstituted_pred) =
+            self.tcx.predicates_of(def_id).instantiate_identity(self.tcx).predicates.into_iter().nth(idx)
+            else { return false; };
+
+        let generics = self.tcx.generics_of(def_id);
+        let predicate_substs = match unsubstituted_pred.kind().skip_binder() {
+            ty::PredicateKind::Clause(ty::Clause::Trait(pred)) => pred.trait_ref.substs,
+            ty::PredicateKind::Clause(ty::Clause::Projection(pred)) => pred.projection_ty.substs,
+            _ => ty::List::empty(),
+        };
+
+        let find_param_matching = |matches: &dyn Fn(&ty::ParamTy) -> bool| {
+            predicate_substs.types().find_map(|ty| {
+                ty.walk().find_map(|arg| {
+                    if let ty::GenericArgKind::Type(ty) = arg.unpack()
+                        && let ty::Param(param_ty) = ty.kind()
+                        && matches(param_ty)
+                    {
+                        Some(arg)
+                    } else {
+                        None
+                    }
+                })
+            })
+        };
+
+        // Prefer generics that are local to the fn item, since these are likely
+        // to be the cause of the unsatisfied predicate.
+        let mut param_to_point_at = find_param_matching(&|param_ty| {
+            self.tcx.parent(generics.type_param(param_ty, self.tcx).def_id) == def_id
+        });
+        // Fall back to generic that isn't local to the fn item. This will come
+        // from a trait or impl, for example.
+        let mut fallback_param_to_point_at = find_param_matching(&|param_ty| {
+            self.tcx.parent(generics.type_param(param_ty, self.tcx).def_id) != def_id
+                && param_ty.name != rustc_span::symbol::kw::SelfUpper
+        });
+        // Finally, the `Self` parameter is possibly the reason that the predicate
+        // is unsatisfied. This is less likely to be true for methods, because
+        // method probe means that we already kinda check that the predicates due
+        // to the `Self` type are true.
+        let mut self_param_to_point_at =
+            find_param_matching(&|param_ty| param_ty.name == rustc_span::symbol::kw::SelfUpper);
+
+        // Finally, for ambiguity-related errors, we actually want to look
+        // for a parameter that is the source of the inference type left
+        // over in this predicate.
+        if let traits::FulfillmentErrorCode::CodeAmbiguity = error.code {
+            fallback_param_to_point_at = None;
+            self_param_to_point_at = None;
+            param_to_point_at =
+                self.find_ambiguous_parameter_in(def_id, error.root_obligation.predicate);
+        }
+
+        if self.closure_span_overlaps_error(error, expr.span) {
+            return false;
+        }
+
+        match &expr.kind {
+            hir::ExprKind::Path(qpath) => {
+                if let hir::Node::Expr(hir::Expr {
+                    kind: hir::ExprKind::Call(callee, args),
+                    hir_id: call_hir_id,
+                    span: call_span,
+                    ..
+                }) = hir.get_parent(expr.hir_id)
+                    && callee.hir_id == expr.hir_id
+                {
+                    if self.closure_span_overlaps_error(error, *call_span) {
+                        return false;
+                    }
+
+                    for param in
+                        [param_to_point_at, fallback_param_to_point_at, self_param_to_point_at]
+                        .into_iter()
+                        .flatten()
+                    {
+                        if self.blame_specific_arg_if_possible(
+                                error,
+                                def_id,
+                                param,
+                                *call_hir_id,
+                                callee.span,
+                                None,
+                                args,
+                            )
+                        {
+                            return true;
+                        }
+                    }
+                }
+                // Notably, we only point to params that are local to the
+                // item we're checking, since those are the ones we are able
+                // to look in the final `hir::PathSegment` for. Everything else
+                // would require a deeper search into the `qpath` than I think
+                // is worthwhile.
+                if let Some(param_to_point_at) = param_to_point_at
+                    && self.point_at_path_if_possible(error, def_id, param_to_point_at, qpath)
+                {
+                    return true;
+                }
+            }
+            hir::ExprKind::MethodCall(segment, receiver, args, ..) => {
+                for param in [param_to_point_at, fallback_param_to_point_at, self_param_to_point_at]
+                    .into_iter()
+                    .flatten()
+                {
+                    if self.blame_specific_arg_if_possible(
+                        error,
+                        def_id,
+                        param,
+                        hir_id,
+                        segment.ident.span,
+                        Some(receiver),
+                        args,
+                    ) {
+                        return true;
+                    }
+                }
+                if let Some(param_to_point_at) = param_to_point_at
+                    && self.point_at_generic_if_possible(error, def_id, param_to_point_at, segment)
+                {
+                    return true;
+                }
+            }
+            hir::ExprKind::Struct(qpath, fields, ..) => {
+                if let Res::Def(
+                    hir::def::DefKind::Struct | hir::def::DefKind::Variant,
+                    variant_def_id,
+                ) = self.typeck_results.borrow().qpath_res(qpath, hir_id)
+                {
+                    for param in
+                        [param_to_point_at, fallback_param_to_point_at, self_param_to_point_at]
+                    {
+                        if let Some(param) = param {
+                            let refined_expr = self.point_at_field_if_possible(
+                                def_id,
+                                param,
+                                variant_def_id,
+                                fields,
+                            );
+
+                            match refined_expr {
+                                None => {}
+                                Some((refined_expr, _)) => {
+                                    error.obligation.cause.span = refined_expr
+                                        .span
+                                        .find_ancestor_in_same_ctxt(error.obligation.cause.span)
+                                        .unwrap_or(refined_expr.span);
+                                    return true;
+                                }
+                            }
+                        }
+                    }
+                }
+                if let Some(param_to_point_at) = param_to_point_at
+                    && self.point_at_path_if_possible(error, def_id, param_to_point_at, qpath)
+                {
+                    return true;
+                }
+            }
+            _ => {}
+        }
+
+        false
+    }
+
+    fn point_at_path_if_possible(
+        &self,
+        error: &mut traits::FulfillmentError<'tcx>,
+        def_id: DefId,
+        param: ty::GenericArg<'tcx>,
+        qpath: &hir::QPath<'tcx>,
+    ) -> bool {
+        match qpath {
+            hir::QPath::Resolved(_, path) => {
+                if let Some(segment) = path.segments.last()
+                    && self.point_at_generic_if_possible(error, def_id, param, segment)
+                {
+                    return true;
+                }
+            }
+            hir::QPath::TypeRelative(_, segment) => {
+                if self.point_at_generic_if_possible(error, def_id, param, segment) {
+                    return true;
+                }
+            }
+            _ => {}
+        }
+
+        false
+    }
+
+    fn point_at_generic_if_possible(
+        &self,
+        error: &mut traits::FulfillmentError<'tcx>,
+        def_id: DefId,
+        param_to_point_at: ty::GenericArg<'tcx>,
+        segment: &hir::PathSegment<'tcx>,
+    ) -> bool {
+        let own_substs = self
+            .tcx
+            .generics_of(def_id)
+            .own_substs(ty::InternalSubsts::identity_for_item(self.tcx, def_id));
+        let Some((index, _)) = own_substs
+            .iter()
+            .filter(|arg| matches!(arg.unpack(), ty::GenericArgKind::Type(_)))
+            .enumerate()
+            .find(|(_, arg)| **arg == param_to_point_at) else { return false };
+        let Some(arg) = segment
+            .args()
+            .args
+            .iter()
+            .filter(|arg| matches!(arg, hir::GenericArg::Type(_)))
+            .nth(index) else { return false; };
+        error.obligation.cause.span = arg
+            .span()
+            .find_ancestor_in_same_ctxt(error.obligation.cause.span)
+            .unwrap_or(arg.span());
+        true
+    }
+
+    fn find_ambiguous_parameter_in<T: TypeVisitable<TyCtxt<'tcx>>>(
+        &self,
+        item_def_id: DefId,
+        t: T,
+    ) -> Option<ty::GenericArg<'tcx>> {
+        struct FindAmbiguousParameter<'a, 'tcx>(&'a FnCtxt<'a, 'tcx>, DefId);
+        impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for FindAmbiguousParameter<'_, 'tcx> {
+            type BreakTy = ty::GenericArg<'tcx>;
+            fn visit_ty(&mut self, ty: Ty<'tcx>) -> std::ops::ControlFlow<Self::BreakTy> {
+                if let Some(origin) = self.0.type_var_origin(ty)
+                    && let rustc_infer::infer::type_variable::TypeVariableOriginKind::TypeParameterDefinition(_, Some(def_id)) =
+                        origin.kind
+                    && let generics = self.0.tcx.generics_of(self.1)
+                    && let Some(index) = generics.param_def_id_to_index(self.0.tcx, def_id)
+                    && let Some(subst) = ty::InternalSubsts::identity_for_item(self.0.tcx, self.1)
+                        .get(index as usize)
+                {
+                    ControlFlow::Break(*subst)
+                } else {
+                    ty.super_visit_with(self)
+                }
+            }
+        }
+        t.visit_with(&mut FindAmbiguousParameter(self, item_def_id)).break_value()
+    }
+
+    fn closure_span_overlaps_error(
+        &self,
+        error: &traits::FulfillmentError<'tcx>,
+        span: Span,
+    ) -> bool {
+        if let traits::FulfillmentErrorCode::CodeSelectionError(
+            traits::SelectionError::OutputTypeParameterMismatch(_, expected, _),
+        ) = error.code
+            && let ty::Closure(def_id, _) | ty::Generator(def_id, ..) = expected.skip_binder().self_ty().kind()
+            && span.overlaps(self.tcx.def_span(*def_id))
+        {
+            true
+        } else {
+            false
+        }
+    }
+
+    fn point_at_field_if_possible(
+        &self,
+        def_id: DefId,
+        param_to_point_at: ty::GenericArg<'tcx>,
+        variant_def_id: DefId,
+        expr_fields: &[hir::ExprField<'tcx>],
+    ) -> Option<(&'tcx hir::Expr<'tcx>, Ty<'tcx>)> {
+        let def = self.tcx.adt_def(def_id);
+
+        let identity_substs = ty::InternalSubsts::identity_for_item(self.tcx, def_id);
+        let fields_referencing_param: Vec<_> = def
+            .variant_with_id(variant_def_id)
+            .fields
+            .iter()
+            .filter(|field| {
+                let field_ty = field.ty(self.tcx, identity_substs);
+                Self::find_param_in_ty(field_ty.into(), param_to_point_at)
+            })
+            .collect();
+
+        if let [field] = fields_referencing_param.as_slice() {
+            for expr_field in expr_fields {
+                // Look for the ExprField that matches the field, using the
+                // same rules that check_expr_struct uses for macro hygiene.
+                if self.tcx.adjust_ident(expr_field.ident, variant_def_id) == field.ident(self.tcx)
+                {
+                    return Some((expr_field.expr, self.tcx.type_of(field.did).subst_identity()));
+                }
+            }
+        }
+
+        None
+    }
+
+    /// - `blame_specific_*` means that the function will recursively traverse the expression,
+    /// looking for the most-specific-possible span to blame.
+    ///
+    /// - `point_at_*` means that the function will only go "one level", pointing at the specific
+    /// expression mentioned.
+    ///
+    /// `blame_specific_arg_if_possible` will find the most-specific expression anywhere inside
+    /// the provided function call expression, and mark it as responsible for the fullfillment
+    /// error.
+    fn blame_specific_arg_if_possible(
+        &self,
+        error: &mut traits::FulfillmentError<'tcx>,
+        def_id: DefId,
+        param_to_point_at: ty::GenericArg<'tcx>,
+        call_hir_id: hir::HirId,
+        callee_span: Span,
+        receiver: Option<&'tcx hir::Expr<'tcx>>,
+        args: &'tcx [hir::Expr<'tcx>],
+    ) -> bool {
+        let ty = self.tcx.type_of(def_id).subst_identity();
+        if !ty.is_fn() {
+            return false;
+        }
+        let sig = ty.fn_sig(self.tcx).skip_binder();
+        let args_referencing_param: Vec<_> = sig
+            .inputs()
+            .iter()
+            .enumerate()
+            .filter(|(_, ty)| Self::find_param_in_ty((**ty).into(), param_to_point_at))
+            .collect();
+        // If there's one field that references the given generic, great!
+        if let [(idx, _)] = args_referencing_param.as_slice()
+            && let Some(arg) = receiver
+                .map_or(args.get(*idx), |rcvr| if *idx == 0 { Some(rcvr) } else { args.get(*idx - 1) }) {
+
+            error.obligation.cause.span = arg.span.find_ancestor_in_same_ctxt(error.obligation.cause.span).unwrap_or(arg.span);
+
+            if let hir::Node::Expr(arg_expr) = self.tcx.hir().get(arg.hir_id) {
+                // This is more specific than pointing at the entire argument.
+                self.blame_specific_expr_if_possible(error, arg_expr)
+            }
+
+            error.obligation.cause.map_code(|parent_code| {
+                ObligationCauseCode::FunctionArgumentObligation {
+                    arg_hir_id: arg.hir_id,
+                    call_hir_id,
+                    parent_code,
+                }
+            });
+            return true;
+        } else if args_referencing_param.len() > 0 {
+            // If more than one argument applies, then point to the callee span at least...
+            // We have chance to fix this up further in `point_at_generics_if_possible`
+            error.obligation.cause.span = callee_span;
+        }
+
+        false
+    }
+
+    /**
+     * Recursively searches for the most-specific blamable expression.
+     * For example, if you have a chain of constraints like:
+     * - want `Vec<i32>: Copy`
+     * - because `Option<Vec<i32>>: Copy` needs `Vec<i32>: Copy` because `impl <T: Copy> Copy for Option<T>`
+     * - because `(Option<Vec<i32>, bool)` needs `Option<Vec<i32>>: Copy` because `impl <A: Copy, B: Copy> Copy for (A, B)`
+     * then if you pass in `(Some(vec![1, 2, 3]), false)`, this helper `point_at_specific_expr_if_possible`
+     * will find the expression `vec![1, 2, 3]` as the "most blameable" reason for this missing constraint.
+     *
+     * This function only updates the error span.
+     */
+    pub fn blame_specific_expr_if_possible(
+        &self,
+        error: &mut traits::FulfillmentError<'tcx>,
+        expr: &'tcx hir::Expr<'tcx>,
+    ) {
+        // Whether it succeeded or failed, it likely made some amount of progress.
+        // In the very worst case, it's just the same `expr` we originally passed in.
+        let expr = match self.blame_specific_expr_if_possible_for_obligation_cause_code(
+            &error.obligation.cause.code(),
+            expr,
+        ) {
+            Ok(expr) => expr,
+            Err(expr) => expr,
+        };
+
+        // Either way, use this expression to update the error span.
+        // If it doesn't overlap the existing span at all, use the original span.
+        // FIXME: It would possibly be better to do this more continuously, at each level...
+        error.obligation.cause.span = expr
+            .span
+            .find_ancestor_in_same_ctxt(error.obligation.cause.span)
+            .unwrap_or(error.obligation.cause.span);
+    }
+
+    fn blame_specific_expr_if_possible_for_obligation_cause_code(
+        &self,
+        obligation_cause_code: &traits::ObligationCauseCode<'tcx>,
+        expr: &'tcx hir::Expr<'tcx>,
+    ) -> Result<&'tcx hir::Expr<'tcx>, &'tcx hir::Expr<'tcx>> {
+        match obligation_cause_code {
+            traits::ObligationCauseCode::ExprBindingObligation(_, _, _, _) => {
+                // This is the "root"; we assume that the `expr` is already pointing here.
+                // Therefore, we return `Ok` so that this `expr` can be refined further.
+                Ok(expr)
+            }
+            traits::ObligationCauseCode::ImplDerivedObligation(impl_derived) => self
+                .blame_specific_expr_if_possible_for_derived_predicate_obligation(
+                    impl_derived,
+                    expr,
+                ),
+            _ => {
+                // We don't recognize this kind of constraint, so we cannot refine the expression
+                // any further.
+                Err(expr)
+            }
+        }
+    }
+
+    /// We want to achieve the error span in the following example:
+    ///
+    /// ```ignore (just for demonstration)
+    /// struct Burrito<Filling> {
+    ///   filling: Filling,
+    /// }
+    /// impl <Filling: Delicious> Delicious for Burrito<Filling> {}
+    /// fn eat_delicious_food<Food: Delicious>(_food: Food) {}
+    ///
+    /// fn will_type_error() {
+    ///   eat_delicious_food(Burrito { filling: Kale });
+    /// } //                                    ^--- The trait bound `Kale: Delicious`
+    ///   //                                         is not satisfied
+    /// ```
+    ///
+    /// Without calling this function, the error span will cover the entire argument expression.
+    ///
+    /// Before we do any of this logic, we recursively call `point_at_specific_expr_if_possible` on the parent
+    /// obligation. Hence we refine the `expr` "outwards-in" and bail at the first kind of expression/impl we don't recognize.
+    ///
+    /// This function returns a `Result<&Expr, &Expr>` - either way, it returns the `Expr` whose span should be
+    /// reported as an error. If it is `Ok`, then it means it refined successfull. If it is `Err`, then it may be
+    /// only a partial success - but it cannot be refined even further.
+    fn blame_specific_expr_if_possible_for_derived_predicate_obligation(
+        &self,
+        obligation: &traits::ImplDerivedObligationCause<'tcx>,
+        expr: &'tcx hir::Expr<'tcx>,
+    ) -> Result<&'tcx hir::Expr<'tcx>, &'tcx hir::Expr<'tcx>> {
+        // First, we attempt to refine the `expr` for our span using the parent obligation.
+        // If this cannot be done, then we are already stuck, so we stop early (hence the use
+        // of the `?` try operator here).
+        let expr = self.blame_specific_expr_if_possible_for_obligation_cause_code(
+            &*obligation.derived.parent_code,
+            expr,
+        )?;
+
+        // This is the "trait" (meaning, the predicate "proved" by this `impl`) which provides the `Self` type we care about.
+        // For the purposes of this function, we hope that it is a `struct` type, and that our current `expr` is a literal of
+        // that struct type.
+        let impl_trait_self_ref = if self.tcx.is_trait_alias(obligation.impl_or_alias_def_id) {
+            self.tcx.mk_trait_ref(
+                obligation.impl_or_alias_def_id,
+                ty::InternalSubsts::identity_for_item(self.tcx, obligation.impl_or_alias_def_id),
+            )
+        } else {
+            self.tcx
+                .impl_trait_ref(obligation.impl_or_alias_def_id)
+                .map(|impl_def| impl_def.skip_binder())
+                // It is possible that this is absent. In this case, we make no progress.
+                .ok_or(expr)?
+        };
+
+        // We only really care about the `Self` type itself, which we extract from the ref.
+        let impl_self_ty: Ty<'tcx> = impl_trait_self_ref.self_ty();
+
+        let impl_predicates: ty::GenericPredicates<'tcx> =
+            self.tcx.predicates_of(obligation.impl_or_alias_def_id);
+        let Some(impl_predicate_index) = obligation.impl_def_predicate_index else {
+            // We don't have the index, so we can only guess.
+            return Err(expr);
+        };
+
+        if impl_predicate_index >= impl_predicates.predicates.len() {
+            // This shouldn't happen, but since this is only a diagnostic improvement, avoid breaking things.
+            return Err(expr);
+        }
+        let relevant_broken_predicate: ty::PredicateKind<'tcx> =
+            impl_predicates.predicates[impl_predicate_index].0.kind().skip_binder();
+
+        match relevant_broken_predicate {
+            ty::PredicateKind::Clause(ty::Clause::Trait(broken_trait)) => {
+                // ...
+                self.blame_specific_part_of_expr_corresponding_to_generic_param(
+                    broken_trait.trait_ref.self_ty().into(),
+                    expr,
+                    impl_self_ty.into(),
+                )
+            }
+            _ => Err(expr),
+        }
+    }
+
+    /// Drills into `expr` to arrive at the equivalent location of `find_generic_param` in `in_ty`.
+    /// For example, given
+    /// - expr: `(Some(vec![1, 2, 3]), false)`
+    /// - param: `T`
+    /// - in_ty: `(Option<Vec<T>, bool)`
+    /// we would drill until we arrive at `vec![1, 2, 3]`.
+    ///
+    /// If successful, we return `Ok(refined_expr)`. If unsuccesful, we return `Err(partially_refined_expr`),
+    /// which will go as far as possible. For example, given `(foo(), false)` instead, we would drill to
+    /// `foo()` and then return `Err("foo()")`.
+    ///
+    /// This means that you can (and should) use the `?` try operator to chain multiple calls to this
+    /// function with different types, since you can only continue drilling the second time if you
+    /// succeeded the first time.
+    fn blame_specific_part_of_expr_corresponding_to_generic_param(
+        &self,
+        param: ty::GenericArg<'tcx>,
+        expr: &'tcx hir::Expr<'tcx>,
+        in_ty: ty::GenericArg<'tcx>,
+    ) -> Result<&'tcx hir::Expr<'tcx>, &'tcx hir::Expr<'tcx>> {
+        if param == in_ty {
+            // The types match exactly, so we have drilled as far as we can.
+            return Ok(expr);
+        }
+
+        let ty::GenericArgKind::Type(in_ty) = in_ty.unpack() else {
+            return Err(expr);
+        };
+
+        if let (
+            hir::ExprKind::AddrOf(_borrow_kind, _borrow_mutability, borrowed_expr),
+            ty::Ref(_ty_region, ty_ref_type, _ty_mutability),
+        ) = (&expr.kind, in_ty.kind())
+        {
+            // We can "drill into" the borrowed expression.
+            return self.blame_specific_part_of_expr_corresponding_to_generic_param(
+                param,
+                borrowed_expr,
+                (*ty_ref_type).into(),
+            );
+        }
+
+        if let (hir::ExprKind::Tup(expr_elements), ty::Tuple(in_ty_elements)) =
+            (&expr.kind, in_ty.kind())
+        {
+            if in_ty_elements.len() != expr_elements.len() {
+                return Err(expr);
+            }
+            // Find out which of `in_ty_elements` refer to `param`.
+            // FIXME: It may be better to take the first if there are multiple,
+            // just so that the error points to a smaller expression.
+            let Some((drill_expr, drill_ty)) = Self::is_iterator_singleton(expr_elements.iter().zip( in_ty_elements.iter()).filter(|(_expr_elem, in_ty_elem)| {
+                Self::find_param_in_ty((*in_ty_elem).into(), param)
+            })) else {
+                // The param is not mentioned, or it is mentioned in multiple indexes.
+                return Err(expr);
+            };
+
+            return self.blame_specific_part_of_expr_corresponding_to_generic_param(
+                param,
+                drill_expr,
+                drill_ty.into(),
+            );
+        }
+
+        if let (
+            hir::ExprKind::Struct(expr_struct_path, expr_struct_fields, _expr_struct_rest),
+            ty::Adt(in_ty_adt, in_ty_adt_generic_args),
+        ) = (&expr.kind, in_ty.kind())
+        {
+            // First, confirm that this struct is the same one as in the types, and if so,
+            // find the right variant.
+            let Res::Def(expr_struct_def_kind, expr_struct_def_id) = self.typeck_results.borrow().qpath_res(expr_struct_path, expr.hir_id) else {
+                return Err(expr);
+            };
+
+            let variant_def_id = match expr_struct_def_kind {
+                hir::def::DefKind::Struct => {
+                    if in_ty_adt.did() != expr_struct_def_id {
+                        // FIXME: Deal with type aliases?
+                        return Err(expr);
+                    }
+                    expr_struct_def_id
+                }
+                hir::def::DefKind::Variant => {
+                    // If this is a variant, its parent is the type definition.
+                    if in_ty_adt.did() != self.tcx.parent(expr_struct_def_id) {
+                        // FIXME: Deal with type aliases?
+                        return Err(expr);
+                    }
+                    expr_struct_def_id
+                }
+                _ => {
+                    return Err(expr);
+                }
+            };
+
+            // We need to know which of the generic parameters mentions our target param.
+            // We expect that at least one of them does, since it is expected to be mentioned.
+            let Some((drill_generic_index, generic_argument_type)) =
+                Self::is_iterator_singleton(
+                    in_ty_adt_generic_args.iter().enumerate().filter(
+                        |(_index, in_ty_generic)| {
+                            Self::find_param_in_ty(*in_ty_generic, param)
+                        },
+                    ),
+                ) else {
+                    return Err(expr);
+                };
+
+            let struct_generic_parameters: &ty::Generics = self.tcx.generics_of(in_ty_adt.did());
+            if drill_generic_index >= struct_generic_parameters.params.len() {
+                return Err(expr);
+            }
+
+            let param_to_point_at_in_struct = self.tcx.mk_param_from_def(
+                struct_generic_parameters.param_at(drill_generic_index, self.tcx),
+            );
+
+            // We make 3 steps:
+            // Suppose we have a type like
+            // ```ignore (just for demonstration)
+            // struct ExampleStruct<T> {
+            //   enabled: bool,
+            //   item: Option<(usize, T, bool)>,
+            // }
+            //
+            // f(ExampleStruct {
+            //   enabled: false,
+            //   item: Some((0, Box::new(String::new()), 1) }, true)),
+            // });
+            // ```
+            // Here, `f` is passed a `ExampleStruct<Box<String>>`, but it wants
+            // for `String: Copy`, which isn't true here.
+            //
+            // (1) First, we drill into `.item` and highlight that expression
+            // (2) Then we use the template type `Option<(usize, T, bool)>` to
+            //     drill into the `T`, arriving at a `Box<String>` expression.
+            // (3) Then we keep going, drilling into this expression using our
+            //     outer contextual information.
+
+            // (1) Find the (unique) field which mentions the type in our constraint:
+            let (field_expr, field_type) = self
+                .point_at_field_if_possible(
+                    in_ty_adt.did(),
+                    param_to_point_at_in_struct,
+                    variant_def_id,
+                    expr_struct_fields,
+                )
+                .ok_or(expr)?;
+
+            // (2) Continue drilling into the struct, ignoring the struct's
+            // generic argument types.
+            let expr = self.blame_specific_part_of_expr_corresponding_to_generic_param(
+                param_to_point_at_in_struct,
+                field_expr,
+                field_type.into(),
+            )?;
+
+            // (3) Continue drilling into the expression, having "passed
+            // through" the struct entirely.
+            return self.blame_specific_part_of_expr_corresponding_to_generic_param(
+                param,
+                expr,
+                generic_argument_type,
+            );
+        }
+
+        if let (
+            hir::ExprKind::Call(expr_callee, expr_args),
+            ty::Adt(in_ty_adt, in_ty_adt_generic_args),
+        ) = (&expr.kind, in_ty.kind())
+        {
+            let hir::ExprKind::Path(expr_callee_path) = &expr_callee.kind else {
+                // FIXME: This case overlaps with another one worth handling,
+                // which should happen above since it applies to non-ADTs:
+                // we can drill down into regular generic functions.
+                return Err(expr);
+            };
+            // This is (possibly) a constructor call, like `Some(...)` or `MyStruct(a, b, c)`.
+
+            let Res::Def(expr_struct_def_kind, expr_ctor_def_id) = self.typeck_results.borrow().qpath_res(expr_callee_path, expr_callee.hir_id) else {
+                return Err(expr);
+            };
+
+            let variant_def_id = match expr_struct_def_kind {
+                hir::def::DefKind::Ctor(hir::def::CtorOf::Struct, hir::def::CtorKind::Fn) => {
+                    if in_ty_adt.did() != self.tcx.parent(expr_ctor_def_id) {
+                        // FIXME: Deal with type aliases?
+                        return Err(expr);
+                    }
+                    self.tcx.parent(expr_ctor_def_id)
+                }
+                hir::def::DefKind::Ctor(hir::def::CtorOf::Variant, hir::def::CtorKind::Fn) => {
+                    // For a typical enum like
+                    // `enum Blah<T> { Variant(T) }`
+                    // we get the following resolutions:
+                    // - expr_ctor_def_id :::                                   DefId(0:29 ~ source_file[b442]::Blah::Variant::{constructor#0})
+                    // - self.tcx.parent(expr_ctor_def_id) :::                  DefId(0:28 ~ source_file[b442]::Blah::Variant)
+                    // - self.tcx.parent(self.tcx.parent(expr_ctor_def_id)) ::: DefId(0:26 ~ source_file[b442]::Blah)
+
+                    // Therefore, we need to go up once to obtain the variant and up twice to obtain the type.
+                    // Note that this pattern still holds even when we `use` a variant or `use` an enum type to rename it, or chain `use` expressions
+                    // together; this resolution is handled automatically by `qpath_res`.
+
+                    // FIXME: Deal with type aliases?
+                    if in_ty_adt.did() == self.tcx.parent(self.tcx.parent(expr_ctor_def_id)) {
+                        // The constructor definition refers to the "constructor" of the variant:
+                        // For example, `Some(5)` triggers this case.
+                        self.tcx.parent(expr_ctor_def_id)
+                    } else {
+                        // FIXME: Deal with type aliases?
+                        return Err(expr);
+                    }
+                }
+                _ => {
+                    return Err(expr);
+                }
+            };
+
+            // We need to know which of the generic parameters mentions our target param.
+            // We expect that at least one of them does, since it is expected to be mentioned.
+            let Some((drill_generic_index, generic_argument_type)) =
+                Self::is_iterator_singleton(
+                    in_ty_adt_generic_args.iter().enumerate().filter(
+                        |(_index, in_ty_generic)| {
+                            Self::find_param_in_ty(*in_ty_generic, param)
+                        },
+                    ),
+                ) else {
+                    return Err(expr);
+                };
+
+            let struct_generic_parameters: &ty::Generics = self.tcx.generics_of(in_ty_adt.did());
+            if drill_generic_index >= struct_generic_parameters.params.len() {
+                return Err(expr);
+            }
+
+            let param_to_point_at_in_struct = self.tcx.mk_param_from_def(
+                struct_generic_parameters.param_at(drill_generic_index, self.tcx),
+            );
+
+            // We make 3 steps:
+            // Suppose we have a type like
+            // ```ignore (just for demonstration)
+            // struct ExampleStruct<T> {
+            //   enabled: bool,
+            //   item: Option<(usize, T, bool)>,
+            // }
+            //
+            // f(ExampleStruct {
+            //   enabled: false,
+            //   item: Some((0, Box::new(String::new()), 1) }, true)),
+            // });
+            // ```
+            // Here, `f` is passed a `ExampleStruct<Box<String>>`, but it wants
+            // for `String: Copy`, which isn't true here.
+            //
+            // (1) First, we drill into `.item` and highlight that expression
+            // (2) Then we use the template type `Option<(usize, T, bool)>` to
+            //     drill into the `T`, arriving at a `Box<String>` expression.
+            // (3) Then we keep going, drilling into this expression using our
+            //     outer contextual information.
+
+            // (1) Find the (unique) field index which mentions the type in our constraint:
+            let Some((field_index, field_type)) = Self::is_iterator_singleton(
+                in_ty_adt
+                    .variant_with_id(variant_def_id)
+                    .fields
+                    .iter()
+                    .map(|field| field.ty(self.tcx, *in_ty_adt_generic_args))
+                    .enumerate()
+                    .filter(|(_index, field_type)| Self::find_param_in_ty((*field_type).into(), param))
+            ) else {
+                return Err(expr);
+            };
+
+            if field_index >= expr_args.len() {
+                return Err(expr);
+            }
+
+            // (2) Continue drilling into the struct, ignoring the struct's
+            // generic argument types.
+            let expr = self.blame_specific_part_of_expr_corresponding_to_generic_param(
+                param_to_point_at_in_struct,
+                &expr_args[field_index],
+                field_type.into(),
+            )?;
+
+            // (3) Continue drilling into the expression, having "passed
+            // through" the struct entirely.
+            return self.blame_specific_part_of_expr_corresponding_to_generic_param(
+                param,
+                expr,
+                generic_argument_type,
+            );
+        }
+
+        // At this point, none of the basic patterns matched.
+        // One major possibility which remains is that we have a function call.
+        // In this case, it's often possible to dive deeper into the call to find something to blame,
+        // but this is not always possible.
+
+        Err(expr)
+    }
+
+    // FIXME: This can be made into a private, non-impl function later.
+    /// Traverses the given ty (either a `ty::Ty` or a `ty::GenericArg`) and searches for references
+    /// to the given `param_to_point_at`. Returns `true` if it finds any use of the param.
+    pub fn find_param_in_ty(
+        ty: ty::GenericArg<'tcx>,
+        param_to_point_at: ty::GenericArg<'tcx>,
+    ) -> bool {
+        let mut walk = ty.walk();
+        while let Some(arg) = walk.next() {
+            if arg == param_to_point_at {
+                return true;
+            }
+            if let ty::GenericArgKind::Type(ty) = arg.unpack()
+                && let ty::Alias(ty::Projection, ..) = ty.kind()
+            {
+                // This logic may seem a bit strange, but typically when
+                // we have a projection type in a function signature, the
+                // argument that's being passed into that signature is
+                // not actually constraining that projection's substs in
+                // a meaningful way. So we skip it, and see improvements
+                // in some UI tests.
+                walk.skip_current_subtree();
+            }
+        }
+        false
+    }
+
+    // FIXME: This can be made into a private, non-impl function later.
+    /// Returns `Some(iterator.next())` if it has exactly one item, and `None` otherwise.
+    pub fn is_iterator_singleton<T>(mut iterator: impl Iterator<Item = T>) -> Option<T> {
+        match (iterator.next(), iterator.next()) {
+            (_, Some(_)) => None,
+            (first, _) => first,
+        }
+    }
+}
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs
index 2d841d53f..a46bdeb41 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs
@@ -10,7 +10,9 @@ use crate::{
 };
 use rustc_ast as ast;
 use rustc_data_structures::fx::FxIndexSet;
-use rustc_errors::{pluralize, Applicability, Diagnostic, DiagnosticId, MultiSpan};
+use rustc_errors::{
+    pluralize, Applicability, Diagnostic, DiagnosticId, ErrorGuaranteed, MultiSpan,
+};
 use rustc_hir as hir;
 use rustc_hir::def::{CtorOf, DefKind, Res};
 use rustc_hir::def_id::DefId;
@@ -25,8 +27,8 @@ use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKi
 use rustc_infer::infer::InferOk;
 use rustc_infer::infer::TypeTrace;
 use rustc_middle::ty::adjustment::AllowTwoPhase;
-use rustc_middle::ty::visit::TypeVisitable;
-use rustc_middle::ty::{self, DefIdTree, IsSuggestable, Ty, TypeSuperVisitable, TypeVisitor};
+use rustc_middle::ty::visit::TypeVisitableExt;
+use rustc_middle::ty::{self, DefIdTree, IsSuggestable, Ty};
 use rustc_session::Session;
 use rustc_span::symbol::{kw, Ident};
 use rustc_span::{self, sym, Span};
@@ -34,8 +36,6 @@ use rustc_trait_selection::traits::{self, ObligationCauseCode, SelectionContext}
 
 use std::iter;
 use std::mem;
-use std::ops::ControlFlow;
-use std::slice;
 
 impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     pub(in super::super) fn check_casts(&mut self) {
@@ -73,13 +73,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 let ty = self.typeck_results.borrow().expr_ty_adjusted(expr);
                 let ty = self.resolve_vars_if_possible(ty);
                 if ty.has_non_region_infer() {
-                    self.tcx.ty_error()
+                    self.tcx.ty_error_misc()
                 } else {
                     self.tcx.erase_regions(ty)
                 }
             };
             InlineAsmCtxt::new_in_fn(self.tcx, self.param_env, get_operand_ty)
-                .check_asm(asm, self.tcx.hir().local_def_id_to_hir_id(enclosing_id));
+                .check_asm(asm, enclosing_id);
         }
     }
 
@@ -101,7 +101,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
             let err_inputs = match tuple_arguments {
                 DontTupleArguments => err_inputs,
-                TupleArguments => vec![self.tcx.intern_tup(&err_inputs)],
+                TupleArguments => vec![self.tcx.mk_tup(&err_inputs)],
             };
 
             self.check_argument_types(
@@ -114,7 +114,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 tuple_arguments,
                 method.ok().map(|method| method.def_id),
             );
-            return self.tcx.ty_error();
+            return self.tcx.ty_error_misc();
         }
 
         let method = method.unwrap();
@@ -534,7 +534,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     .typeck_results
                     .borrow()
                     .expr_ty_adjusted_opt(*expr)
-                    .unwrap_or_else(|| tcx.ty_error());
+                    .unwrap_or_else(|| tcx.ty_error_misc());
                 (self.resolve_vars_if_possible(ty), normalize_span(expr.span))
             })
             .collect();
@@ -641,7 +641,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 && provided_arg_tys.len() == formal_and_expected_inputs.len() - 1 + tys.len()
             {
                 // Wrap up the N provided arguments starting at this position in a tuple.
-                let provided_as_tuple = tcx.mk_tup(
+                let provided_as_tuple = tcx.mk_tup_from_iter(
                     provided_arg_tys.iter().map(|(ty, _)| *ty).skip(mismatch_idx).take(tys.len()),
                 );
 
@@ -756,15 +756,20 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
 
         errors.drain_filter(|error| {
-                let Error::Invalid(provided_idx, expected_idx, Compatibility::Incompatible(Some(e))) = error else { return false };
-                let (provided_ty, provided_span) = provided_arg_tys[*provided_idx];
-                let trace = mk_trace(provided_span, formal_and_expected_inputs[*expected_idx], provided_ty);
-                if !matches!(trace.cause.as_failure_code(*e), FailureCode::Error0308(_)) {
-                    self.err_ctxt().report_and_explain_type_error(trace, *e).emit();
-                    return true;
-                }
-                false
-            });
+            let Error::Invalid(
+                provided_idx,
+                expected_idx,
+                Compatibility::Incompatible(Some(e)),
+            ) = error else { return false };
+            let (provided_ty, provided_span) = provided_arg_tys[*provided_idx];
+            let trace =
+                mk_trace(provided_span, formal_and_expected_inputs[*expected_idx], provided_ty);
+            if !matches!(trace.cause.as_failure_code(*e), FailureCode::Error0308(_)) {
+                self.err_ctxt().report_and_explain_type_error(trace, *e).emit();
+                return true;
+            }
+            false
+        });
 
         // We're done if we found errors, but we already emitted them.
         if errors.is_empty() {
@@ -798,6 +803,24 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 full_call_span,
                 format!("arguments to this {} are incorrect", call_name),
             );
+            if let (Some(callee_ty), hir::ExprKind::MethodCall(_, rcvr, _, _)) =
+                (callee_ty, &call_expr.kind)
+            {
+                // Type that would have accepted this argument if it hadn't been inferred earlier.
+                // FIXME: We leave an inference variable for now, but it'd be nice to get a more
+                // specific type to increase the accuracy of the diagnostic.
+                let expected = self.infcx.next_ty_var(TypeVariableOrigin {
+                    kind: TypeVariableOriginKind::MiscVariable,
+                    span: full_call_span,
+                });
+                self.point_at_expr_source_of_inferred_type(
+                    &mut err,
+                    rcvr,
+                    expected,
+                    callee_ty,
+                    provided_arg_span,
+                );
+            }
             // Call out where the function is defined
             self.label_fn_like(
                 &mut err,
@@ -847,7 +870,27 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
         let mut suggestion_text = SuggestionText::None;
 
+        let ty_to_snippet = |ty: Ty<'tcx>, expected_idx: ExpectedIdx| {
+            if ty.is_unit() {
+                "()".to_string()
+            } else if ty.is_suggestable(tcx, false) {
+                format!("/* {} */", ty)
+            } else if let Some(fn_def_id) = fn_def_id
+                && self.tcx.def_kind(fn_def_id).is_fn_like()
+                && let self_implicit =
+                    matches!(call_expr.kind, hir::ExprKind::MethodCall(..)) as usize
+                && let Some(arg) = self.tcx.fn_arg_names(fn_def_id)
+                    .get(expected_idx.as_usize() + self_implicit)
+                && arg.name != kw::SelfLower
+            {
+                format!("/* {} */", arg.name)
+            } else {
+                "/* value */".to_string()
+            }
+        };
+
         let mut errors = errors.into_iter().peekable();
+        let mut suggestions = vec![];
         while let Some(error) = errors.next() {
             match error {
                 Error::Invalid(provided_idx, expected_idx, compatibility) => {
@@ -888,12 +931,24 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         "".to_string()
                     };
                     labels
-                        .push((provided_span, format!("argument{} unexpected", provided_ty_name)));
-                    suggestion_text = match suggestion_text {
-                        SuggestionText::None => SuggestionText::Remove(false),
-                        SuggestionText::Remove(_) => SuggestionText::Remove(true),
-                        _ => SuggestionText::DidYouMean,
-                    };
+                        .push((provided_span, format!("unexpected argument{}", provided_ty_name)));
+                    let mut span = provided_span;
+                    if span.can_be_used_for_suggestions() {
+                        if arg_idx.index() > 0
+                        && let Some((_, prev)) = provided_arg_tys
+                            .get(ProvidedIdx::from_usize(arg_idx.index() - 1)
+                    ) {
+                        // Include previous comma
+                        span = prev.shrink_to_hi().to(span);
+                    }
+                        suggestions.push((span, String::new()));
+
+                        suggestion_text = match suggestion_text {
+                            SuggestionText::None => SuggestionText::Remove(false),
+                            SuggestionText::Remove(_) => SuggestionText::Remove(true),
+                            _ => SuggestionText::DidYouMean,
+                        };
+                    }
                 }
                 Error::Missing(expected_idx) => {
                     // If there are multiple missing arguments adjacent to each other,
@@ -1078,6 +1133,45 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             }
         }
 
+        // Incorporate the argument changes in the removal suggestion.
+        // When a type is *missing*, and the rest are additional, we want to suggest these with a
+        // multipart suggestion, but in order to do so we need to figure out *where* the arg that
+        // was provided but had the wrong type should go, because when looking at `expected_idx`
+        // that is the position in the argument list in the definition, while `provided_idx` will
+        // not be present. So we have to look at what the *last* provided position was, and point
+        // one after to suggest the replacement. FIXME(estebank): This is hacky, and there's
+        // probably a better more involved change we can make to make this work.
+        // For example, if we have
+        // ```
+        // fn foo(i32, &'static str) {}
+        // foo((), (), ());
+        // ```
+        // what should be suggested is
+        // ```
+        // foo(/* i32 */, /* &str */);
+        // ```
+        // which includes the replacement of the first two `()` for the correct type, and the
+        // removal of the last `()`.
+        let mut prev = -1;
+        for (expected_idx, provided_idx) in matched_inputs.iter_enumerated() {
+            // We want to point not at the *current* argument expression index, but rather at the
+            // index position where it *should have been*, which is *after* the previous one.
+            if let Some(provided_idx) = provided_idx {
+                prev = provided_idx.index() as i64;
+            }
+            let idx = ProvidedIdx::from_usize((prev + 1) as usize);
+            if let None = provided_idx
+                && let Some((_, arg_span)) = provided_arg_tys.get(idx)
+            {
+                // There is a type that was *not* found anywhere, so it isn't a move, but a
+                // replacement and we look at what type it should have been. This will allow us
+                // To suggest a multipart suggestion when encountering `foo(1, "")` where the def
+                // was `fn foo(())`.
+                let (_, expected_ty) = formal_and_expected_inputs[expected_idx];
+                suggestions.push((*arg_span, ty_to_snippet(expected_ty, expected_idx)));
+            }
+        }
+
         // If we have less than 5 things to say, it would be useful to call out exactly what's wrong
         if labels.len() <= 5 {
             for (span, label) in labels {
@@ -1095,7 +1189,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 Some(format!("provide the argument{}", if plural { "s" } else { "" }))
             }
             SuggestionText::Remove(plural) => {
-                Some(format!("remove the extra argument{}", if plural { "s" } else { "" }))
+                err.multipart_suggestion(
+                    &format!("remove the extra argument{}", if plural { "s" } else { "" }),
+                    suggestions,
+                    Applicability::HasPlaceholders,
+                );
+                None
             }
             SuggestionText::Swap => Some("swap these arguments".to_string()),
             SuggestionText::Reorder => Some("reorder these arguments".to_string()),
@@ -1134,20 +1233,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 } else {
                     // Propose a placeholder of the correct type
                     let (_, expected_ty) = formal_and_expected_inputs[expected_idx];
-                    if expected_ty.is_unit() {
-                        "()".to_string()
-                    } else if expected_ty.is_suggestable(tcx, false) {
-                        format!("/* {} */", expected_ty)
-                    } else if let Some(fn_def_id) = fn_def_id
-                        && self.tcx.def_kind(fn_def_id).is_fn_like()
-                        && let self_implicit = matches!(call_expr.kind, hir::ExprKind::MethodCall(..)) as usize
-                        && let Some(arg) = self.tcx.fn_arg_names(fn_def_id).get(expected_idx.as_usize() + self_implicit)
-                        && arg.name != kw::SelfLower
-                    {
-                        format!("/* {} */", arg.name)
-                    } else {
-                        "/* value */".to_string()
-                    }
+                    ty_to_snippet(expected_ty, expected_idx)
                 };
                 suggestion += &suggestion_text;
             }
@@ -1201,7 +1287,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 opt_ty.unwrap_or_else(|| self.next_float_var())
             }
             ast::LitKind::Bool(_) => tcx.types.bool,
-            ast::LitKind::Err => tcx.ty_error(),
+            ast::LitKind::Err => tcx.ty_error_misc(),
         }
     }
 
@@ -1209,15 +1295,15 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         &self,
         qpath: &QPath<'_>,
         hir_id: hir::HirId,
-    ) -> Option<(&'tcx ty::VariantDef, Ty<'tcx>)> {
+    ) -> Result<(&'tcx ty::VariantDef, Ty<'tcx>), ErrorGuaranteed> {
         let path_span = qpath.span();
         let (def, ty) = self.finish_resolving_struct_path(qpath, path_span, hir_id);
         let variant = match def {
             Res::Err => {
-                self.set_tainted_by_errors(
-                    self.tcx.sess.delay_span_bug(path_span, "`Res::Err` but no error emitted"),
-                );
-                return None;
+                let guar =
+                    self.tcx.sess.delay_span_bug(path_span, "`Res::Err` but no error emitted");
+                self.set_tainted_by_errors(guar);
+                return Err(guar);
             }
             Res::Def(DefKind::Variant, _) => match ty.normalized.ty_adt_def() {
                 Some(adt) => {
@@ -1245,28 +1331,26 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // Check bounds on type arguments used in the path.
             self.add_required_obligations_for_hir(path_span, did, substs, hir_id);
 
-            Some((variant, ty.normalized))
+            Ok((variant, ty.normalized))
         } else {
-            match ty.normalized.kind() {
-                ty::Error(_) => {
+            Err(match *ty.normalized.kind() {
+                ty::Error(guar) => {
                     // E0071 might be caused by a spelling error, which will have
                     // already caused an error message and probably a suggestion
                     // elsewhere. Refrain from emitting more unhelpful errors here
                     // (issue #88844).
+                    guar
                 }
-                _ => {
-                    struct_span_err!(
-                        self.tcx.sess,
-                        path_span,
-                        E0071,
-                        "expected struct, variant or union type, found {}",
-                        ty.normalized.sort_string(self.tcx)
-                    )
-                    .span_label(path_span, "not a struct")
-                    .emit();
-                }
-            }
-            None
+                _ => struct_span_err!(
+                    self.tcx.sess,
+                    path_span,
+                    E0071,
+                    "expected struct, variant or union type, found {}",
+                    ty.normalized.sort_string(self.tcx)
+                )
+                .span_label(path_span, "not a struct")
+                .emit(),
+            })
         }
     }
 
@@ -1313,11 +1397,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         // Does the expected pattern type originate from an expression and what is the span?
         let (origin_expr, ty_span) = match (decl.ty, decl.init) {
-            (Some(ty), _) => (false, Some(ty.span)), // Bias towards the explicit user type.
+            (Some(ty), _) => (None, Some(ty.span)), // Bias towards the explicit user type.
             (_, Some(init)) => {
-                (true, Some(init.span.find_ancestor_inside(decl.span).unwrap_or(init.span)))
+                (Some(init), Some(init.span.find_ancestor_inside(decl.span).unwrap_or(init.span)))
             } // No explicit type; so use the scrutinee.
-            _ => (false, None), // We have `let $pat;`, so the expected type is unconstrained.
+            _ => (None, None), // We have `let $pat;`, so the expected type is unconstrained.
         };
 
         // Type check the pattern. Override if necessary to avoid knock-on errors.
@@ -1422,11 +1506,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let coerce = if blk.targeted_by_break {
             CoerceMany::new(coerce_to_ty)
         } else {
-            let tail_expr: &[&hir::Expr<'_>] = match tail_expr {
-                Some(e) => slice::from_ref(e),
-                None => &[],
-            };
-            CoerceMany::with_coercion_sites(coerce_to_ty, tail_expr)
+            CoerceMany::with_coercion_sites(coerce_to_ty, blk.expr.as_slice())
         };
 
         let prev_diverges = self.diverges.get();
@@ -1630,9 +1710,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         pat: &'tcx hir::Pat<'tcx>,
         ty: Ty<'tcx>,
     ) {
-        if ty.references_error() {
+        if let Err(guar) = ty.error_reported() {
             // Override the types everywhere with `err()` to avoid knock on errors.
-            let err = self.tcx.ty_error();
+            let err = self.tcx.ty_error(guar);
             self.write_ty(hir_id, err);
             self.write_ty(pat.hir_id, err);
             let local_ty = LocalTy { decl_ty: err, revealed_ty: err };
@@ -1652,7 +1732,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         match *qpath {
             QPath::Resolved(ref maybe_qself, ref path) => {
                 let self_ty = maybe_qself.as_ref().map(|qself| self.to_ty(qself).raw);
-                let ty = self.astconv().res_to_ty(self_ty, path, true);
+                let ty = self.astconv().res_to_ty(self_ty, path, hir_id, true);
                 (path.res, self.handle_raw_ty(path_span, ty))
             }
             QPath::TypeRelative(ref qself, ref segment) => {
@@ -1661,7 +1741,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 let result = self
                     .astconv()
                     .associated_path_to_ty(hir_id, path_span, ty.raw, qself, segment, true);
-                let ty = result.map(|(ty, _, _)| ty).unwrap_or_else(|_| self.tcx().ty_error());
+                let ty =
+                    result.map(|(ty, _, _)| ty).unwrap_or_else(|guar| self.tcx().ty_error(guar));
                 let ty = self.handle_raw_ty(path_span, ty);
                 let result = result.map(|(_, kind, def_id)| (kind, def_id));
 
@@ -1739,353 +1820,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
     }
 
-    fn adjust_fulfillment_error_for_expr_obligation(
-        &self,
-        error: &mut traits::FulfillmentError<'tcx>,
-    ) -> bool {
-        let (traits::ExprItemObligation(def_id, hir_id, idx) | traits::ExprBindingObligation(def_id, _, hir_id, idx))
-            = *error.obligation.cause.code().peel_derives() else { return false; };
-        let hir = self.tcx.hir();
-        let hir::Node::Expr(expr) = hir.get(hir_id) else { return false; };
-
-        let Some(unsubstituted_pred) =
-            self.tcx.predicates_of(def_id).instantiate_identity(self.tcx).predicates.into_iter().nth(idx)
-            else { return false; };
-
-        let generics = self.tcx.generics_of(def_id);
-        let predicate_substs = match unsubstituted_pred.kind().skip_binder() {
-            ty::PredicateKind::Clause(ty::Clause::Trait(pred)) => pred.trait_ref.substs,
-            ty::PredicateKind::Clause(ty::Clause::Projection(pred)) => pred.projection_ty.substs,
-            _ => ty::List::empty(),
-        };
-
-        let find_param_matching = |matches: &dyn Fn(&ty::ParamTy) -> bool| {
-            predicate_substs.types().find_map(|ty| {
-                ty.walk().find_map(|arg| {
-                    if let ty::GenericArgKind::Type(ty) = arg.unpack()
-                        && let ty::Param(param_ty) = ty.kind()
-                        && matches(param_ty)
-                    {
-                        Some(arg)
-                    } else {
-                        None
-                    }
-                })
-            })
-        };
-
-        // Prefer generics that are local to the fn item, since these are likely
-        // to be the cause of the unsatisfied predicate.
-        let mut param_to_point_at = find_param_matching(&|param_ty| {
-            self.tcx.parent(generics.type_param(param_ty, self.tcx).def_id) == def_id
-        });
-        // Fall back to generic that isn't local to the fn item. This will come
-        // from a trait or impl, for example.
-        let mut fallback_param_to_point_at = find_param_matching(&|param_ty| {
-            self.tcx.parent(generics.type_param(param_ty, self.tcx).def_id) != def_id
-                && param_ty.name != rustc_span::symbol::kw::SelfUpper
-        });
-        // Finally, the `Self` parameter is possibly the reason that the predicate
-        // is unsatisfied. This is less likely to be true for methods, because
-        // method probe means that we already kinda check that the predicates due
-        // to the `Self` type are true.
-        let mut self_param_to_point_at =
-            find_param_matching(&|param_ty| param_ty.name == rustc_span::symbol::kw::SelfUpper);
-
-        // Finally, for ambiguity-related errors, we actually want to look
-        // for a parameter that is the source of the inference type left
-        // over in this predicate.
-        if let traits::FulfillmentErrorCode::CodeAmbiguity = error.code {
-            fallback_param_to_point_at = None;
-            self_param_to_point_at = None;
-            param_to_point_at =
-                self.find_ambiguous_parameter_in(def_id, error.root_obligation.predicate);
-        }
-
-        if self.closure_span_overlaps_error(error, expr.span) {
-            return false;
-        }
-
-        match &expr.kind {
-            hir::ExprKind::Path(qpath) => {
-                if let hir::Node::Expr(hir::Expr {
-                    kind: hir::ExprKind::Call(callee, args),
-                    hir_id: call_hir_id,
-                    span: call_span,
-                    ..
-                }) = hir.get_parent(expr.hir_id)
-                    && callee.hir_id == expr.hir_id
-                {
-                    if self.closure_span_overlaps_error(error, *call_span) {
-                        return false;
-                    }
-
-                    for param in
-                        [param_to_point_at, fallback_param_to_point_at, self_param_to_point_at]
-                        .into_iter()
-                        .flatten()
-                    {
-                        if self.point_at_arg_if_possible(
-                                error,
-                                def_id,
-                                param,
-                                *call_hir_id,
-                                callee.span,
-                                None,
-                                args,
-                            )
-                        {
-                            return true;
-                        }
-                    }
-                }
-                // Notably, we only point to params that are local to the
-                // item we're checking, since those are the ones we are able
-                // to look in the final `hir::PathSegment` for. Everything else
-                // would require a deeper search into the `qpath` than I think
-                // is worthwhile.
-                if let Some(param_to_point_at) = param_to_point_at
-                    && self.point_at_path_if_possible(error, def_id, param_to_point_at, qpath)
-                {
-                    return true;
-                }
-            }
-            hir::ExprKind::MethodCall(segment, receiver, args, ..) => {
-                for param in [param_to_point_at, fallback_param_to_point_at, self_param_to_point_at]
-                    .into_iter()
-                    .flatten()
-                {
-                    if self.point_at_arg_if_possible(
-                        error,
-                        def_id,
-                        param,
-                        hir_id,
-                        segment.ident.span,
-                        Some(receiver),
-                        args,
-                    ) {
-                        return true;
-                    }
-                }
-                if let Some(param_to_point_at) = param_to_point_at
-                    && self.point_at_generic_if_possible(error, def_id, param_to_point_at, segment)
-                {
-                    return true;
-                }
-            }
-            hir::ExprKind::Struct(qpath, fields, ..) => {
-                if let Res::Def(DefKind::Struct | DefKind::Variant, variant_def_id) =
-                    self.typeck_results.borrow().qpath_res(qpath, hir_id)
-                {
-                    for param in
-                        [param_to_point_at, fallback_param_to_point_at, self_param_to_point_at]
-                    {
-                        if let Some(param) = param
-                            && self.point_at_field_if_possible(
-                                error,
-                                def_id,
-                                param,
-                                variant_def_id,
-                                fields,
-                            )
-                        {
-                            return true;
-                        }
-                    }
-                }
-                if let Some(param_to_point_at) = param_to_point_at
-                    && self.point_at_path_if_possible(error, def_id, param_to_point_at, qpath)
-                {
-                    return true;
-                }
-            }
-            _ => {}
-        }
-
-        false
-    }
-
-    fn closure_span_overlaps_error(
-        &self,
-        error: &traits::FulfillmentError<'tcx>,
-        span: Span,
-    ) -> bool {
-        if let traits::FulfillmentErrorCode::CodeSelectionError(
-            traits::SelectionError::OutputTypeParameterMismatch(_, expected, _),
-        ) = error.code
-            && let ty::Closure(def_id, _) | ty::Generator(def_id, ..) = expected.skip_binder().self_ty().kind()
-            && span.overlaps(self.tcx.def_span(*def_id))
-        {
-            true
-        } else {
-            false
-        }
-    }
-
-    fn point_at_arg_if_possible(
-        &self,
-        error: &mut traits::FulfillmentError<'tcx>,
-        def_id: DefId,
-        param_to_point_at: ty::GenericArg<'tcx>,
-        call_hir_id: hir::HirId,
-        callee_span: Span,
-        receiver: Option<&'tcx hir::Expr<'tcx>>,
-        args: &'tcx [hir::Expr<'tcx>],
-    ) -> bool {
-        let ty = self.tcx.type_of(def_id);
-        if !ty.is_fn() {
-            return false;
-        }
-        let sig = ty.fn_sig(self.tcx).skip_binder();
-        let args_referencing_param: Vec<_> = sig
-            .inputs()
-            .iter()
-            .enumerate()
-            .filter(|(_, ty)| find_param_in_ty(**ty, param_to_point_at))
-            .collect();
-        // If there's one field that references the given generic, great!
-        if let [(idx, _)] = args_referencing_param.as_slice()
-            && let Some(arg) = receiver
-                .map_or(args.get(*idx), |rcvr| if *idx == 0 { Some(rcvr) } else { args.get(*idx - 1) }) {
-            error.obligation.cause.span = arg.span.find_ancestor_in_same_ctxt(error.obligation.cause.span).unwrap_or(arg.span);
-            error.obligation.cause.map_code(|parent_code| {
-                ObligationCauseCode::FunctionArgumentObligation {
-                    arg_hir_id: arg.hir_id,
-                    call_hir_id,
-                    parent_code,
-                }
-            });
-            return true;
-        } else if args_referencing_param.len() > 0 {
-            // If more than one argument applies, then point to the callee span at least...
-            // We have chance to fix this up further in `point_at_generics_if_possible`
-            error.obligation.cause.span = callee_span;
-        }
-
-        false
-    }
-
-    fn point_at_field_if_possible(
-        &self,
-        error: &mut traits::FulfillmentError<'tcx>,
-        def_id: DefId,
-        param_to_point_at: ty::GenericArg<'tcx>,
-        variant_def_id: DefId,
-        expr_fields: &[hir::ExprField<'tcx>],
-    ) -> bool {
-        let def = self.tcx.adt_def(def_id);
-
-        let identity_substs = ty::InternalSubsts::identity_for_item(self.tcx, def_id);
-        let fields_referencing_param: Vec<_> = def
-            .variant_with_id(variant_def_id)
-            .fields
-            .iter()
-            .filter(|field| {
-                let field_ty = field.ty(self.tcx, identity_substs);
-                find_param_in_ty(field_ty, param_to_point_at)
-            })
-            .collect();
-
-        if let [field] = fields_referencing_param.as_slice() {
-            for expr_field in expr_fields {
-                // Look for the ExprField that matches the field, using the
-                // same rules that check_expr_struct uses for macro hygiene.
-                if self.tcx.adjust_ident(expr_field.ident, variant_def_id) == field.ident(self.tcx)
-                {
-                    error.obligation.cause.span = expr_field
-                        .expr
-                        .span
-                        .find_ancestor_in_same_ctxt(error.obligation.cause.span)
-                        .unwrap_or(expr_field.span);
-                    return true;
-                }
-            }
-        }
-
-        false
-    }
-
-    fn point_at_path_if_possible(
-        &self,
-        error: &mut traits::FulfillmentError<'tcx>,
-        def_id: DefId,
-        param: ty::GenericArg<'tcx>,
-        qpath: &QPath<'tcx>,
-    ) -> bool {
-        match qpath {
-            hir::QPath::Resolved(_, path) => {
-                if let Some(segment) = path.segments.last()
-                    && self.point_at_generic_if_possible(error, def_id, param, segment)
-                {
-                    return true;
-                }
-            }
-            hir::QPath::TypeRelative(_, segment) => {
-                if self.point_at_generic_if_possible(error, def_id, param, segment) {
-                    return true;
-                }
-            }
-            _ => {}
-        }
-
-        false
-    }
-
-    fn point_at_generic_if_possible(
-        &self,
-        error: &mut traits::FulfillmentError<'tcx>,
-        def_id: DefId,
-        param_to_point_at: ty::GenericArg<'tcx>,
-        segment: &hir::PathSegment<'tcx>,
-    ) -> bool {
-        let own_substs = self
-            .tcx
-            .generics_of(def_id)
-            .own_substs(ty::InternalSubsts::identity_for_item(self.tcx, def_id));
-        let Some((index, _)) = own_substs
-            .iter()
-            .filter(|arg| matches!(arg.unpack(), ty::GenericArgKind::Type(_)))
-            .enumerate()
-            .find(|(_, arg)| **arg == param_to_point_at) else { return false };
-        let Some(arg) = segment
-            .args()
-            .args
-            .iter()
-            .filter(|arg| matches!(arg, hir::GenericArg::Type(_)))
-            .nth(index) else { return false; };
-        error.obligation.cause.span = arg
-            .span()
-            .find_ancestor_in_same_ctxt(error.obligation.cause.span)
-            .unwrap_or(arg.span());
-        true
-    }
-
-    fn find_ambiguous_parameter_in<T: TypeVisitable<'tcx>>(
-        &self,
-        item_def_id: DefId,
-        t: T,
-    ) -> Option<ty::GenericArg<'tcx>> {
-        struct FindAmbiguousParameter<'a, 'tcx>(&'a FnCtxt<'a, 'tcx>, DefId);
-        impl<'tcx> TypeVisitor<'tcx> for FindAmbiguousParameter<'_, 'tcx> {
-            type BreakTy = ty::GenericArg<'tcx>;
-            fn visit_ty(&mut self, ty: Ty<'tcx>) -> std::ops::ControlFlow<Self::BreakTy> {
-                if let Some(origin) = self.0.type_var_origin(ty)
-                    && let TypeVariableOriginKind::TypeParameterDefinition(_, Some(def_id)) =
-                        origin.kind
-                    && let generics = self.0.tcx.generics_of(self.1)
-                    && let Some(index) = generics.param_def_id_to_index(self.0.tcx, def_id)
-                    && let Some(subst) = ty::InternalSubsts::identity_for_item(self.0.tcx, self.1)
-                        .get(index as usize)
-                {
-                    ControlFlow::Break(*subst)
-                } else {
-                    ty.super_visit_with(self)
-                }
-            }
-        }
-        t.visit_with(&mut FindAmbiguousParameter(self, item_def_id)).break_value()
-    }
-
     fn label_fn_like(
         &self,
         err: &mut Diagnostic,
@@ -2114,7 +1848,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             match *callee_ty.kind() {
                 ty::Param(param) => {
                     let param =
-                        self.tcx.generics_of(self.body_id.owner).type_param(&param, self.tcx);
+                        self.tcx.generics_of(self.body_id).type_param(&param, self.tcx);
                     if param.kind.is_synthetic() {
                         // if it's `impl Fn() -> ..` then just fall down to the def-id based logic
                         def_id = param.def_id;
@@ -2123,7 +1857,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         // and point at that.
                         let instantiated = self
                             .tcx
-                            .explicit_predicates_of(self.body_id.owner)
+                            .explicit_predicates_of(self.body_id)
                             .instantiate_identity(self.tcx);
                         // FIXME(compiler-errors): This could be problematic if something has two
                         // fn-like predicates with different args, but callable types really never
@@ -2200,8 +1934,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 spans.push_span_label(param.span, "");
             }
 
-            let def_kind = self.tcx.def_kind(def_id);
-            err.span_note(spans, &format!("{} defined here", def_kind.descr(def_id)));
+            err.span_note(spans, &format!("{} defined here", self.tcx.def_descr(def_id)));
         } else if let Some(hir::Node::Expr(e)) = self.tcx.hir().get_if_local(def_id)
             && let hir::ExprKind::Closure(hir::Closure { body, .. }) = &e.kind
         {
@@ -2214,31 +1947,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             };
             err.span_note(span, &format!("{} defined here", kind));
         } else {
-            let def_kind = self.tcx.def_kind(def_id);
             err.span_note(
                 self.tcx.def_span(def_id),
-                &format!("{} defined here", def_kind.descr(def_id)),
+                &format!("{} defined here", self.tcx.def_descr(def_id)),
             );
         }
     }
 }
-
-fn find_param_in_ty<'tcx>(ty: Ty<'tcx>, param_to_point_at: ty::GenericArg<'tcx>) -> bool {
-    let mut walk = ty.walk();
-    while let Some(arg) = walk.next() {
-        if arg == param_to_point_at {
-            return true;
-        } else if let ty::GenericArgKind::Type(ty) = arg.unpack()
-            && let ty::Alias(ty::Projection, ..) = ty.kind()
-        {
-            // This logic may seem a bit strange, but typically when
-            // we have a projection type in a function signature, the
-            // argument that's being passed into that signature is
-            // not actually constraining that projection's substs in
-            // a meaningful way. So we skip it, and see improvements
-            // in some UI tests.
-            walk.skip_current_subtree();
-        }
-    }
-    false
-}
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs
index 428fde642..1dea3e6f9 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs
@@ -1,4 +1,5 @@
 mod _impl;
+mod adjust_fulfillment_errors;
 mod arg_matrix;
 mod checks;
 mod suggestions;
@@ -10,14 +11,14 @@ pub use suggestions::*;
 use crate::coercion::DynamicCoerceMany;
 use crate::{Diverges, EnclosingBreakables, Inherited};
 use rustc_hir as hir;
-use rustc_hir::def_id::DefId;
+use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir_analysis::astconv::AstConv;
 use rustc_infer::infer;
 use rustc_infer::infer::error_reporting::TypeErrCtxt;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
 use rustc_middle::infer::unify_key::{ConstVariableOrigin, ConstVariableOriginKind};
 use rustc_middle::ty::subst::GenericArgKind;
-use rustc_middle::ty::{self, Const, Ty, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, Const, Ty, TyCtxt, TypeVisitableExt};
 use rustc_session::Session;
 use rustc_span::symbol::Ident;
 use rustc_span::{self, Span, DUMMY_SP};
@@ -38,7 +39,7 @@ use std::ops::Deref;
 /// [`ItemCtxt`]: rustc_hir_analysis::collect::ItemCtxt
 /// [`InferCtxt`]: infer::InferCtxt
 pub struct FnCtxt<'a, 'tcx> {
-    pub(super) body_id: hir::HirId,
+    pub(super) body_id: LocalDefId,
 
     /// The parameter environment used for proving trait obligations
     /// in this function. This can change when we descend into
@@ -117,7 +118,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     pub fn new(
         inh: &'a Inherited<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
-        body_id: hir::HirId,
+        body_id: LocalDefId,
     ) -> FnCtxt<'a, 'tcx> {
         FnCtxt {
             body_id,
@@ -204,7 +205,7 @@ impl<'a, 'tcx> AstConv<'tcx> for FnCtxt<'a, 'tcx> {
     }
 
     fn item_def_id(&self) -> DefId {
-        self.body_id.owner.to_def_id()
+        self.body_id.to_def_id()
     }
 
     fn get_type_parameter_bounds(
@@ -288,7 +289,7 @@ impl<'a, 'tcx> AstConv<'tcx> for FnCtxt<'a, 'tcx> {
         item_segment: &hir::PathSegment<'_>,
         poly_trait_ref: ty::PolyTraitRef<'tcx>,
     ) -> Ty<'tcx> {
-        let trait_ref = self.replace_bound_vars_with_fresh_vars(
+        let trait_ref = self.instantiate_binder_with_fresh_vars(
             span,
             infer::LateBoundRegionConversionTime::AssocTypeProjection(item_def_id),
             poly_trait_ref,
@@ -324,6 +325,10 @@ impl<'a, 'tcx> AstConv<'tcx> for FnCtxt<'a, 'tcx> {
         let ty = if !ty.has_escaping_bound_vars() { self.normalize(span, ty) } else { ty };
         self.write_ty(hir_id, ty)
     }
+
+    fn infcx(&self) -> Option<&infer::InferCtxt<'tcx>> {
+        Some(&self.infcx)
+    }
 }
 
 /// Represents a user-provided type in the raw form (never normalized).
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
index 4d673ac91..c49621b7c 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
@@ -1,6 +1,7 @@
 use super::FnCtxt;
 
 use crate::errors::{AddReturnTypeSuggestion, ExpectedReturnTypeLabel};
+use crate::fluent_generated as fluent;
 use crate::method::probe::{IsSuggestion, Mode, ProbeScope};
 use rustc_ast::util::parser::{ExprPrecedence, PREC_POSTFIX};
 use rustc_errors::{Applicability, Diagnostic, MultiSpan};
@@ -13,9 +14,10 @@ use rustc_hir::{
 use rustc_hir_analysis::astconv::AstConv;
 use rustc_infer::traits::{self, StatementAsExpression};
 use rustc_middle::lint::in_external_macro;
+use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{
     self, suggest_constraining_type_params, Binder, DefIdTree, IsSuggestable, ToPredicate, Ty,
-    TypeVisitable,
+    TypeVisitableExt,
 };
 use rustc_session::errors::ExprParenthesesNeeded;
 use rustc_span::source_map::Spanned;
@@ -31,7 +33,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         self.typeck_results
             .borrow()
             .liberated_fn_sigs()
-            .get(self.tcx.hir().parent_id(self.body_id))
+            .get(self.tcx.hir().local_def_id_to_hir_id(self.body_id))
             .copied()
     }
 
@@ -119,7 +121,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 DefIdOrName::DefId(def_id) => match self.tcx.def_kind(def_id) {
                     DefKind::Ctor(CtorOf::Struct, _) => "construct this tuple struct".to_string(),
                     DefKind::Ctor(CtorOf::Variant, _) => "construct this tuple variant".to_string(),
-                    kind => format!("call this {}", kind.descr(def_id)),
+                    kind => format!("call this {}", self.tcx.def_kind_descr(kind, def_id)),
                 },
                 DefIdOrName::Name(name) => format!("call this {name}"),
             };
@@ -164,7 +166,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         &self,
         ty: Ty<'tcx>,
     ) -> Option<(DefIdOrName, Ty<'tcx>, Vec<Ty<'tcx>>)> {
-        self.err_ctxt().extract_callable_info(self.body_id, self.param_env, ty)
+        let body_hir_id = self.tcx.hir().local_def_id_to_hir_id(self.body_id);
+        self.err_ctxt().extract_callable_info(body_hir_id, self.param_env, ty)
     }
 
     pub fn suggest_two_fn_call(
@@ -337,7 +340,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     CtorOf::Variant => "an enum variant",
                 }));
             } else {
-                let descr = kind.descr(def_id);
+                let descr = self.tcx.def_kind_descr(kind, def_id);
                 err.span_label(sp, format!("{descr} `{name}` defined here"));
             }
             return true;
@@ -413,11 +416,16 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 if let ty::Adt(adt, _) = peeled.kind()
                     && Some(adt.did()) == self.tcx.lang_items().string()
                 {
+                    let sugg = if ref_cnt == 0 {
+                        ".as_deref()"
+                    } else {
+                        ".map(|x| x.as_str())"
+                    };
                     err.span_suggestion_verbose(
                         expr.span.shrink_to_hi(),
-                        "try converting the passed type into a `&str`",
-                        format!(".map(|x| &*{}x)", "*".repeat(ref_cnt)),
-                        Applicability::MaybeIncorrect,
+                        fluent::hir_typeck_convert_to_str,
+                        sugg,
+                        Applicability::MachineApplicable,
                     );
                     return true;
                 }
@@ -681,7 +689,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 return true;
             }
             &hir::FnRetTy::DefaultReturn(span) if expected.is_unit() => {
-                if found.is_suggestable(self.tcx, false) {
+                if let Some(found) = found.make_suggestable(self.tcx, false) {
                     err.subdiagnostic(AddReturnTypeSuggestion::Add { span, found: found.to_string() });
                     return true;
                 } else if let ty::Closure(_, substs) = found.kind()
@@ -698,10 +706,38 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 }
             }
             hir::FnRetTy::Return(ty) => {
+                let span = ty.span;
+
+                if let hir::TyKind::OpaqueDef(item_id, ..) = ty.kind
+                && let hir::Node::Item(hir::Item {
+                    kind: hir::ItemKind::OpaqueTy(op_ty),
+                    ..
+                }) = self.tcx.hir().get(item_id.hir_id())
+                && let hir::OpaqueTy {
+                    bounds: [bound], ..
+                } = op_ty
+                && let hir::GenericBound::LangItemTrait(
+                    hir::LangItem::Future, _, _, generic_args) = bound
+                && let hir::GenericArgs { bindings: [ty_binding], .. } = generic_args
+                && let hir::TypeBinding { kind, .. } = ty_binding
+                && let hir::TypeBindingKind::Equality { term } = kind
+                && let hir::Term::Ty(term_ty) = term {
+                    // Check if async function's return type was omitted.
+                    // Don't emit suggestions if the found type is `impl Future<...>`.
+                    debug!("suggest_missing_return_type: found = {:?}", found);
+                    if found.is_suggestable(self.tcx, false) {
+                        if term_ty.span.is_empty() {
+                            err.subdiagnostic(AddReturnTypeSuggestion::Add { span, found: found.to_string() });
+                            return true;
+                        } else {
+                            err.subdiagnostic(ExpectedReturnTypeLabel::Other { span, expected });
+                        }
+                    }
+                }
+
                 // Only point to return type if the expected type is the return type, as if they
                 // are not, the expectation must have been caused by something else.
                 debug!("suggest_missing_return_type: return type {:?} node {:?}", ty, ty.kind);
-                let span = ty.span;
                 let ty = self.astconv().ast_ty_to_ty(ty);
                 debug!("suggest_missing_return_type: return type {:?}", ty);
                 debug!("suggest_missing_return_type: expected type {:?}", ty);
@@ -980,7 +1016,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             let expr_inner_ty = substs.type_at(0);
             let expected_inner_ty = expected_substs.type_at(0);
             if let ty::Ref(_, ty, hir::Mutability::Not) = expr_inner_ty.kind()
-                && self.can_eq(self.param_env, *ty, expected_inner_ty).is_ok()
+                && self.can_eq(self.param_env, *ty, expected_inner_ty)
             {
                 let def_path = self.tcx.def_path_str(adt_def.did());
                 if self.type_is_copy_modulo_regions(self.param_env, *ty, expr.span) {
@@ -1019,7 +1055,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         if let Some(result_did) = self.tcx.get_diagnostic_item(sym::Result)
             && adt_def.did() == result_did
             // Check that the error types are equal
-            && self.can_eq(self.param_env, substs.type_at(1), expected_substs.type_at(1)).is_ok()
+            && self.can_eq(self.param_env, substs.type_at(1), expected_substs.type_at(1))
         {
             return suggest_copied_or_cloned();
         } else if let Some(option_did) = self.tcx.get_diagnostic_item(sym::Option)
@@ -1238,6 +1274,49 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
     }
 
+    /// Suggest providing `std::ptr::null()` or `std::ptr::null_mut()` if they
+    /// pass in a literal 0 to an raw pointer.
+    #[instrument(skip(self, err))]
+    pub(crate) fn suggest_null_ptr_for_literal_zero_given_to_ptr_arg(
+        &self,
+        err: &mut Diagnostic,
+        expr: &hir::Expr<'_>,
+        expected_ty: Ty<'tcx>,
+    ) -> bool {
+        // Expected type needs to be a raw pointer.
+        let ty::RawPtr(ty::TypeAndMut { mutbl, .. }) = expected_ty.kind() else {
+            return false;
+        };
+
+        // Provided expression needs to be a literal `0`.
+        let ExprKind::Lit(Spanned {
+            node: rustc_ast::LitKind::Int(0, _),
+            span,
+        }) = expr.kind else {
+            return false;
+        };
+
+        // We need to find a null pointer symbol to suggest
+        let null_sym = match mutbl {
+            hir::Mutability::Not => sym::ptr_null,
+            hir::Mutability::Mut => sym::ptr_null_mut,
+        };
+        let Some(null_did) = self.tcx.get_diagnostic_item(null_sym) else {
+            return false;
+        };
+        let null_path_str = with_no_trimmed_paths!(self.tcx.def_path_str(null_did));
+
+        // We have satisfied all requirements to provide a suggestion. Emit it.
+        err.span_suggestion(
+            span,
+            format!("if you meant to create a null pointer, use `{null_path_str}()`"),
+            null_path_str + "()",
+            Applicability::MachineApplicable,
+        );
+
+        true
+    }
+
     pub(crate) fn suggest_associated_const(
         &self,
         err: &mut Diagnostic,
@@ -1258,16 +1337,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 hir::Path { segments: [segment], .. },
             ))
             | hir::ExprKind::Path(QPath::TypeRelative(ty, segment)) => {
-                let self_ty = self.astconv().ast_ty_to_ty(ty);
-                if let Ok(pick) = self.probe_for_name(
-                    Mode::Path,
-                    Ident::new(capitalized_name, segment.ident.span),
-                    Some(expected_ty),
-                    IsSuggestion(true),
-                    self_ty,
-                    expr.hir_id,
-                    ProbeScope::TraitsInScope,
-                ) {
+                if let Some(self_ty) = self.typeck_results.borrow().node_type_opt(ty.hir_id)
+                    && let Ok(pick) = self.probe_for_name(
+                        Mode::Path,
+                        Ident::new(capitalized_name, segment.ident.span),
+                        Some(expected_ty),
+                        IsSuggestion(true),
+                        self_ty,
+                        expr.hir_id,
+                        ProbeScope::TraitsInScope,
+                    )
+                {
                     (pick.item, segment)
                 } else {
                     return false;
@@ -1299,7 +1379,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // Same item
             return false;
         }
-        let item_ty = self.tcx.type_of(item.def_id);
+        let item_ty = self.tcx.type_of(item.def_id).subst_identity();
         // FIXME(compiler-errors): This check is *so* rudimentary
         if item_ty.needs_subst() {
             return false;
@@ -1379,6 +1459,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     generics,
                     diag,
                     vec![(param.name.as_str(), "Clone", Some(clone_trait_did))].into_iter(),
+                    None,
                 );
             } else {
                 self.suggest_derive(diag, &[(trait_ref.to_predicate(self.tcx), None, None)]);
diff --git a/compiler/rustc_hir_typeck/src/gather_locals.rs b/compiler/rustc_hir_typeck/src/gather_locals.rs
index 15dd3412c..38445f284 100644
--- a/compiler/rustc_hir_typeck/src/gather_locals.rs
+++ b/compiler/rustc_hir_typeck/src/gather_locals.rs
@@ -5,6 +5,7 @@ use rustc_hir::PatKind;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
 use rustc_middle::ty::Ty;
 use rustc_middle::ty::UserType;
+use rustc_span::def_id::LocalDefId;
 use rustc_span::Span;
 use rustc_trait_selection::traits;
 
@@ -156,7 +157,7 @@ impl<'a, 'tcx> Visitor<'tcx> for GatherLocalsVisitor<'a, 'tcx> {
         _: &'tcx hir::FnDecl<'tcx>,
         _: hir::BodyId,
         _: Span,
-        _: hir::HirId,
+        _: LocalDefId,
     ) {
     }
 }
diff --git a/compiler/rustc_hir_typeck/src/generator_interior/drop_ranges/cfg_build.rs b/compiler/rustc_hir_typeck/src/generator_interior/drop_ranges/cfg_build.rs
index b3dd3031d..7c0402b1c 100644
--- a/compiler/rustc_hir_typeck/src/generator_interior/drop_ranges/cfg_build.rs
+++ b/compiler/rustc_hir_typeck/src/generator_interior/drop_ranges/cfg_build.rs
@@ -12,7 +12,7 @@ use rustc_index::vec::IndexVec;
 use rustc_infer::infer::InferCtxt;
 use rustc_middle::{
     hir::map::Map,
-    ty::{ParamEnv, TyCtxt, TypeVisitable, TypeckResults},
+    ty::{ParamEnv, TyCtxt, TypeVisitableExt, TypeckResults},
 };
 use std::mem::swap;
 
@@ -219,7 +219,7 @@ impl<'a, 'tcx> DropRangeVisitor<'a, 'tcx> {
             | ExprKind::Struct(..)
             | ExprKind::Repeat(..)
             | ExprKind::Yield(..)
-            | ExprKind::Err => (),
+            | ExprKind::Err(_) => (),
         }
     }
 
@@ -483,7 +483,7 @@ impl<'a, 'tcx> Visitor<'tcx> for DropRangeVisitor<'a, 'tcx> {
             | ExprKind::Closure { .. }
             | ExprKind::ConstBlock(..)
             | ExprKind::DropTemps(..)
-            | ExprKind::Err
+            | ExprKind::Err(_)
             | ExprKind::Field(..)
             | ExprKind::Index(..)
             | ExprKind::InlineAsm(..)
diff --git a/compiler/rustc_hir_typeck/src/generator_interior/drop_ranges/record_consumed_borrow.rs b/compiler/rustc_hir_typeck/src/generator_interior/drop_ranges/record_consumed_borrow.rs
index ed3d89031..fa3887362 100644
--- a/compiler/rustc_hir_typeck/src/generator_interior/drop_ranges/record_consumed_borrow.rs
+++ b/compiler/rustc_hir_typeck/src/generator_interior/drop_ranges/record_consumed_borrow.rs
@@ -9,7 +9,7 @@ use rustc_hir as hir;
 use rustc_middle::ty::{ParamEnv, TyCtxt};
 use rustc_middle::{
     hir::place::{PlaceBase, Projection, ProjectionKind},
-    ty::TypeVisitable,
+    ty::TypeVisitableExt,
 };
 
 pub(super) fn find_consumed_and_borrowed<'a, 'tcx>(
diff --git a/compiler/rustc_hir_typeck/src/generator_interior/mod.rs b/compiler/rustc_hir_typeck/src/generator_interior/mod.rs
index 7af526053..2e41c2041 100644
--- a/compiler/rustc_hir_typeck/src/generator_interior/mod.rs
+++ b/compiler/rustc_hir_typeck/src/generator_interior/mod.rs
@@ -16,7 +16,7 @@ use rustc_hir::{Arm, Expr, ExprKind, Guard, HirId, Pat, PatKind};
 use rustc_infer::infer::RegionVariableOrigin;
 use rustc_middle::middle::region::{self, Scope, ScopeData, YieldData};
 use rustc_middle::ty::fold::FnMutDelegate;
-use rustc_middle::ty::{self, BoundVariableKind, RvalueScopes, Ty, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, BoundVariableKind, RvalueScopes, Ty, TyCtxt, TypeVisitableExt};
 use rustc_span::symbol::sym;
 use rustc_span::Span;
 use smallvec::{smallvec, SmallVec};
@@ -271,15 +271,13 @@ pub fn resolve_interior<'a, 'tcx>(
                     },
                     _ => mk_bound_region(None),
                 };
-                let r = fcx.tcx.mk_region(ty::ReLateBound(current_depth, br));
+                let r = fcx.tcx.mk_re_late_bound(current_depth, br);
                 r
             });
-            if captured_tys.insert(ty) {
+            captured_tys.insert(ty).then(|| {
                 cause.ty = ty;
-                Some(cause)
-            } else {
-                None
-            }
+                cause
+            })
         })
         .collect();
 
@@ -302,7 +300,7 @@ pub fn resolve_interior<'a, 'tcx>(
                     let var = ty::BoundVar::from_usize(bound_vars.len());
                     bound_vars.push(ty::BoundVariableKind::Region(kind));
                     counter += 1;
-                    fcx.tcx.mk_region(ty::ReLateBound(ty::INNERMOST, ty::BoundRegion { var, kind }))
+                    fcx.tcx.mk_re_late_bound(ty::INNERMOST, ty::BoundRegion { var, kind })
                 },
                 types: &mut |b| bug!("unexpected bound ty in binder: {b:?}"),
                 consts: &mut |b, ty| bug!("unexpected bound ct in binder: {b:?} {ty}"),
@@ -313,8 +311,8 @@ pub fn resolve_interior<'a, 'tcx>(
     };
 
     // Extract type components to build the witness type.
-    let type_list = fcx.tcx.mk_type_list(type_causes.iter().map(|cause| cause.ty));
-    let bound_vars = fcx.tcx.mk_bound_variable_kinds(bound_vars.iter());
+    let type_list = fcx.tcx.mk_type_list_from_iter(type_causes.iter().map(|cause| cause.ty));
+    let bound_vars = fcx.tcx.mk_bound_variable_kinds(&bound_vars);
     let witness =
         fcx.tcx.mk_generator_witness(ty::Binder::bind_with_vars(type_list, bound_vars.clone()));
 
@@ -364,7 +362,7 @@ impl<'a, 'tcx> Visitor<'tcx> for InteriorVisitor<'a, 'tcx> {
                             let ty = tcx.mk_ref(
                                 // Use `ReErased` as `resolve_interior` is going to replace all the
                                 // regions anyway.
-                                tcx.mk_region(ty::ReErased),
+                                tcx.lifetimes.re_erased,
                                 ty::TypeAndMut { ty, mutbl: hir::Mutability::Not },
                             );
                             self.interior_visitor.record(
@@ -647,7 +645,8 @@ fn check_must_not_suspend_ty<'tcx>(
                 hir_id,
                 SuspendCheckData {
                     descr_pre,
-                    plural_len: len.try_eval_usize(fcx.tcx, fcx.param_env).unwrap_or(0) as usize
+                    plural_len: len.try_eval_target_usize(fcx.tcx, fcx.param_env).unwrap_or(0)
+                        as usize
                         + 1,
                     ..data
                 },
diff --git a/compiler/rustc_hir_typeck/src/inherited.rs b/compiler/rustc_hir_typeck/src/inherited.rs
index b33e7b8d6..26020382d 100644
--- a/compiler/rustc_hir_typeck/src/inherited.rs
+++ b/compiler/rustc_hir_typeck/src/inherited.rs
@@ -1,16 +1,17 @@
 use super::callee::DeferredCallResolution;
 
-use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_hir as hir;
 use rustc_hir::def_id::LocalDefId;
 use rustc_hir::HirIdMap;
 use rustc_infer::infer;
 use rustc_infer::infer::{DefiningAnchor, InferCtxt, InferOk, TyCtxtInferExt};
-use rustc_middle::ty::visit::TypeVisitable;
+use rustc_middle::ty::visit::TypeVisitableExt;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_span::def_id::LocalDefIdMap;
 use rustc_span::{self, Span};
-use rustc_trait_selection::traits::{self, TraitEngine, TraitEngineExt as _};
+use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt;
+use rustc_trait_selection::traits::{self, PredicateObligation, TraitEngine, TraitEngineExt as _};
 
 use std::cell::RefCell;
 use std::ops::Deref;
@@ -55,7 +56,7 @@ pub struct Inherited<'tcx> {
     pub(super) deferred_asm_checks: RefCell<Vec<(&'tcx hir::InlineAsm<'tcx>, hir::HirId)>>,
 
     pub(super) deferred_generator_interiors:
-        RefCell<Vec<(hir::BodyId, Ty<'tcx>, hir::GeneratorKind)>>,
+        RefCell<Vec<(LocalDefId, hir::BodyId, Ty<'tcx>, hir::GeneratorKind)>>,
 
     pub(super) body_id: Option<hir::BodyId>,
 
@@ -63,6 +64,8 @@ pub struct Inherited<'tcx> {
     /// we record that type variable here. This is later used to inform
     /// fallback. See the `fallback` module for details.
     pub(super) diverging_type_vars: RefCell<FxHashSet<Ty<'tcx>>>,
+
+    pub(super) infer_var_info: RefCell<FxHashMap<ty::TyVid, ty::InferVarInfo>>,
 }
 
 impl<'tcx> Deref for Inherited<'tcx> {
@@ -128,6 +131,7 @@ impl<'tcx> Inherited<'tcx> {
             deferred_generator_interiors: RefCell::new(Vec::new()),
             diverging_type_vars: RefCell::new(Default::default()),
             body_id,
+            infer_var_info: RefCell::new(Default::default()),
         }
     }
 
@@ -136,6 +140,9 @@ impl<'tcx> Inherited<'tcx> {
         if obligation.has_escaping_bound_vars() {
             span_bug!(obligation.cause.span, "escaping bound vars in predicate {:?}", obligation);
         }
+
+        self.update_infer_var_info(&obligation);
+
         self.fulfillment_cx.borrow_mut().register_predicate_obligation(self, obligation);
     }
 
@@ -152,4 +159,43 @@ impl<'tcx> Inherited<'tcx> {
         self.register_predicates(infer_ok.obligations);
         infer_ok.value
     }
+
+    pub fn update_infer_var_info(&self, obligation: &PredicateObligation<'tcx>) {
+        let infer_var_info = &mut self.infer_var_info.borrow_mut();
+
+        // (*) binder skipped
+        if let ty::PredicateKind::Clause(ty::Clause::Trait(tpred)) = obligation.predicate.kind().skip_binder()
+            && let Some(ty) = self.shallow_resolve(tpred.self_ty()).ty_vid().map(|t| self.root_var(t))
+            && self.tcx.lang_items().sized_trait().map_or(false, |st| st != tpred.trait_ref.def_id)
+        {
+            let new_self_ty = self.tcx.types.unit;
+
+            // Then construct a new obligation with Self = () added
+            // to the ParamEnv, and see if it holds.
+            let o = obligation.with(self.tcx,
+                obligation
+                    .predicate
+                    .kind()
+                    .rebind(
+                        // (*) binder moved here
+                        ty::PredicateKind::Clause(ty::Clause::Trait(tpred.with_self_ty(self.tcx, new_self_ty)))
+                    ),
+            );
+            // Don't report overflow errors. Otherwise equivalent to may_hold.
+            if let Ok(result) = self.probe(|_| self.evaluate_obligation(&o)) && result.may_apply() {
+                infer_var_info.entry(ty).or_default().self_in_trait = true;
+            }
+        }
+
+        if let ty::PredicateKind::Clause(ty::Clause::Projection(predicate)) =
+            obligation.predicate.kind().skip_binder()
+        {
+            // If the projection predicate (Foo::Bar == X) has X as a non-TyVid,
+            // we need to make it into one.
+            if let Some(vid) = predicate.term.ty().and_then(|ty| ty.ty_vid()) {
+                debug!("infer_var_info: {:?}.output = true", vid);
+                infer_var_info.entry(vid).or_default().output = true;
+            }
+        }
+    }
 }
diff --git a/compiler/rustc_hir_typeck/src/intrinsicck.rs b/compiler/rustc_hir_typeck/src/intrinsicck.rs
index 3c873024c..19d2befc4 100644
--- a/compiler/rustc_hir_typeck/src/intrinsicck.rs
+++ b/compiler/rustc_hir_typeck/src/intrinsicck.rs
@@ -3,7 +3,7 @@ use rustc_errors::struct_span_err;
 use rustc_hir as hir;
 use rustc_index::vec::Idx;
 use rustc_middle::ty::layout::{LayoutError, SizeSkeleton};
-use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitableExt};
 use rustc_target::abi::{Pointer, VariantIdx};
 
 use super::FnCtxt;
@@ -38,6 +38,7 @@ fn unpack_option_like<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Ty<'tcx> {
 impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     pub fn check_transmute(&self, from: Ty<'tcx>, to: Ty<'tcx>, hir_id: HirId) {
         let tcx = self.tcx;
+        let dl = &tcx.data_layout;
         let span = tcx.hir().span(hir_id);
         let normalize = |ty| {
             let ty = self.resolve_vars_if_possible(ty);
@@ -69,7 +70,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // Special-case transmuting from `typeof(function)` and
             // `Option<typeof(function)>` to present a clearer error.
             let from = unpack_option_like(tcx, from);
-            if let (&ty::FnDef(..), SizeSkeleton::Known(size_to)) = (from.kind(), sk_to) && size_to == Pointer.size(&tcx) {
+            if let (&ty::FnDef(..), SizeSkeleton::Known(size_to)) = (from.kind(), sk_to) && size_to == Pointer(dl.instruction_address_space).size(&tcx) {
                 struct_span_err!(tcx.sess, span, E0591, "can't transmute zero-sized type")
                     .note(&format!("source type: {from}"))
                     .note(&format!("target type: {to}"))
diff --git a/compiler/rustc_hir_typeck/src/lib.rs b/compiler/rustc_hir_typeck/src/lib.rs
index 7ddf9eaa4..e397dfd45 100644
--- a/compiler/rustc_hir_typeck/src/lib.rs
+++ b/compiler/rustc_hir_typeck/src/lib.rs
@@ -5,6 +5,7 @@
 #![feature(min_specialization)]
 #![feature(control_flow_enum)]
 #![feature(drain_filter)]
+#![feature(option_as_slice)]
 #![allow(rustc::potential_query_instability)]
 #![recursion_limit = "256"]
 
@@ -53,7 +54,10 @@ use crate::check::check_fn;
 use crate::coercion::DynamicCoerceMany;
 use crate::gather_locals::GatherLocalsVisitor;
 use rustc_data_structures::unord::UnordSet;
-use rustc_errors::{struct_span_err, DiagnosticId, ErrorGuaranteed, MultiSpan};
+use rustc_errors::{
+    struct_span_err, DiagnosticId, DiagnosticMessage, ErrorGuaranteed, MultiSpan,
+    SubdiagnosticMessage,
+};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::intravisit::Visitor;
@@ -61,13 +65,16 @@ use rustc_hir::{HirIdMap, Node};
 use rustc_hir_analysis::astconv::AstConv;
 use rustc_hir_analysis::check::check_abi;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
+use rustc_macros::fluent_messages;
 use rustc_middle::traits;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_session::config;
 use rustc_session::Session;
 use rustc_span::def_id::{DefId, LocalDefId};
-use rustc_span::Span;
+use rustc_span::{sym, Span};
+
+fluent_messages! { "../locales/en-US.ftl" }
 
 #[macro_export]
 macro_rules! type_error_struct {
@@ -154,7 +161,7 @@ fn typeck_const_arg<'tcx>(
     tcx: TyCtxt<'tcx>,
     (did, param_did): (LocalDefId, DefId),
 ) -> &ty::TypeckResults<'tcx> {
-    let fallback = move || tcx.type_of(param_did);
+    let fallback = move || tcx.type_of(param_did).subst_identity();
     typeck_with_fallback(tcx, did, fallback)
 }
 
@@ -162,7 +169,7 @@ fn typeck<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> &ty::TypeckResults<'tc
     if let Some(param_did) = tcx.opt_const_param_of(def_id) {
         tcx.typeck_const_arg((def_id, param_did))
     } else {
-        let fallback = move || tcx.type_of(def_id.to_def_id());
+        let fallback = move || tcx.type_of(def_id.to_def_id()).subst_identity();
         typeck_with_fallback(tcx, def_id, fallback)
     }
 }
@@ -201,13 +208,18 @@ fn typeck_with_fallback<'tcx>(
 
     let typeck_results = Inherited::build(tcx, def_id).enter(|inh| {
         let param_env = tcx.param_env(def_id);
-        let mut fcx = FnCtxt::new(&inh, param_env, body.value.hir_id);
+        let param_env = if tcx.has_attr(def_id.to_def_id(), sym::rustc_do_not_const_check) {
+            param_env.without_const()
+        } else {
+            param_env
+        };
+        let mut fcx = FnCtxt::new(&inh, param_env, def_id);
 
         if let Some(hir::FnSig { header, decl, .. }) = fn_sig {
             let fn_sig = if rustc_hir_analysis::collect::get_infer_ret_ty(&decl.output).is_some() {
                 fcx.astconv().ty_of_fn(id, header.unsafety, header.abi, decl, None, None)
             } else {
-                tcx.fn_sig(def_id)
+                tcx.fn_sig(def_id).subst_identity()
             };
 
             check_abi(tcx, id, span, fn_sig.abi());
@@ -294,14 +306,24 @@ fn typeck_with_fallback<'tcx>(
         // Before the generator analysis, temporary scopes shall be marked to provide more
         // precise information on types to be captured.
         fcx.resolve_rvalue_scopes(def_id.to_def_id());
-        fcx.resolve_generator_interiors(def_id.to_def_id());
 
         for (ty, span, code) in fcx.deferred_sized_obligations.borrow_mut().drain(..) {
             let ty = fcx.normalize(span, ty);
             fcx.require_type_is_sized(ty, span, code);
         }
 
-        fcx.select_all_obligations_or_error();
+        fcx.select_obligations_where_possible(|_| {});
+
+        debug!(pending_obligations = ?fcx.fulfillment_cx.borrow().pending_obligations());
+
+        // This must be the last thing before `report_ambiguity_errors`.
+        fcx.resolve_generator_interiors(def_id.to_def_id());
+
+        debug!(pending_obligations = ?fcx.fulfillment_cx.borrow().pending_obligations());
+
+        if let None = fcx.infcx.tainted_by_errors() {
+            fcx.report_ambiguity_errors();
+        }
 
         if let None = fcx.infcx.tainted_by_errors() {
             fcx.check_transmutes();
diff --git a/compiler/rustc_hir_typeck/src/mem_categorization.rs b/compiler/rustc_hir_typeck/src/mem_categorization.rs
index 48c75cde9..4d3969d28 100644
--- a/compiler/rustc_hir_typeck/src/mem_categorization.rs
+++ b/compiler/rustc_hir_typeck/src/mem_categorization.rs
@@ -51,8 +51,7 @@
 use rustc_middle::hir::place::*;
 use rustc_middle::ty::adjustment;
 use rustc_middle::ty::fold::TypeFoldable;
-use rustc_middle::ty::visit::TypeVisitable;
-use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitableExt};
 
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_hir as hir;
@@ -127,7 +126,7 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
 
     fn resolve_vars_if_possible<T>(&self, value: T) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         self.infcx.resolve_vars_if_possible(value)
     }
@@ -155,8 +154,7 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
             None if self.is_tainted_by_errors() => Err(()),
             None => {
                 bug!(
-                    "no type for node {}: {} in mem_categorization",
-                    id,
+                    "no type for node {} in mem_categorization",
                     self.tcx().hir().node_to_string(id)
                 );
             }
@@ -385,7 +383,7 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
             | hir::ExprKind::Repeat(..)
             | hir::ExprKind::InlineAsm(..)
             | hir::ExprKind::Box(..)
-            | hir::ExprKind::Err => Ok(self.cat_rvalue(expr.hir_id, expr.span, expr_ty)),
+            | hir::ExprKind::Err(_) => Ok(self.cat_rvalue(expr.hir_id, expr.span, expr_ty)),
         }
     }
 
@@ -603,7 +601,6 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
         }
     }
 
-    // FIXME(#19596) This is a workaround, but there should be a better way to do this
     fn cat_pattern_<F>(
         &self,
         mut place_with_id: PlaceWithHirId<'tcx>,
@@ -639,7 +636,7 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
         // `&&Some(x,)` `place_foo`
         //  `&Some(x,)` `deref { place_foo}`
         //   `Some(x,)` `deref { deref { place_foo }}`
-        //        (x,)` `field0 { deref { deref { place_foo }}}` <- resulting place
+        //       `(x,)` `field0 { deref { deref { place_foo }}}` <- resulting place
         //
         // The above example has no adjustments. If the code were instead the (after adjustments,
         // equivalent) version
diff --git a/compiler/rustc_hir_typeck/src/method/confirm.rs b/compiler/rustc_hir_typeck/src/method/confirm.rs
index 372ea30eb..169f128e0 100644
--- a/compiler/rustc_hir_typeck/src/method/confirm.rs
+++ b/compiler/rustc_hir_typeck/src/method/confirm.rs
@@ -14,7 +14,7 @@ use rustc_middle::ty::adjustment::{Adjust, Adjustment, PointerCast};
 use rustc_middle::ty::adjustment::{AllowTwoPhase, AutoBorrow, AutoBorrowMutability};
 use rustc_middle::ty::fold::TypeFoldable;
 use rustc_middle::ty::subst::{self, SubstsRef};
-use rustc_middle::ty::{self, GenericParamDefKind, Ty};
+use rustc_middle::ty::{self, GenericParamDefKind, Ty, TyCtxt};
 use rustc_middle::ty::{InternalSubsts, UserSubsts, UserType};
 use rustc_span::{Span, DUMMY_SP};
 use rustc_trait_selection::traits;
@@ -262,7 +262,7 @@ impl<'a, 'tcx> ConfirmContext<'a, 'tcx> {
                     let original_poly_trait_ref = principal.with_self_ty(this.tcx, object_ty);
                     let upcast_poly_trait_ref = this.upcast(original_poly_trait_ref, trait_def_id);
                     let upcast_trait_ref =
-                        this.replace_bound_vars_with_fresh_vars(upcast_poly_trait_ref);
+                        this.instantiate_binder_with_fresh_vars(upcast_poly_trait_ref);
                     debug!(
                         "original_poly_trait_ref={:?} upcast_trait_ref={:?} target_trait={:?}",
                         original_poly_trait_ref, upcast_trait_ref, trait_def_id
@@ -285,7 +285,7 @@ impl<'a, 'tcx> ConfirmContext<'a, 'tcx> {
             probe::WhereClausePick(poly_trait_ref) => {
                 // Where clauses can have bound regions in them. We need to instantiate
                 // those to convert from a poly-trait-ref to a trait-ref.
-                self.replace_bound_vars_with_fresh_vars(poly_trait_ref).substs
+                self.instantiate_binder_with_fresh_vars(poly_trait_ref).substs
             }
         }
     }
@@ -384,7 +384,15 @@ impl<'a, 'tcx> ConfirmContext<'a, 'tcx> {
                     }
                     (GenericParamDefKind::Const { .. }, GenericArg::Infer(inf)) => {
                         let tcx = self.cfcx.tcx();
-                        self.cfcx.ct_infer(tcx.type_of(param.def_id), Some(param), inf.span).into()
+                        self.cfcx
+                            .ct_infer(
+                                tcx.type_of(param.def_id)
+                                    .no_bound_vars()
+                                    .expect("const parameter types cannot be generic"),
+                                Some(param),
+                                inf.span,
+                            )
+                            .into()
                     }
                     _ => unreachable!(),
                 }
@@ -503,12 +511,10 @@ impl<'a, 'tcx> ConfirmContext<'a, 'tcx> {
 
         debug!("method_predicates after subst = {:?}", method_predicates);
 
-        let sig = self.tcx.bound_fn_sig(def_id);
-
-        let sig = sig.subst(self.tcx, all_substs);
+        let sig = self.tcx.fn_sig(def_id).subst(self.tcx, all_substs);
         debug!("type scheme substituted, sig={:?}", sig);
 
-        let sig = self.replace_bound_vars_with_fresh_vars(sig);
+        let sig = self.instantiate_binder_with_fresh_vars(sig);
         debug!("late-bound lifetimes from method instantiated, sig={:?}", sig);
 
         (sig, method_predicates)
@@ -627,10 +633,10 @@ impl<'a, 'tcx> ConfirmContext<'a, 'tcx> {
         upcast_trait_refs.into_iter().next().unwrap()
     }
 
-    fn replace_bound_vars_with_fresh_vars<T>(&self, value: ty::Binder<'tcx, T>) -> T
+    fn instantiate_binder_with_fresh_vars<T>(&self, value: ty::Binder<'tcx, T>) -> T
     where
-        T: TypeFoldable<'tcx> + Copy,
+        T: TypeFoldable<TyCtxt<'tcx>> + Copy,
     {
-        self.fcx.replace_bound_vars_with_fresh_vars(self.span, infer::FnCall, value)
+        self.fcx.instantiate_binder_with_fresh_vars(self.span, infer::FnCall, value)
     }
 }
diff --git a/compiler/rustc_hir_typeck/src/method/mod.rs b/compiler/rustc_hir_typeck/src/method/mod.rs
index b810a967a..0456dd56c 100644
--- a/compiler/rustc_hir_typeck/src/method/mod.rs
+++ b/compiler/rustc_hir_typeck/src/method/mod.rs
@@ -20,7 +20,7 @@ use rustc_hir::def_id::DefId;
 use rustc_infer::infer::{self, InferOk};
 use rustc_middle::traits::ObligationCause;
 use rustc_middle::ty::subst::{InternalSubsts, SubstsRef};
-use rustc_middle::ty::{self, GenericParamDefKind, Ty, TypeVisitable};
+use rustc_middle::ty::{self, GenericParamDefKind, Ty, TypeVisitableExt};
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
 use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt;
@@ -76,7 +76,7 @@ pub struct NoMatchData<'tcx> {
     pub unsatisfied_predicates:
         Vec<(ty::Predicate<'tcx>, Option<ty::Predicate<'tcx>>, Option<ObligationCause<'tcx>>)>,
     pub out_of_scope_traits: Vec<DefId>,
-    pub lev_candidate: Option<ty::AssocItem>,
+    pub similar_candidate: Option<ty::AssocItem>,
     pub mode: probe::Mode,
 }
 
@@ -145,7 +145,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             )
             .map(|pick| {
                 let sig = self.tcx.fn_sig(pick.item.def_id);
-                sig.inputs().skip_binder().len().saturating_sub(1)
+                sig.skip_binder().inputs().skip_binder().len().saturating_sub(1)
             })
             .unwrap_or(0);
 
@@ -380,6 +380,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             );
             return None;
         };
+
+        if method_item.kind != ty::AssocKind::Fn {
+            self.tcx.sess.delay_span_bug(tcx.def_span(method_item.def_id), "not a method");
+            return None;
+        }
+
         let def_id = method_item.def_id;
         let generics = tcx.generics_of(def_id);
 
@@ -399,10 +405,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // N.B., instantiate late-bound regions before normalizing the
         // function signature so that normalization does not need to deal
         // with bound regions.
-        let fn_sig = tcx.bound_fn_sig(def_id);
-        let fn_sig = fn_sig.subst(self.tcx, substs);
+        let fn_sig = tcx.fn_sig(def_id).subst(self.tcx, substs);
         let fn_sig =
-            self.replace_bound_vars_with_fresh_vars(obligation.cause.span, infer::FnCall, fn_sig);
+            self.instantiate_binder_with_fresh_vars(obligation.cause.span, infer::FnCall, fn_sig);
 
         let InferOk { value, obligations: o } =
             self.at(&obligation.cause, self.param_env).normalize(fn_sig);
diff --git a/compiler/rustc_hir_typeck/src/method/probe.rs b/compiler/rustc_hir_typeck/src/method/probe.rs
index a24814313..3bef5cfcd 100644
--- a/compiler/rustc_hir_typeck/src/method/probe.rs
+++ b/compiler/rustc_hir_typeck/src/method/probe.rs
@@ -9,24 +9,23 @@ use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::Applicability;
 use rustc_hir as hir;
 use rustc_hir::def::DefKind;
+use rustc_hir_analysis::astconv::InferCtxtExt as _;
 use rustc_hir_analysis::autoderef::{self, Autoderef};
 use rustc_infer::infer::canonical::OriginalQueryValues;
 use rustc_infer::infer::canonical::{Canonical, QueryResponse};
-use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
 use rustc_infer::infer::{self, InferOk, TyCtxtInferExt};
-use rustc_middle::infer::unify_key::{ConstVariableOrigin, ConstVariableOriginKind};
 use rustc_middle::middle::stability;
 use rustc_middle::ty::fast_reject::{simplify_type, TreatParams};
 use rustc_middle::ty::AssocItem;
 use rustc_middle::ty::GenericParamDefKind;
 use rustc_middle::ty::ToPredicate;
-use rustc_middle::ty::{self, ParamEnvAnd, Ty, TyCtxt, TypeFoldable, TypeVisitable};
+use rustc_middle::ty::{self, ParamEnvAnd, Ty, TyCtxt, TypeFoldable, TypeVisitableExt};
 use rustc_middle::ty::{InternalSubsts, SubstsRef};
 use rustc_session::lint;
 use rustc_span::def_id::DefId;
 use rustc_span::def_id::LocalDefId;
-use rustc_span::lev_distance::{
-    find_best_match_for_name_with_substrings, lev_distance_with_substrings,
+use rustc_span::edit_distance::{
+    edit_distance_with_substrings, find_best_match_for_name_with_substrings,
 };
 use rustc_span::symbol::sym;
 use rustc_span::{symbol::Ident, Span, Symbol, DUMMY_SP};
@@ -70,7 +69,7 @@ struct ProbeContext<'a, 'tcx> {
     impl_dups: FxHashSet<DefId>,
 
     /// When probing for names, include names that are close to the
-    /// requested name (by Levenshtein distance)
+    /// requested name (by edit distance)
     allow_similar_names: bool,
 
     /// Some(candidate) if there is a private candidate
@@ -461,7 +460,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     static_candidates: Vec::new(),
                     unsatisfied_predicates: Vec::new(),
                     out_of_scope_traits: Vec::new(),
-                    lev_candidate: None,
+                    similar_candidate: None,
                     mode,
                 }));
             }
@@ -508,16 +507,16 @@ fn method_autoderef_steps<'tcx>(
     let (ref infcx, goal, inference_vars) = tcx.infer_ctxt().build_with_canonical(DUMMY_SP, &goal);
     let ParamEnvAnd { param_env, value: self_ty } = goal;
 
-    let mut autoderef = Autoderef::new(infcx, param_env, hir::CRATE_HIR_ID, DUMMY_SP, self_ty)
-        .include_raw_pointers()
-        .silence_errors();
+    let mut autoderef =
+        Autoderef::new(infcx, param_env, hir::def_id::CRATE_DEF_ID, DUMMY_SP, self_ty)
+            .include_raw_pointers()
+            .silence_errors();
     let mut reached_raw_pointer = false;
     let mut steps: Vec<_> = autoderef
         .by_ref()
         .map(|(ty, d)| {
             let step = CandidateStep {
-                self_ty: infcx
-                    .make_query_response_ignoring_pending_obligations(inference_vars.clone(), ty),
+                self_ty: infcx.make_query_response_ignoring_pending_obligations(inference_vars, ty),
                 autoderefs: d,
                 from_unsafe_deref: reached_raw_pointer,
                 unsize: false,
@@ -610,10 +609,9 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
     fn push_candidate(&mut self, candidate: Candidate<'tcx>, is_inherent: bool) {
         let is_accessible = if let Some(name) = self.method_name {
             let item = candidate.item;
-            let def_scope = self
-                .tcx
-                .adjust_ident_and_get_scope(name, item.container_id(self.tcx), self.body_id)
-                .1;
+            let hir_id = self.tcx.hir().local_def_id_to_hir_id(self.body_id);
+            let def_scope =
+                self.tcx.adjust_ident_and_get_scope(name, item.container_id(self.tcx), hir_id).1;
             item.visibility(self.tcx).is_accessible_from(def_scope, self.tcx)
         } else {
             true
@@ -736,7 +734,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
             debug!("impl_ty: {:?}", impl_ty);
 
             // Determine the receiver type that the method itself expects.
-            let (xform_self_ty, xform_ret_ty) = self.xform_self_ty(&item, impl_ty, impl_substs);
+            let (xform_self_ty, xform_ret_ty) = self.xform_self_ty(item, impl_ty, impl_substs);
             debug!("xform_self_ty: {:?}, xform_ret_ty: {:?}", xform_self_ty, xform_ret_ty);
 
             // We can't use normalize_associated_types_in as it will pollute the
@@ -797,7 +795,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
             let new_trait_ref = this.erase_late_bound_regions(new_trait_ref);
 
             let (xform_self_ty, xform_ret_ty) =
-                this.xform_self_ty(&item, new_trait_ref.self_ty(), new_trait_ref.substs);
+                this.xform_self_ty(item, new_trait_ref.self_ty(), new_trait_ref.substs);
             this.push_candidate(
                 Candidate {
                     xform_self_ty,
@@ -827,6 +825,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                     }
                 }
                 ty::PredicateKind::Subtype(..)
+                | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
                 | ty::PredicateKind::Coerce(..)
                 | ty::PredicateKind::Clause(ty::Clause::Projection(..))
                 | ty::PredicateKind::Clause(ty::Clause::RegionOutlives(..))
@@ -837,6 +836,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                 | ty::PredicateKind::ConstEvaluatable(..)
                 | ty::PredicateKind::ConstEquate(..)
                 | ty::PredicateKind::Ambiguous
+                | ty::PredicateKind::AliasEq(..)
                 | ty::PredicateKind::TypeWellFormedFromEnv(..) => None,
             }
         });
@@ -845,7 +845,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
             let trait_ref = this.erase_late_bound_regions(poly_trait_ref);
 
             let (xform_self_ty, xform_ret_ty) =
-                this.xform_self_ty(&item, trait_ref.self_ty(), trait_ref.substs);
+                this.xform_self_ty(item, trait_ref.self_ty(), trait_ref.substs);
 
             // Because this trait derives from a where-clause, it
             // should not contain any inference variables or other
@@ -916,31 +916,27 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
 
     fn matches_return_type(
         &self,
-        method: &ty::AssocItem,
+        method: ty::AssocItem,
         self_ty: Option<Ty<'tcx>>,
         expected: Ty<'tcx>,
     ) -> bool {
         match method.kind {
-            ty::AssocKind::Fn => {
-                let fty = self.tcx.bound_fn_sig(method.def_id);
-                self.probe(|_| {
-                    let substs = self.fresh_substs_for_item(self.span, method.def_id);
-                    let fty = fty.subst(self.tcx, substs);
-                    let fty =
-                        self.replace_bound_vars_with_fresh_vars(self.span, infer::FnCall, fty);
-
-                    if let Some(self_ty) = self_ty {
-                        if self
-                            .at(&ObligationCause::dummy(), self.param_env)
-                            .sup(fty.inputs()[0], self_ty)
-                            .is_err()
-                        {
-                            return false;
-                        }
+            ty::AssocKind::Fn => self.probe(|_| {
+                let substs = self.fresh_substs_for_item(self.span, method.def_id);
+                let fty = self.tcx.fn_sig(method.def_id).subst(self.tcx, substs);
+                let fty = self.instantiate_binder_with_fresh_vars(self.span, infer::FnCall, fty);
+
+                if let Some(self_ty) = self_ty {
+                    if self
+                        .at(&ObligationCause::dummy(), self.param_env)
+                        .sup(fty.inputs()[0], self_ty)
+                        .is_err()
+                    {
+                        return false;
                     }
-                    self.can_sub(self.param_env, fty.output(), expected).is_ok()
-                })
-            }
+                }
+                self.can_sub(self.param_env, fty.output(), expected)
+            }),
             _ => false,
         }
     }
@@ -955,24 +951,35 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
         let trait_ref = self.tcx.mk_trait_ref(trait_def_id, trait_substs);
 
         if self.tcx.is_trait_alias(trait_def_id) {
-            // For trait aliases, assume all supertraits are relevant.
-            let bounds = iter::once(ty::Binder::dummy(trait_ref));
-            self.elaborate_bounds(bounds, |this, new_trait_ref, item| {
-                let new_trait_ref = this.erase_late_bound_regions(new_trait_ref);
-
-                let (xform_self_ty, xform_ret_ty) =
-                    this.xform_self_ty(&item, new_trait_ref.self_ty(), new_trait_ref.substs);
-                this.push_candidate(
-                    Candidate {
-                        xform_self_ty,
-                        xform_ret_ty,
-                        item,
-                        import_ids: import_ids.clone(),
-                        kind: TraitCandidate(new_trait_ref),
-                    },
-                    false,
-                );
-            });
+            // For trait aliases, recursively assume all explicitly named traits are relevant
+            for expansion in traits::expand_trait_aliases(
+                self.tcx,
+                iter::once((ty::Binder::dummy(trait_ref), self.span)),
+            ) {
+                let bound_trait_ref = expansion.trait_ref();
+                for item in self.impl_or_trait_item(bound_trait_ref.def_id()) {
+                    if !self.has_applicable_self(&item) {
+                        self.record_static_candidate(CandidateSource::Trait(
+                            bound_trait_ref.def_id(),
+                        ));
+                    } else {
+                        let new_trait_ref = self.erase_late_bound_regions(bound_trait_ref);
+
+                        let (xform_self_ty, xform_ret_ty) =
+                            self.xform_self_ty(item, new_trait_ref.self_ty(), new_trait_ref.substs);
+                        self.push_candidate(
+                            Candidate {
+                                xform_self_ty,
+                                xform_ret_ty,
+                                item,
+                                import_ids: import_ids.clone(),
+                                kind: TraitCandidate(new_trait_ref),
+                            },
+                            false,
+                        );
+                    }
+                }
+            }
         } else {
             debug_assert!(self.tcx.is_trait(trait_def_id));
             if self.tcx.trait_is_auto(trait_def_id) {
@@ -987,7 +994,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                 }
 
                 let (xform_self_ty, xform_ret_ty) =
-                    self.xform_self_ty(&item, trait_ref.self_ty(), trait_substs);
+                    self.xform_self_ty(item, trait_ref.self_ty(), trait_substs);
                 self.push_candidate(
                     Candidate {
                         xform_self_ty,
@@ -1014,7 +1021,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
             .filter(|candidate| candidate_filter(&candidate.item))
             .filter(|candidate| {
                 if let Some(return_ty) = self.return_type {
-                    self.matches_return_type(&candidate.item, None, return_ty)
+                    self.matches_return_type(candidate.item, None, return_ty)
                 } else {
                     true
                 }
@@ -1076,29 +1083,20 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
         if let Some((kind, def_id)) = private_candidate {
             return Err(MethodError::PrivateMatch(kind, def_id, out_of_scope_traits));
         }
-        let lev_candidate = self.probe_for_lev_candidate()?;
+        let similar_candidate = self.probe_for_similar_candidate()?;
 
         Err(MethodError::NoMatch(NoMatchData {
             static_candidates,
             unsatisfied_predicates,
             out_of_scope_traits,
-            lev_candidate,
+            similar_candidate,
             mode: self.mode,
         }))
     }
 
     fn pick_core(&self) -> Option<PickResult<'tcx>> {
-        let pick = self.pick_all_method(Some(&mut vec![]));
-
-        // In this case unstable picking is done by `pick_method`.
-        if !self.tcx.sess.opts.unstable_opts.pick_stable_methods_before_any_unstable {
-            return pick;
-        }
-
-        if pick.is_none() {
-            return self.pick_all_method(None);
-        }
-        pick
+        // Pick stable methods only first, and consider unstable candidates if not found.
+        self.pick_all_method(Some(&mut vec![])).or_else(|| self.pick_all_method(None))
     }
 
     fn pick_all_method(
@@ -1237,54 +1235,11 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
         })
     }
 
-    fn pick_method_with_unstable(&self, self_ty: Ty<'tcx>) -> Option<PickResult<'tcx>> {
-        debug!("pick_method_with_unstable(self_ty={})", self.ty_to_string(self_ty));
-
-        let mut possibly_unsatisfied_predicates = Vec::new();
-
-        for (kind, candidates) in
-            &[("inherent", &self.inherent_candidates), ("extension", &self.extension_candidates)]
-        {
-            debug!("searching {} candidates", kind);
-            let res = self.consider_candidates(
-                self_ty,
-                candidates,
-                &mut possibly_unsatisfied_predicates,
-                Some(&mut vec![]),
-            );
-            if res.is_some() {
-                return res;
-            }
-        }
-
-        for (kind, candidates) in
-            &[("inherent", &self.inherent_candidates), ("extension", &self.extension_candidates)]
-        {
-            debug!("searching unstable {kind} candidates");
-            let res = self.consider_candidates(
-                self_ty,
-                candidates,
-                &mut possibly_unsatisfied_predicates,
-                None,
-            );
-            if res.is_some() {
-                return res;
-            }
-        }
-
-        self.unsatisfied_predicates.borrow_mut().extend(possibly_unsatisfied_predicates);
-        None
-    }
-
     fn pick_method(
         &self,
         self_ty: Ty<'tcx>,
         mut unstable_candidates: Option<&mut Vec<(Candidate<'tcx>, Symbol)>>,
     ) -> Option<PickResult<'tcx>> {
-        if !self.tcx.sess.opts.unstable_opts.pick_stable_methods_before_any_unstable {
-            return self.pick_method_with_unstable(self_ty);
-        }
-
         debug!("pick_method(self_ty={})", self.ty_to_string(self_ty));
 
         let mut possibly_unsatisfied_predicates = Vec::new();
@@ -1358,13 +1313,12 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
             return Some(Err(MethodError::Ambiguity(sources)));
         }
 
-        applicable_candidates.pop().map(|(probe, status)| {
-            if status == ProbeResult::Match {
+        applicable_candidates.pop().map(|(probe, status)| match status {
+            ProbeResult::Match => {
                 Ok(probe
                     .to_unadjusted_pick(self_ty, unstable_candidates.cloned().unwrap_or_default()))
-            } else {
-                Err(MethodError::BadReturnType)
             }
+            ProbeResult::NoMatch | ProbeResult::BadReturnType => Err(MethodError::BadReturnType),
         })
     }
 }
@@ -1412,8 +1366,8 @@ impl<'tcx> Pick<'tcx> {
             span,
             format!(
                 "{} {} with this name may be added to the standard library in the future",
-                def_kind.article(),
-                def_kind.descr(self.item.def_id),
+                tcx.def_kind_descr_article(def_kind, self.item.def_id),
+                tcx.def_kind_descr(def_kind, self.item.def_id),
             ),
             |lint| {
                 match (self.item.kind, self.item.container) {
@@ -1482,7 +1436,6 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
             TraitCandidate(trait_ref) => self.probe(|_| {
                 let _ = self
                     .at(&ObligationCause::dummy(), self.param_env)
-                    .define_opaque_types(false)
                     .sup(candidate.xform_self_ty, self_ty);
                 match self.select_trait_candidate(trait_ref) {
                     Ok(Some(traits::ImplSource::UserDefined(ref impl_data))) => {
@@ -1512,7 +1465,6 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
             // First check that the self type can be related.
             let sub_obligations = match self
                 .at(&ObligationCause::dummy(), self.param_env)
-                .define_opaque_types(false)
                 .sup(probe.xform_self_ty, self_ty)
             {
                 Ok(InferOk { obligations, value: () }) => obligations,
@@ -1567,7 +1519,8 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                                 traits::ImplDerivedObligation(Box::new(
                                     traits::ImplDerivedObligationCause {
                                         derived,
-                                        impl_def_id,
+                                        impl_or_alias_def_id: impl_def_id,
+                                        impl_def_predicate_index: None,
                                         span,
                                     },
                                 ))
@@ -1728,7 +1681,6 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                 if let ProbeResult::Match = result
                     && self
                     .at(&ObligationCause::dummy(), self.param_env)
-                    .define_opaque_types(false)
                     .sup(return_ty, xform_ret_ty)
                     .is_err()
                 {
@@ -1786,8 +1738,8 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
 
     /// Similarly to `probe_for_return_type`, this method attempts to find the best matching
     /// candidate method where the method name may have been misspelled. Similarly to other
-    /// Levenshtein based suggestions, we provide at most one such suggestion.
-    fn probe_for_lev_candidate(&mut self) -> Result<Option<ty::AssocItem>, MethodError<'tcx>> {
+    /// edit distance based suggestions, we provide at most one such suggestion.
+    fn probe_for_similar_candidate(&mut self) -> Result<Option<ty::AssocItem>, MethodError<'tcx>> {
         debug!("probing for method names similar to {:?}", self.method_name);
 
         let steps = self.steps.clone();
@@ -1831,6 +1783,12 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                         None,
                     )
                 }
+                .or_else(|| {
+                    applicable_close_candidates
+                        .iter()
+                        .find(|cand| self.matches_by_doc_alias(cand.def_id))
+                        .map(|cand| cand.name)
+                })
                 .unwrap();
                 Ok(applicable_close_candidates.into_iter().find(|method| method.name == best_name))
             }
@@ -1867,7 +1825,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
     #[instrument(level = "debug", skip(self))]
     fn xform_self_ty(
         &self,
-        item: &ty::AssocItem,
+        item: ty::AssocItem,
         impl_ty: Ty<'tcx>,
         substs: SubstsRef<'tcx>,
     ) -> (Ty<'tcx>, Option<Ty<'tcx>>) {
@@ -1881,7 +1839,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
 
     #[instrument(level = "debug", skip(self))]
     fn xform_method_sig(&self, method: DefId, substs: SubstsRef<'tcx>) -> ty::FnSig<'tcx> {
-        let fn_sig = self.tcx.bound_fn_sig(method);
+        let fn_sig = self.tcx.fn_sig(method);
         debug!(?fn_sig);
 
         assert!(!substs.has_escaping_bound_vars());
@@ -1924,27 +1882,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
         &self,
         impl_def_id: DefId,
     ) -> (ty::EarlyBinder<Ty<'tcx>>, SubstsRef<'tcx>) {
-        (self.tcx.bound_type_of(impl_def_id), self.fresh_item_substs(impl_def_id))
-    }
-
-    fn fresh_item_substs(&self, def_id: DefId) -> SubstsRef<'tcx> {
-        InternalSubsts::for_item(self.tcx, def_id, |param, _| match param.kind {
-            GenericParamDefKind::Lifetime => self.tcx.lifetimes.re_erased.into(),
-            GenericParamDefKind::Type { .. } => self
-                .next_ty_var(TypeVariableOrigin {
-                    kind: TypeVariableOriginKind::SubstitutionPlaceholder,
-                    span: self.tcx.def_span(def_id),
-                })
-                .into(),
-            GenericParamDefKind::Const { .. } => {
-                let span = self.tcx.def_span(def_id);
-                let origin = ConstVariableOrigin {
-                    kind: ConstVariableOriginKind::SubstitutionPlaceholder,
-                    span,
-                };
-                self.next_const_var(self.tcx.type_of(param.def_id), origin).into()
-            }
-        })
+        (self.tcx.type_of(impl_def_id), self.fresh_item_substs(impl_def_id))
     }
 
     /// Replaces late-bound-regions bound by `value` with `'static` using
@@ -1967,7 +1905,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
     ///    so forth.
     fn erase_late_bound_regions<T>(&self, value: ty::Binder<'tcx, T>) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         self.tcx.erase_late_bound_regions(value)
     }
@@ -1981,6 +1919,38 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
         }
     }
 
+    /// Determine if the associated item withe the given DefId matches
+    /// the desired name via a doc alias.
+    fn matches_by_doc_alias(&self, def_id: DefId) -> bool {
+        let Some(name) = self.method_name else { return false; };
+        let Some(local_def_id) = def_id.as_local() else { return false; };
+        let hir_id = self.fcx.tcx.hir().local_def_id_to_hir_id(local_def_id);
+        let attrs = self.fcx.tcx.hir().attrs(hir_id);
+        for attr in attrs {
+            let sym::doc = attr.name_or_empty() else { continue; };
+            let Some(values) = attr.meta_item_list() else { continue; };
+            for v in values {
+                if v.name_or_empty() != sym::alias {
+                    continue;
+                }
+                if let Some(nested) = v.meta_item_list() {
+                    // #[doc(alias("foo", "bar"))]
+                    for n in nested {
+                        if let Some(lit) = n.lit() && name.as_str() == lit.symbol.as_str() {
+                            return true;
+                        }
+                    }
+                } else if let Some(meta) = v.meta_item()
+                    && let Some(lit) = meta.name_value_literal()
+                    && name.as_str() == lit.symbol.as_str() {
+                        // #[doc(alias = "foo")]
+                        return true;
+                }
+            }
+        }
+        false
+    }
+
     /// Finds the method with the appropriate name (or return type, as the case may be). If
     /// `allow_similar_names` is set, find methods with close-matching names.
     // The length of the returned iterator is nearly always 0 or 1 and this
@@ -1996,8 +1966,14 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                         if !self.is_relevant_kind_for_mode(x.kind) {
                             return false;
                         }
-                        match lev_distance_with_substrings(name.as_str(), x.name.as_str(), max_dist)
-                        {
+                        if self.matches_by_doc_alias(x.def_id) {
+                            return true;
+                        }
+                        match edit_distance_with_substrings(
+                            name.as_str(),
+                            x.name.as_str(),
+                            max_dist,
+                        ) {
                             Some(d) => d > 0,
                             None => false,
                         }
diff --git a/compiler/rustc_hir_typeck/src/method/suggest.rs b/compiler/rustc_hir_typeck/src/method/suggest.rs
index 2e1fc4c38..60d56263d 100644
--- a/compiler/rustc_hir_typeck/src/method/suggest.rs
+++ b/compiler/rustc_hir_typeck/src/method/suggest.rs
@@ -27,11 +27,11 @@ use rustc_middle::traits::util::supertraits;
 use rustc_middle::ty::fast_reject::DeepRejectCtxt;
 use rustc_middle::ty::fast_reject::{simplify_type, TreatParams};
 use rustc_middle::ty::print::{with_crate_prefix, with_forced_trimmed_paths};
-use rustc_middle::ty::{self, DefIdTree, GenericArgKind, Ty, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, DefIdTree, GenericArgKind, Ty, TyCtxt, TypeVisitableExt};
 use rustc_middle::ty::{IsSuggestable, ToPolyTraitRef};
 use rustc_span::symbol::{kw, sym, Ident};
 use rustc_span::Symbol;
-use rustc_span::{lev_distance, source_map, ExpnKind, FileName, MacroKind, Span};
+use rustc_span::{edit_distance, source_map, ExpnKind, FileName, MacroKind, Span};
 use rustc_trait_selection::traits::error_reporting::on_unimplemented::OnUnimplementedNote;
 use rustc_trait_selection::traits::error_reporting::on_unimplemented::TypeErrCtxtExt as _;
 use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt as _;
@@ -160,7 +160,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             }
 
             MethodError::PrivateMatch(kind, def_id, out_of_scope_traits) => {
-                let kind = kind.descr(def_id);
+                let kind = self.tcx.def_kind_descr(kind, def_id);
                 let mut err = struct_span_err!(
                     self.tcx.sess,
                     item_name.span,
@@ -259,10 +259,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let mode = no_match_data.mode;
         let tcx = self.tcx;
         let rcvr_ty = self.resolve_vars_if_possible(rcvr_ty);
-        let ty_str = with_forced_trimmed_paths!(self.ty_to_string(rcvr_ty));
+        let (ty_str, ty_file) = tcx.short_ty_string(rcvr_ty);
+        let short_ty_str = with_forced_trimmed_paths!(rcvr_ty.to_string());
         let is_method = mode == Mode::MethodCall;
         let unsatisfied_predicates = &no_match_data.unsatisfied_predicates;
-        let lev_candidate = no_match_data.lev_candidate;
+        let similar_candidate = no_match_data.similar_candidate;
         let item_kind = if is_method {
             "method"
         } else if rcvr_ty.is_enum() {
@@ -276,11 +277,24 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             }
         };
 
-        if self.suggest_wrapping_range_with_parens(tcx, rcvr_ty, source, span, item_name, &ty_str)
-            || self.suggest_constraining_numerical_ty(
-                tcx, rcvr_ty, source, span, item_kind, item_name, &ty_str,
-            )
-        {
+        // We could pass the file for long types into these two, but it isn't strictly necessary
+        // given how targetted they are.
+        if self.suggest_wrapping_range_with_parens(
+            tcx,
+            rcvr_ty,
+            source,
+            span,
+            item_name,
+            &short_ty_str,
+        ) || self.suggest_constraining_numerical_ty(
+            tcx,
+            rcvr_ty,
+            source,
+            span,
+            item_kind,
+            item_name,
+            &short_ty_str,
+        ) {
             return None;
         }
         span = item_name.span;
@@ -319,6 +333,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             rcvr_ty.prefix_string(self.tcx),
             ty_str_reported,
         );
+        if tcx.sess.source_map().is_multiline(sugg_span) {
+            err.span_label(sugg_span.with_hi(span.lo()), "");
+        }
+        let ty_str = if short_ty_str.len() < ty_str.len() && ty_str.len() > 10 {
+            short_ty_str
+        } else {
+            ty_str
+        };
+        if let Some(file) = ty_file {
+            err.note(&format!("the full type name has been written to '{}'", file.display(),));
+        }
         if rcvr_ty.references_error() {
             err.downgrade_to_delayed_bug();
         }
@@ -352,7 +377,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let ty_span = match rcvr_ty.kind() {
             ty::Param(param_type) => {
-                Some(param_type.span_from_generics(self.tcx, self.body_id.owner.to_def_id()))
+                Some(param_type.span_from_generics(self.tcx, self.body_id.to_def_id()))
             }
             ty::Adt(def, _) if def.did().is_local() => Some(tcx.def_span(def.did())),
             _ => None,
@@ -403,7 +428,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 args,
                 sugg_span,
             );
-
             self.note_candidates_on_method_error(
                 rcvr_ty,
                 item_name,
@@ -496,9 +520,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             ty::Param(_) => {
                                 // Account for `fn` items like in `issue-35677.rs` to
                                 // suggest restricting its type params.
-                                let parent_body =
-                                    hir.body_owner(hir::BodyId { hir_id: self.body_id });
-                                Some(hir.get(parent_body))
+                                Some(hir.get_by_def_id(self.body_id))
                             }
                             ty::Adt(def, _) => {
                                 def.did().as_local().map(|def_id| hir.get_by_def_id(def_id))
@@ -555,7 +577,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         // `<Foo as Iterator>::Item = String`.
                         let projection_ty = pred.skip_binder().projection_ty;
 
-                        let substs_with_infer_self = tcx.mk_substs(
+                        let substs_with_infer_self = tcx.mk_substs_from_iter(
                             iter::once(tcx.mk_ty_var(ty::TyVid::from_u32(0)).into())
                                 .chain(projection_ty.substs.iter().skip(1)),
                         );
@@ -597,7 +619,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     ObligationCauseCode::ImplDerivedObligation(data)
                         if matches!(p.kind().skip_binder(), ty::PredicateKind::Clause(_)) =>
                     {
-                        Some((p, parent, data.impl_def_id, data))
+                        Some((p, parent, data.impl_or_alias_def_id, data))
                     }
                     _ => None,
                 })
@@ -695,7 +717,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         );
                     }
                     Some(Node::Item(hir::Item {
-                        ident, kind: hir::ItemKind::Trait(..), ..
+                        ident,
+                        kind: hir::ItemKind::Trait(..) | hir::ItemKind::TraitAlias(..),
+                        ..
                     })) => {
                         skip_list.insert(p);
                         let entry = spanned_predicates.entry(ident.span);
@@ -829,7 +853,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 let primary_message = primary_message.unwrap_or_else(|| {
                     format!(
                         "the {item_kind} `{item_name}` exists for {actual_prefix} `{ty_str}`, \
-                    but its trait bounds were not satisfied"
+                         but its trait bounds were not satisfied"
                     )
                 });
                 err.set_primary_message(&primary_message);
@@ -887,8 +911,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                                         // different from the received one
                                         // So we avoid suggestion method with Box<Self>
                                         // for instance
-                                        self.tcx.at(span).type_of(*def_id) != rcvr_ty
-                                            && self.tcx.at(span).type_of(*def_id) != rcvr_ty
+                                        self.tcx.at(span).type_of(*def_id).subst_identity()
+                                            != rcvr_ty
+                                            && self.tcx.at(span).type_of(*def_id).subst_identity()
+                                                != rcvr_ty
                                     }
                                     (Mode::Path, false, _) => true,
                                     _ => false,
@@ -908,7 +934,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             .iter()
                             .take(limit)
                             .map(|impl_item| {
-                                format!("- `{}`", self.tcx.at(span).type_of(*impl_item))
+                                format!(
+                                    "- `{}`",
+                                    self.tcx.at(span).type_of(*impl_item).subst_identity()
+                                )
                             })
                             .collect::<Vec<_>>()
                             .join("\n");
@@ -937,7 +966,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // give a helping note that it has to be called as `(x.f)(...)`.
         if let SelfSource::MethodCall(expr) = source {
             if !self.suggest_calling_field_as_fn(span, rcvr_ty, expr, item_name, &mut err)
-                && lev_candidate.is_none()
+                && similar_candidate.is_none()
                 && !custom_span_label
             {
                 label_span_not_found(&mut err);
@@ -988,7 +1017,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // that had unsatisfied trait bounds
         if unsatisfied_predicates.is_empty() && rcvr_ty.is_enum() {
             let adt_def = rcvr_ty.ty_adt_def().expect("enum is not an ADT");
-            if let Some(suggestion) = lev_distance::find_best_match_for_name(
+            if let Some(suggestion) = edit_distance::find_best_match_for_name(
                 &adt_def.variants().iter().map(|s| s.name).collect::<Vec<_>>(),
                 item_name.name,
                 None,
@@ -1015,20 +1044,20 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             if fallback_span {
                 err.span_label(span, msg);
             }
-        } else if let Some(lev_candidate) = lev_candidate {
+        } else if let Some(similar_candidate) = similar_candidate {
             // Don't emit a suggestion if we found an actual method
             // that had unsatisfied trait bounds
             if unsatisfied_predicates.is_empty() {
-                let def_kind = lev_candidate.kind.as_def_kind();
+                let def_kind = similar_candidate.kind.as_def_kind();
                 // Methods are defined within the context of a struct and their first parameter is always self,
                 // which represents the instance of the struct the method is being called on
                 // Associated functions don’t take self as a parameter and
                 // they are not methods because they don’t have an instance of the struct to work with.
-                if def_kind == DefKind::AssocFn && lev_candidate.fn_has_self_parameter {
+                if def_kind == DefKind::AssocFn && similar_candidate.fn_has_self_parameter {
                     err.span_suggestion(
                         span,
                         "there is a method with a similar name",
-                        lev_candidate.name,
+                        similar_candidate.name,
                         Applicability::MaybeIncorrect,
                     );
                 } else {
@@ -1036,10 +1065,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         span,
                         &format!(
                             "there is {} {} with a similar name",
-                            def_kind.article(),
-                            def_kind.descr(lev_candidate.def_id),
+                            self.tcx.def_kind_descr_article(def_kind, similar_candidate.def_id),
+                            self.tcx.def_kind_descr(def_kind, similar_candidate.def_id)
                         ),
-                        lev_candidate.name,
+                        similar_candidate.name,
                         Applicability::MaybeIncorrect,
                     );
                 }
@@ -1085,7 +1114,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         None
                     };
 
-                    let impl_ty = self.tcx.at(span).type_of(impl_did);
+                    let impl_ty = self.tcx.at(span).type_of(impl_did).subst_identity();
 
                     let insertion = match self.tcx.impl_trait_ref(impl_did) {
                         None => String::new(),
@@ -1131,6 +1160,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             ty::AssocKind::Fn => self
                                 .tcx
                                 .fn_sig(item.def_id)
+                                .subst_identity()
                                 .inputs()
                                 .skip_binder()
                                 .get(0)
@@ -1145,7 +1175,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             path,
                             ty,
                             item.kind,
-                            item.def_id,
+                            self.tcx.def_kind_descr(item.kind.as_def_kind(), item.def_id),
                             sugg_span,
                             idx,
                             self.tcx.sess.source_map(),
@@ -1181,7 +1211,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             path,
                             rcvr_ty,
                             item.kind,
-                            item.def_id,
+                            self.tcx.def_kind_descr(item.kind.as_def_kind(), item.def_id),
                             sugg_span,
                             idx,
                             self.tcx.sess.source_map(),
@@ -1213,7 +1243,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // When the "method" is resolved through dereferencing, we really want the
             // original type that has the associated function for accurate suggestions.
             // (#61411)
-            let impl_ty = self.tcx.type_of(*impl_did);
+            let impl_ty = self.tcx.type_of(*impl_did).subst_identity();
             let target_ty = self
                 .autoderef(sugg_span, rcvr_ty)
                 .find(|(rcvr_ty, _)| {
@@ -1225,7 +1255,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             if let ty::Adt(def, substs) = target_ty.kind() {
                 // If there are any inferred arguments, (`{integer}`), we should replace
                 // them with underscores to allow the compiler to infer them
-                let infer_substs = self.tcx.mk_substs(substs.into_iter().map(|arg| {
+                let infer_substs = self.tcx.mk_substs_from_iter(substs.into_iter().map(|arg| {
                     if !arg.is_suggestable(self.tcx, true) {
                         has_unsuggestable_args = true;
                         match arg.unpack() {
@@ -1267,7 +1297,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 && let Some(assoc) = self.associated_value(*impl_did, item_name)
                 && assoc.kind == ty::AssocKind::Fn
             {
-                let sig = self.tcx.fn_sig(assoc.def_id);
+                let sig = self.tcx.fn_sig(assoc.def_id).subst_identity();
                 sig.inputs().skip_binder().get(0).and_then(|first| if first.peel_refs() == rcvr_ty.peel_refs() {
                     None
                 } else {
@@ -1343,7 +1373,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             _ => None,
         });
         if let Some((field, field_ty)) = field_receiver {
-            let scope = tcx.parent_module(self.body_id);
+            let scope = tcx.parent_module_from_def_id(self.body_id);
             let is_accessible = field.vis.is_accessible_from(scope, tcx);
 
             if is_accessible {
@@ -1433,8 +1463,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     }
 
                     let range_def_id = self.tcx.require_lang_item(lang_item.unwrap(), None);
-                    let range_ty =
-                        self.tcx.bound_type_of(range_def_id).subst(self.tcx, &[actual.into()]);
+                    let range_ty = self.tcx.type_of(range_def_id).subst(self.tcx, &[actual.into()]);
 
                     let pick = self.lookup_probe_for_diagnostic(
                         item_name,
@@ -1593,7 +1622,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 else { return };
 
         let map = self.infcx.tcx.hir();
-        let body = map.body(rustc_hir::BodyId { hir_id: self.body_id });
+        let body_id = self.tcx.hir().body_owned_by(self.body_id);
+        let body = map.body(body_id);
         struct LetVisitor<'a> {
             result: Option<&'a hir::Expr<'a>>,
             ident_name: Symbol,
@@ -2100,7 +2130,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     // just changing the path.
                     && pick.item.fn_has_self_parameter
                     && let Some(self_ty) =
-                        self.tcx.fn_sig(pick.item.def_id).inputs().skip_binder().get(0)
+                        self.tcx.fn_sig(pick.item.def_id).subst_identity().inputs().skip_binder().get(0)
                     && self_ty.is_ref()
                 {
                     let suggested_path = match deref_ty.kind() {
@@ -2195,7 +2225,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             true
         });
 
-        let module_did = self.tcx.parent_module(self.body_id);
+        let module_did = self.tcx.parent_module_from_def_id(self.body_id);
         let (module, _, _) = self.tcx.hir().get_module(module_did);
         let span = module.spans.inject_use_span;
 
@@ -2353,7 +2383,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         // implement the `AsRef` trait.
                         let skip = skippable.contains(&did)
                             || (("Pin::new" == *pre) && (sym::as_ref == item_name.name))
-                            || inputs_len.map_or(false, |inputs_len| pick.item.kind == ty::AssocKind::Fn && self.tcx.fn_sig(pick.item.def_id).skip_binder().inputs().len() != inputs_len);
+                            || inputs_len.map_or(false, |inputs_len| pick.item.kind == ty::AssocKind::Fn && self.tcx.fn_sig(pick.item.def_id).skip_binder().skip_binder().inputs().len() != inputs_len);
                         // Make sure the method is defined for the *actual* receiver: we don't
                         // want to treat `Box<Self>` as a receiver if it only works because of
                         // an autoderef to `&self`
@@ -2517,7 +2547,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             };
             // Obtain the span for `param` and use it for a structured suggestion.
             if let Some(param) = param_type {
-                let generics = self.tcx.generics_of(self.body_id.owner.to_def_id());
+                let generics = self.tcx.generics_of(self.body_id.to_def_id());
                 let type_param = generics.type_param(param, self.tcx);
                 let hir = self.tcx.hir();
                 if let Some(def_id) = type_param.def_id.as_local() {
@@ -2733,7 +2763,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // check the method arguments number
             if let Ok(pick) = probe &&
                 let fn_sig = self.tcx.fn_sig(pick.item.def_id) &&
-                let fn_args = fn_sig.skip_binder().inputs() &&
+                let fn_args = fn_sig.skip_binder().skip_binder().inputs() &&
                 fn_args.len() == args.len() + 1 {
                 err.span_suggestion_verbose(
                     method_name.span.shrink_to_hi(),
@@ -2826,7 +2856,7 @@ fn print_disambiguation_help<'tcx>(
     trait_name: String,
     rcvr_ty: Ty<'_>,
     kind: ty::AssocKind,
-    def_id: DefId,
+    def_kind_descr: &'static str,
     span: Span,
     candidate: Option<usize>,
     source_map: &source_map::SourceMap,
@@ -2859,7 +2889,7 @@ fn print_disambiguation_help<'tcx>(
         span,
         &format!(
             "disambiguate the {} for {}",
-            kind.as_def_kind().descr(def_id),
+            def_kind_descr,
             if let Some(candidate) = candidate {
                 format!("candidate #{}", candidate)
             } else {
diff --git a/compiler/rustc_hir_typeck/src/op.rs b/compiler/rustc_hir_typeck/src/op.rs
index 78cea1f4d..80279ed96 100644
--- a/compiler/rustc_hir_typeck/src/op.rs
+++ b/compiler/rustc_hir_typeck/src/op.rs
@@ -13,7 +13,7 @@ use rustc_middle::ty::adjustment::{
 };
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{
-    self, DefIdTree, IsSuggestable, Ty, TyCtxt, TypeFolder, TypeSuperFoldable, TypeVisitable,
+    self, DefIdTree, IsSuggestable, Ty, TyCtxt, TypeFolder, TypeSuperFoldable, TypeVisitableExt,
 };
 use rustc_session::errors::ExprParenthesesNeeded;
 use rustc_span::source_map::Spanned;
@@ -297,7 +297,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 method.sig.output()
             }
             // error types are considered "builtin"
-            Err(_) if lhs_ty.references_error() || rhs_ty.references_error() => self.tcx.ty_error(),
+            Err(_) if lhs_ty.references_error() || rhs_ty.references_error() => {
+                self.tcx.ty_error_misc()
+            }
             Err(errors) => {
                 let (_, trait_def_id) =
                     lang_item_for_op(self.tcx, Op::Binary(op, is_assign), op.span);
@@ -335,7 +337,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                                 format!("cannot divide `{lhs_ty}` by `{rhs_ty}`")
                             }
                             hir::BinOpKind::Rem => {
-                                format!("cannot mod `{lhs_ty}` by `{rhs_ty}`")
+                                format!(
+                                    "cannot calculate the remainder of `{lhs_ty}` divided by `{rhs_ty}`"
+                                )
                             }
                             hir::BinOpKind::BitAnd => {
                                 format!("no implementation for `{lhs_ty} & {rhs_ty}`")
@@ -488,9 +492,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                                             if let Some(output_def_id) = output_def_id
                                                 && let Some(trait_def_id) = trait_def_id
                                                 && self.tcx.parent(output_def_id) == trait_def_id
-                                                && output_ty.is_suggestable(self.tcx, false)
+                                                && let Some(output_ty) = output_ty.make_suggestable(self.tcx, false)
                                             {
-                                                Some(("Output", *output_ty))
+                                                Some(("Output", output_ty))
                                             } else {
                                                 None
                                             }
@@ -516,7 +520,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     }
                 }
                 let reported = err.emit();
-                self.tcx.ty_error_with_guaranteed(reported)
+                self.tcx.ty_error(reported)
             }
         };
 
@@ -629,7 +633,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             }
             Err(errors) => {
                 let actual = self.resolve_vars_if_possible(operand_ty);
-                if !actual.references_error() {
+                let guar = actual.error_reported().err().unwrap_or_else(|| {
                     let mut err = struct_span_err!(
                         self.tcx.sess,
                         ex.span,
@@ -699,9 +703,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             }
                         }
                     }
-                    err.emit();
-                }
-                self.tcx.ty_error()
+                    err.emit()
+                });
+                self.tcx.ty_error(guar)
             }
         }
     }
@@ -745,14 +749,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
 
         let opname = Ident::with_dummy_span(opname);
-        let input_types =
-            opt_rhs.as_ref().map(|(_, ty)| std::slice::from_ref(ty)).unwrap_or_default();
+        let (opt_rhs_expr, opt_rhs_ty) = opt_rhs.unzip();
+        let input_types = opt_rhs_ty.as_slice();
         let cause = self.cause(
             span,
             traits::BinOp {
-                rhs_span: opt_rhs.map(|(expr, _)| expr.span),
-                is_lit: opt_rhs
-                    .map_or(false, |(expr, _)| matches!(expr.kind, hir::ExprKind::Lit(_))),
+                rhs_span: opt_rhs_expr.map(|expr| expr.span),
+                is_lit: opt_rhs_expr
+                    .map_or(false, |expr| matches!(expr.kind, hir::ExprKind::Lit(_))),
                 output_ty: expected.only_has_type(self),
             },
         );
@@ -961,8 +965,8 @@ fn is_builtin_binop<'tcx>(lhs: Ty<'tcx>, rhs: Ty<'tcx>, op: hir::BinOp) -> bool
 
 struct TypeParamEraser<'a, 'tcx>(&'a FnCtxt<'a, 'tcx>, Span);
 
-impl<'tcx> TypeFolder<'tcx> for TypeParamEraser<'_, 'tcx> {
-    fn tcx(&self) -> TyCtxt<'tcx> {
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for TypeParamEraser<'_, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.0.tcx
     }
 
diff --git a/compiler/rustc_hir_typeck/src/pat.rs b/compiler/rustc_hir_typeck/src/pat.rs
index 467992452..c36c75e44 100644
--- a/compiler/rustc_hir_typeck/src/pat.rs
+++ b/compiler/rustc_hir_typeck/src/pat.rs
@@ -12,10 +12,10 @@ use rustc_hir::{HirId, Pat, PatKind};
 use rustc_infer::infer;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
 use rustc_middle::middle::stability::EvalResult;
-use rustc_middle::ty::{self, Adt, BindingMode, Ty, TypeVisitable};
+use rustc_middle::ty::{self, Adt, BindingMode, Ty, TypeVisitableExt};
 use rustc_session::lint::builtin::NON_EXHAUSTIVE_OMITTED_PATTERNS;
+use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::hygiene::DesugaringKind;
-use rustc_span::lev_distance::find_best_match_for_name;
 use rustc_span::source_map::{Span, Spanned};
 use rustc_span::symbol::{kw, sym, Ident};
 use rustc_span::{BytePos, DUMMY_SP};
@@ -46,7 +46,7 @@ struct TopInfo<'tcx> {
     /// Was the origin of the `span` from a scrutinee expression?
     ///
     /// Otherwise there is no scrutinee and it could be e.g. from the type of a formal parameter.
-    origin_expr: bool,
+    origin_expr: Option<&'tcx hir::Expr<'tcx>>,
     /// The span giving rise to the `expected` type, if one could be provided.
     ///
     /// If `origin_expr` is `true`, then this is the span of the scrutinee as in:
@@ -74,7 +74,8 @@ struct TopInfo<'tcx> {
 
 impl<'tcx> FnCtxt<'_, 'tcx> {
     fn pattern_cause(&self, ti: TopInfo<'tcx>, cause_span: Span) -> ObligationCause<'tcx> {
-        let code = Pattern { span: ti.span, root_ty: ti.expected, origin_expr: ti.origin_expr };
+        let code =
+            Pattern { span: ti.span, root_ty: ti.expected, origin_expr: ti.origin_expr.is_some() };
         self.cause(cause_span, code)
     }
 
@@ -85,7 +86,14 @@ impl<'tcx> FnCtxt<'_, 'tcx> {
         actual: Ty<'tcx>,
         ti: TopInfo<'tcx>,
     ) -> Option<DiagnosticBuilder<'tcx, ErrorGuaranteed>> {
-        self.demand_eqtype_with_origin(&self.pattern_cause(ti, cause_span), expected, actual)
+        let mut diag =
+            self.demand_eqtype_with_origin(&self.pattern_cause(ti, cause_span), expected, actual)?;
+        if let Some(expr) = ti.origin_expr {
+            self.suggest_fn_call(&mut diag, expr, expected, |output| {
+                self.can_eq(self.param_env, output, actual)
+            });
+        }
+        Some(diag)
     }
 
     fn demand_eqtype_pat(
@@ -127,7 +135,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         pat: &'tcx Pat<'tcx>,
         expected: Ty<'tcx>,
         span: Option<Span>,
-        origin_expr: bool,
+        origin_expr: Option<&'tcx hir::Expr<'tcx>>,
     ) {
         let info = TopInfo { expected, origin_expr, span };
         self.check_pat(pat, expected, INITIAL_BM, info);
@@ -467,8 +475,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         if let (Some((true, ..)), _) | (_, Some((true, ..))) = (lhs, rhs) {
             // There exists a side that didn't meet our criteria that the end-point
             // be of a numeric or char type, as checked in `calc_side` above.
-            self.emit_err_pat_range(span, lhs, rhs);
-            return self.tcx.ty_error();
+            let guar = self.emit_err_pat_range(span, lhs, rhs);
+            return self.tcx.ty_error(guar);
         }
 
         // Unify each side with `expected`.
@@ -488,7 +496,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         demand_eqtype(&mut rhs, lhs);
 
         if let (Some((true, ..)), _) | (_, Some((true, ..))) = (lhs, rhs) {
-            return self.tcx.ty_error();
+            return self.tcx.ty_error_misc();
         }
 
         // Find the unified type and check if it's of numeric or char type again.
@@ -503,8 +511,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             if let Some((ref mut fail, _, _)) = rhs {
                 *fail = true;
             }
-            self.emit_err_pat_range(span, lhs, rhs);
-            return self.tcx.ty_error();
+            let guar = self.emit_err_pat_range(span, lhs, rhs);
+            return self.tcx.ty_error(guar);
         }
         ty
     }
@@ -520,7 +528,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         span: Span,
         lhs: Option<(bool, Ty<'tcx>, Span)>,
         rhs: Option<(bool, Ty<'tcx>, Span)>,
-    ) {
+    ) -> ErrorGuaranteed {
         let span = match (lhs, rhs) {
             (Some((true, ..)), Some((true, ..))) => span,
             (Some((true, _, sp)), _) => sp,
@@ -565,7 +573,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     type between two end-points, you can use a guard.",
             );
         }
-        err.emit();
+        err.emit()
     }
 
     fn check_pat_ident(
@@ -667,7 +675,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     ) {
         match (expected.kind(), actual.kind(), ba) {
             (ty::Ref(_, inner_ty, _), _, hir::BindingAnnotation::NONE)
-                if self.can_eq(self.param_env, *inner_ty, actual).is_ok() =>
+                if self.can_eq(self.param_env, *inner_ty, actual) =>
             {
                 err.span_suggestion_verbose(
                     span.shrink_to_lo(),
@@ -677,7 +685,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 );
             }
             (_, ty::Ref(_, inner_ty, _), hir::BindingAnnotation::REF)
-                if self.can_eq(self.param_env, expected, *inner_ty).is_ok() =>
+                if self.can_eq(self.param_env, expected, *inner_ty) =>
             {
                 err.span_suggestion_verbose(
                     span.with_hi(span.lo() + BytePos(4)),
@@ -799,29 +807,33 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
     }
 
-    pub fn check_dereferenceable(&self, span: Span, expected: Ty<'tcx>, inner: &Pat<'_>) -> bool {
+    pub fn check_dereferenceable(
+        &self,
+        span: Span,
+        expected: Ty<'tcx>,
+        inner: &Pat<'_>,
+    ) -> Result<(), ErrorGuaranteed> {
         if let PatKind::Binding(..) = inner.kind
             && let Some(mt) = self.shallow_resolve(expected).builtin_deref(true)
             && let ty::Dynamic(..) = mt.ty.kind()
         {
-                    // This is "x = SomeTrait" being reduced from
-                    // "let &x = &SomeTrait" or "let box x = Box<SomeTrait>", an error.
-                    let type_str = self.ty_to_string(expected);
-                    let mut err = struct_span_err!(
-                        self.tcx.sess,
-                        span,
-                        E0033,
-                        "type `{}` cannot be dereferenced",
-                        type_str
-                    );
-                    err.span_label(span, format!("type `{type_str}` cannot be dereferenced"));
-                    if self.tcx.sess.teach(&err.get_code().unwrap()) {
-                        err.note(CANNOT_IMPLICITLY_DEREF_POINTER_TRAIT_OBJ);
-                    }
-                    err.emit();
-                    return false;
-                }
-        true
+            // This is "x = SomeTrait" being reduced from
+            // "let &x = &SomeTrait" or "let box x = Box<SomeTrait>", an error.
+            let type_str = self.ty_to_string(expected);
+            let mut err = struct_span_err!(
+                self.tcx.sess,
+                span,
+                E0033,
+                "type `{}` cannot be dereferenced",
+                type_str
+            );
+            err.span_label(span, format!("type `{type_str}` cannot be dereferenced"));
+            if self.tcx.sess.teach(&err.get_code().unwrap()) {
+                err.note(CANNOT_IMPLICITLY_DEREF_POINTER_TRAIT_OBJ);
+            }
+            return Err(err.emit());
+        }
+        Ok(())
     }
 
     fn check_pat_struct(
@@ -835,13 +847,16 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         ti: TopInfo<'tcx>,
     ) -> Ty<'tcx> {
         // Resolve the path and check the definition for errors.
-        let Some((variant, pat_ty)) = self.check_struct_path(qpath, pat.hir_id) else {
-            let err = self.tcx.ty_error();
-            for field in fields {
-                let ti = ti;
-                self.check_pat(field.pat, err, def_bm, ti);
+        let (variant, pat_ty) = match self.check_struct_path(qpath, pat.hir_id) {
+            Ok(data) => data,
+            Err(guar) => {
+                let err = self.tcx.ty_error(guar);
+                for field in fields {
+                    let ti = ti;
+                    self.check_pat(field.pat, err, def_bm, ti);
+                }
+                return err;
             }
-            return err;
         };
 
         // Type-check the path.
@@ -851,7 +866,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         if self.check_struct_pat_fields(pat_ty, &pat, variant, fields, has_rest_pat, def_bm, ti) {
             pat_ty
         } else {
-            self.tcx.ty_error()
+            self.tcx.ty_error_misc()
         }
     }
 
@@ -871,12 +886,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             Res::Err => {
                 let e = tcx.sess.delay_span_bug(qpath.span(), "`Res::Err` but no error emitted");
                 self.set_tainted_by_errors(e);
-                return tcx.ty_error_with_guaranteed(e);
+                return tcx.ty_error(e);
             }
             Res::Def(DefKind::AssocFn | DefKind::Ctor(_, CtorKind::Fn) | DefKind::Variant, _) => {
                 let expected = "unit struct, unit variant or constant";
                 let e = report_unexpected_variant_res(tcx, res, qpath, pat.span, "E0533", expected);
-                return tcx.ty_error_with_guaranteed(e);
+                return tcx.ty_error(e);
             }
             Res::SelfCtor(..)
             | Res::Def(
@@ -1019,7 +1034,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let tcx = self.tcx;
         let on_error = |e| {
             for pat in subpats {
-                self.check_pat(pat, tcx.ty_error_with_guaranteed(e), def_bm, ti);
+                self.check_pat(pat, tcx.ty_error(e), def_bm, ti);
             }
         };
         let report_unexpected_res = |res: Res| {
@@ -1036,7 +1051,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             let e = tcx.sess.delay_span_bug(pat.span, "`Res::Err` but no error emitted");
             self.set_tainted_by_errors(e);
             on_error(e);
-            return tcx.ty_error_with_guaranteed(e);
+            return tcx.ty_error(e);
         }
 
         // Type-check the path.
@@ -1044,7 +1059,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             self.instantiate_value_path(segments, opt_ty, res, pat.span, pat.hir_id);
         if !pat_ty.is_fn() {
             let e = report_unexpected_res(res);
-            return tcx.ty_error_with_guaranteed(e);
+            return tcx.ty_error(e);
         }
 
         let variant = match res {
@@ -1052,11 +1067,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 let e = tcx.sess.delay_span_bug(pat.span, "`Res::Err` but no error emitted");
                 self.set_tainted_by_errors(e);
                 on_error(e);
-                return tcx.ty_error_with_guaranteed(e);
+                return tcx.ty_error(e);
             }
             Res::Def(DefKind::AssocConst | DefKind::AssocFn, _) => {
                 let e = report_unexpected_res(res);
-                return tcx.ty_error_with_guaranteed(e);
+                return tcx.ty_error(e);
             }
             Res::Def(DefKind::Ctor(_, CtorKind::Fn), _) => tcx.expect_variant_res(res),
             _ => bug!("unexpected pattern resolution: {:?}", res),
@@ -1097,7 +1112,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // Pattern has wrong number of fields.
             let e = self.e0023(pat.span, res, qpath, subpats, &variant.fields, expected, had_err);
             on_error(e);
-            return tcx.ty_error_with_guaranteed(e);
+            return tcx.ty_error(e);
         }
         pat_ty
     }
@@ -1287,17 +1302,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 TypeVariableOrigin { kind: TypeVariableOriginKind::TypeInference, span },
             )
         });
-        let element_tys = tcx.mk_type_list(element_tys_iter);
-        let pat_ty = tcx.mk_ty(ty::Tuple(element_tys));
+        let element_tys = tcx.mk_type_list_from_iter(element_tys_iter);
+        let pat_ty = tcx.mk_tup(element_tys);
         if let Some(mut err) = self.demand_eqtype_pat_diag(span, expected, pat_ty, ti) {
             let reported = err.emit();
             // Walk subpatterns with an expected type of `err` in this case to silence
             // further errors being emitted when using the bindings. #50333
-            let element_tys_iter = (0..max_len).map(|_| tcx.ty_error_with_guaranteed(reported));
+            let element_tys_iter = (0..max_len).map(|_| tcx.ty_error(reported));
             for (_, elem) in elements.iter().enumerate_and_adjust(max_len, ddpos) {
-                self.check_pat(elem, tcx.ty_error_with_guaranteed(reported), def_bm, ti);
+                self.check_pat(elem, tcx.ty_error(reported), def_bm, ti);
             }
-            tcx.mk_tup(element_tys_iter)
+            tcx.mk_tup_from_iter(element_tys_iter)
         } else {
             for (i, elem) in elements.iter().enumerate_and_adjust(max_len, ddpos) {
                 self.check_pat(elem, element_tys[i], def_bm, ti);
@@ -1341,9 +1356,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             let ident = tcx.adjust_ident(field.ident, variant.def_id);
             let field_ty = match used_fields.entry(ident) {
                 Occupied(occupied) => {
-                    self.error_field_already_bound(span, field.ident, *occupied.get());
                     no_field_errors = false;
-                    tcx.ty_error()
+                    let guar = self.error_field_already_bound(span, field.ident, *occupied.get());
+                    tcx.ty_error(guar)
                 }
                 Vacant(vacant) => {
                     vacant.insert(span);
@@ -1357,7 +1372,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         .unwrap_or_else(|| {
                             inexistent_fields.push(field);
                             no_field_errors = false;
-                            tcx.ty_error()
+                            tcx.ty_error_misc()
                         })
                 }
             };
@@ -1528,7 +1543,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         err.emit();
     }
 
-    fn error_field_already_bound(&self, span: Span, ident: Ident, other_field: Span) {
+    fn error_field_already_bound(
+        &self,
+        span: Span,
+        ident: Ident,
+        other_field: Span,
+    ) -> ErrorGuaranteed {
         struct_span_err!(
             self.tcx.sess,
             span,
@@ -1538,7 +1558,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         )
         .span_label(span, format!("multiple uses of `{ident}` in pattern"))
         .span_label(other_field, format!("first use of `{ident}`"))
-        .emit();
+        .emit()
     }
 
     fn error_inexistent_fields(
@@ -1911,19 +1931,22 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         ti: TopInfo<'tcx>,
     ) -> Ty<'tcx> {
         let tcx = self.tcx;
-        let (box_ty, inner_ty) = if self.check_dereferenceable(span, expected, inner) {
-            // Here, `demand::subtype` is good enough, but I don't
-            // think any errors can be introduced by using `demand::eqtype`.
-            let inner_ty = self.next_ty_var(TypeVariableOrigin {
-                kind: TypeVariableOriginKind::TypeInference,
-                span: inner.span,
-            });
-            let box_ty = tcx.mk_box(inner_ty);
-            self.demand_eqtype_pat(span, expected, box_ty, ti);
-            (box_ty, inner_ty)
-        } else {
-            let err = tcx.ty_error();
-            (err, err)
+        let (box_ty, inner_ty) = match self.check_dereferenceable(span, expected, inner) {
+            Ok(()) => {
+                // Here, `demand::subtype` is good enough, but I don't
+                // think any errors can be introduced by using `demand::eqtype`.
+                let inner_ty = self.next_ty_var(TypeVariableOrigin {
+                    kind: TypeVariableOriginKind::TypeInference,
+                    span: inner.span,
+                });
+                let box_ty = tcx.mk_box(inner_ty);
+                self.demand_eqtype_pat(span, expected, box_ty, ti);
+                (box_ty, inner_ty)
+            }
+            Err(guar) => {
+                let err = tcx.ty_error(guar);
+                (err, err)
+            }
         };
         self.check_pat(inner, inner_ty, def_bm, ti);
         box_ty
@@ -1941,37 +1964,40 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     ) -> Ty<'tcx> {
         let tcx = self.tcx;
         let expected = self.shallow_resolve(expected);
-        let (ref_ty, inner_ty) = if self.check_dereferenceable(pat.span, expected, inner) {
-            // `demand::subtype` would be good enough, but using `eqtype` turns
-            // out to be equally general. See (note_1) for details.
-
-            // Take region, inner-type from expected type if we can,
-            // to avoid creating needless variables. This also helps with
-            // the bad interactions of the given hack detailed in (note_1).
-            debug!("check_pat_ref: expected={:?}", expected);
-            match *expected.kind() {
-                ty::Ref(_, r_ty, r_mutbl) if r_mutbl == mutbl => (expected, r_ty),
-                _ => {
-                    let inner_ty = self.next_ty_var(TypeVariableOrigin {
-                        kind: TypeVariableOriginKind::TypeInference,
-                        span: inner.span,
-                    });
-                    let ref_ty = self.new_ref_ty(pat.span, mutbl, inner_ty);
-                    debug!("check_pat_ref: demanding {:?} = {:?}", expected, ref_ty);
-                    let err = self.demand_eqtype_pat_diag(pat.span, expected, ref_ty, ti);
-
-                    // Look for a case like `fn foo(&foo: u32)` and suggest
-                    // `fn foo(foo: &u32)`
-                    if let Some(mut err) = err {
-                        self.borrow_pat_suggestion(&mut err, pat);
-                        err.emit();
+        let (ref_ty, inner_ty) = match self.check_dereferenceable(pat.span, expected, inner) {
+            Ok(()) => {
+                // `demand::subtype` would be good enough, but using `eqtype` turns
+                // out to be equally general. See (note_1) for details.
+
+                // Take region, inner-type from expected type if we can,
+                // to avoid creating needless variables. This also helps with
+                // the bad interactions of the given hack detailed in (note_1).
+                debug!("check_pat_ref: expected={:?}", expected);
+                match *expected.kind() {
+                    ty::Ref(_, r_ty, r_mutbl) if r_mutbl == mutbl => (expected, r_ty),
+                    _ => {
+                        let inner_ty = self.next_ty_var(TypeVariableOrigin {
+                            kind: TypeVariableOriginKind::TypeInference,
+                            span: inner.span,
+                        });
+                        let ref_ty = self.new_ref_ty(pat.span, mutbl, inner_ty);
+                        debug!("check_pat_ref: demanding {:?} = {:?}", expected, ref_ty);
+                        let err = self.demand_eqtype_pat_diag(pat.span, expected, ref_ty, ti);
+
+                        // Look for a case like `fn foo(&foo: u32)` and suggest
+                        // `fn foo(foo: &u32)`
+                        if let Some(mut err) = err {
+                            self.borrow_pat_suggestion(&mut err, pat);
+                            err.emit();
+                        }
+                        (ref_ty, inner_ty)
                     }
-                    (ref_ty, inner_ty)
                 }
             }
-        } else {
-            let err = tcx.ty_error();
-            (err, err)
+            Err(guar) => {
+                let err = tcx.ty_error(guar);
+                (err, err)
+            }
         };
         self.check_pat(inner, inner_ty, def_bm, ti);
         ref_ty
@@ -2019,10 +2045,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             ty::Slice(element_ty) => (element_ty, Some(expected), expected),
             // The expected type must be an array or slice, but was neither, so error.
             _ => {
-                if !expected.references_error() {
-                    self.error_expected_array_or_slice(span, expected, ti);
-                }
-                let err = self.tcx.ty_error();
+                let guar = expected
+                    .error_reported()
+                    .err()
+                    .unwrap_or_else(|| self.error_expected_array_or_slice(span, expected, ti));
+                let err = self.tcx.ty_error(guar);
                 (err, Some(err), err)
             }
         };
@@ -2055,7 +2082,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         len: ty::Const<'tcx>,
         min_len: u64,
     ) -> (Option<Ty<'tcx>>, Ty<'tcx>) {
-        if let Some(len) = len.try_eval_usize(self.tcx, self.param_env) {
+        let guar = if let Some(len) = len.try_eval_target_usize(self.tcx, self.param_env) {
             // Now we know the length...
             if slice.is_none() {
                 // ...and since there is no variable-length pattern,
@@ -2065,7 +2092,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     return (None, arr_ty);
                 }
 
-                self.error_scrutinee_inconsistent_length(span, min_len, len);
+                self.error_scrutinee_inconsistent_length(span, min_len, len)
             } else if let Some(pat_len) = len.checked_sub(min_len) {
                 // The variable-length pattern was there,
                 // so it has an array type with the remaining elements left as its size...
@@ -2073,7 +2100,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             } else {
                 // ...however, in this case, there were no remaining elements.
                 // That is, the slice pattern requires more than the array type offers.
-                self.error_scrutinee_with_rest_inconsistent_length(span, min_len, len);
+                self.error_scrutinee_with_rest_inconsistent_length(span, min_len, len)
             }
         } else if slice.is_none() {
             // We have a pattern with a fixed length,
@@ -2085,14 +2112,19 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // We have a variable-length pattern and don't know the array length.
             // This happens if we have e.g.,
             // `let [a, b, ..] = arr` where `arr: [T; N]` where `const N: usize`.
-            self.error_scrutinee_unfixed_length(span);
-        }
+            self.error_scrutinee_unfixed_length(span)
+        };
 
         // If we get here, we must have emitted an error.
-        (Some(self.tcx.ty_error()), arr_ty)
+        (Some(self.tcx.ty_error(guar)), arr_ty)
     }
 
-    fn error_scrutinee_inconsistent_length(&self, span: Span, min_len: u64, size: u64) {
+    fn error_scrutinee_inconsistent_length(
+        &self,
+        span: Span,
+        min_len: u64,
+        size: u64,
+    ) -> ErrorGuaranteed {
         struct_span_err!(
             self.tcx.sess,
             span,
@@ -2103,10 +2135,15 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             size,
         )
         .span_label(span, format!("expected {} element{}", size, pluralize!(size)))
-        .emit();
+        .emit()
     }
 
-    fn error_scrutinee_with_rest_inconsistent_length(&self, span: Span, min_len: u64, size: u64) {
+    fn error_scrutinee_with_rest_inconsistent_length(
+        &self,
+        span: Span,
+        min_len: u64,
+        size: u64,
+    ) -> ErrorGuaranteed {
         struct_span_err!(
             self.tcx.sess,
             span,
@@ -2120,20 +2157,25 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             span,
             format!("pattern cannot match array of {} element{}", size, pluralize!(size),),
         )
-        .emit();
+        .emit()
     }
 
-    fn error_scrutinee_unfixed_length(&self, span: Span) {
+    fn error_scrutinee_unfixed_length(&self, span: Span) -> ErrorGuaranteed {
         struct_span_err!(
             self.tcx.sess,
             span,
             E0730,
             "cannot pattern-match on an array without a fixed length",
         )
-        .emit();
+        .emit()
     }
 
-    fn error_expected_array_or_slice(&self, span: Span, expected_ty: Ty<'tcx>, ti: TopInfo<'tcx>) {
+    fn error_expected_array_or_slice(
+        &self,
+        span: Span,
+        expected_ty: Ty<'tcx>,
+        ti: TopInfo<'tcx>,
+    ) -> ErrorGuaranteed {
         let mut err = struct_span_err!(
             self.tcx.sess,
             span,
@@ -2146,7 +2188,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             err.help("the semantics of slice patterns changed recently; see issue #62254");
         } else if self.autoderef(span, expected_ty)
             .any(|(ty, _)| matches!(ty.kind(), ty::Slice(..) | ty::Array(..)))
-            && let (Some(span), true) = (ti.span, ti.origin_expr)
+            && let Some(span) = ti.span
+            && let Some(_) = ti.origin_expr
             && let Ok(snippet) = self.tcx.sess.source_map().span_to_snippet(span)
         {
             let ty = self.resolve_vars_if_possible(ti.expected);
@@ -2176,7 +2219,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             }
         }
         err.span_label(span, format!("pattern cannot match with input type `{expected_ty}`"));
-        err.emit();
+        err.emit()
     }
 
     fn is_slice_or_array_or_vector(&self, ty: Ty<'tcx>) -> (bool, Ty<'tcx>) {
diff --git a/compiler/rustc_hir_typeck/src/place_op.rs b/compiler/rustc_hir_typeck/src/place_op.rs
index ae0df5aa8..2cca45de5 100644
--- a/compiler/rustc_hir_typeck/src/place_op.rs
+++ b/compiler/rustc_hir_typeck/src/place_op.rs
@@ -11,7 +11,6 @@ use rustc_middle::ty::adjustment::{AllowTwoPhase, AutoBorrow, AutoBorrowMutabili
 use rustc_middle::ty::{self, Ty};
 use rustc_span::symbol::{sym, Ident};
 use rustc_span::Span;
-use std::slice;
 
 impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     /// Type-check `*oprnd_expr` with `oprnd_expr` type-checked already.
@@ -91,10 +90,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             );
         }
         let reported = err.emit();
-        Some((
-            self.tcx.ty_error_with_guaranteed(reported),
-            self.tcx.ty_error_with_guaranteed(reported),
-        ))
+        Some((self.tcx.ty_error(reported), self.tcx.ty_error(reported)))
     }
 
     /// To type-check `base_expr[index_expr]`, we progressively autoderef
@@ -396,11 +392,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 Some(self.typeck_results.borrow().node_substs(expr.hir_id).type_at(1))
             }
         };
-        let arg_tys = match arg_ty {
-            None => &[],
-            Some(ref ty) => slice::from_ref(ty),
-        };
-
+        let arg_tys = arg_ty.as_slice();
         let method = self.try_mutable_overloaded_place_op(expr.span, base_ty, arg_tys, op);
         let method = match method {
             Some(ok) => self.register_infer_ok_obligations(ok),
diff --git a/compiler/rustc_hir_typeck/src/upvar.rs b/compiler/rustc_hir_typeck/src/upvar.rs
index e12a575d5..4a432328c 100644
--- a/compiler/rustc_hir_typeck/src/upvar.rs
+++ b/compiler/rustc_hir_typeck/src/upvar.rs
@@ -231,7 +231,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         // We now fake capture information for all variables that are mentioned within the closure
         // We do this after handling migrations so that min_captures computes before
-        if !enable_precise_capture(self.tcx, span) {
+        if !enable_precise_capture(span) {
             let mut capture_information: InferredCaptureInformation<'tcx> = Default::default();
 
             if let Some(upvars) = self.tcx.upvars_mentioned(closure_def_id) {
@@ -265,7 +265,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
             // If we have an origin, store it.
             if let Some(origin) = origin {
-                let origin = if enable_precise_capture(self.tcx, span) {
+                let origin = if enable_precise_capture(span) {
                     (origin.0, origin.1)
                 } else {
                     (origin.0, Place { projections: vec![], ..origin.1 })
@@ -301,7 +301,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         // Build a tuple (U0..Un) of the final upvar types U0..Un
         // and unify the upvar tuple type in the closure with it:
-        let final_tupled_upvars_type = self.tcx.mk_tup(final_upvar_tys.iter());
+        let final_tupled_upvars_type = self.tcx.mk_tup(&final_upvar_tys);
         self.demand_suptype(span, substs.tupled_upvars_ty(), final_tupled_upvars_type);
 
         let fake_reads = delegate
@@ -315,8 +315,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             self.typeck_results.borrow_mut().closure_size_eval.insert(
                 closure_def_id,
                 ClosureSizeProfileData {
-                    before_feature_tys: self.tcx.mk_tup(before_feature_tys.into_iter()),
-                    after_feature_tys: self.tcx.mk_tup(after_feature_tys.into_iter()),
+                    before_feature_tys: self.tcx.mk_tup(&before_feature_tys),
+                    after_feature_tys: self.tcx.mk_tup(&after_feature_tys),
                 },
             );
         }
@@ -526,10 +526,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 PlaceBase::Upvar(upvar_id) => upvar_id.var_path.hir_id,
                 base => bug!("Expected upvar, found={:?}", base),
             };
+            let var_ident = self.tcx.hir().ident(var_hir_id);
 
             let Some(min_cap_list) = root_var_min_capture_list.get_mut(&var_hir_id) else {
                 let mutability = self.determine_capture_mutability(&typeck_results, &place);
                 let min_cap_list = vec![ty::CapturedPlace {
+                    var_ident,
                     place,
                     info: capture_info,
                     mutability,
@@ -628,6 +630,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             if !ancestor_found {
                 let mutability = self.determine_capture_mutability(&typeck_results, &place);
                 let captured_place = ty::CapturedPlace {
+                    var_ident,
                     place,
                     info: updated_capture_info,
                     mutability,
@@ -1240,8 +1243,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     ///
     /// This will make more sense with an example:
     ///
-    /// ```rust
-    /// #![feature(capture_disjoint_fields)]
+    /// ```rust,edition2021
     ///
     /// struct FancyInteger(i32); // This implements Drop
     ///
@@ -2221,7 +2223,7 @@ fn determine_place_ancestry_relation<'tcx>(
 ///     || drop(&*m.a.field_of_a)
 ///     // Here we really do want to capture `*m.a` because that outlives `'static`
 ///
-///     // If we capture `m`, then the closure no longer outlives `'static'
+///     // If we capture `m`, then the closure no longer outlives `'static`
 ///     // it is constrained to `'a`
 /// }
 /// ```
@@ -2247,12 +2249,10 @@ fn truncate_capture_for_optimization(
     (place, curr_mode)
 }
 
-/// Precise capture is enabled if the feature gate `capture_disjoint_fields` is enabled or if
-/// user is using Rust Edition 2021 or higher.
-///
+/// Precise capture is enabled if user is using Rust Edition 2021 or higher.
 /// `span` is the span of the closure.
-fn enable_precise_capture(tcx: TyCtxt<'_>, span: Span) -> bool {
+fn enable_precise_capture(span: Span) -> bool {
     // We use span here to ensure that if the closure was generated by a macro with a different
     // edition.
-    tcx.features().capture_disjoint_fields || span.rust_2021()
+    span.rust_2021()
 }
diff --git a/compiler/rustc_hir_typeck/src/writeback.rs b/compiler/rustc_hir_typeck/src/writeback.rs
index 250f4cd3f..00348f3af 100644
--- a/compiler/rustc_hir_typeck/src/writeback.rs
+++ b/compiler/rustc_hir_typeck/src/writeback.rs
@@ -14,7 +14,7 @@ use rustc_middle::hir::place::Place as HirPlace;
 use rustc_middle::mir::FakeReadCause;
 use rustc_middle::ty::adjustment::{Adjust, Adjustment, PointerCast};
 use rustc_middle::ty::fold::{TypeFoldable, TypeFolder, TypeSuperFoldable};
-use rustc_middle::ty::visit::{TypeSuperVisitable, TypeVisitable};
+use rustc_middle::ty::visit::{TypeSuperVisitable, TypeVisitable, TypeVisitableExt};
 use rustc_middle::ty::TypeckResults;
 use rustc_middle::ty::{self, ClosureSizeProfileData, Ty, TyCtxt};
 use rustc_span::symbol::sym;
@@ -40,8 +40,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         &self,
         body: &'tcx hir::Body<'tcx>,
     ) -> &'tcx ty::TypeckResults<'tcx> {
-        let item_id = self.tcx.hir().body_owner(body.id());
-        let item_def_id = self.tcx.hir().local_def_id(item_id);
+        let item_def_id = self.tcx.hir().body_owner_def_id(body.id());
 
         // This attribute causes us to dump some writeback information
         // in the form of errors, which is used for unit tests.
@@ -55,7 +54,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // Type only exists for constants and statics, not functions.
         match self.tcx.hir().body_owner_kind(item_def_id) {
             hir::BodyOwnerKind::Const | hir::BodyOwnerKind::Static(_) => {
-                wbcx.visit_node_id(body.value.span, item_id);
+                let item_hir_id = self.tcx.hir().local_def_id_to_hir_id(item_def_id);
+                wbcx.visit_node_id(body.value.span, item_hir_id);
             }
             hir::BodyOwnerKind::Closure | hir::BodyOwnerKind::Fn => (),
         }
@@ -545,6 +545,10 @@ impl<'cx, 'tcx> WritebackCx<'cx, 'tcx> {
         assert_eq!(fcx_typeck_results.hir_owner, self.typeck_results.hir_owner);
         self.typeck_results.generator_interior_types =
             fcx_typeck_results.generator_interior_types.clone();
+        for (&expr_def_id, predicates) in fcx_typeck_results.generator_interior_predicates.iter() {
+            let predicates = self.resolve(predicates.clone(), &self.fcx.tcx.def_span(expr_def_id));
+            self.typeck_results.generator_interior_predicates.insert(expr_def_id, predicates);
+        }
     }
 
     #[instrument(skip(self), level = "debug")]
@@ -557,7 +561,7 @@ impl<'cx, 'tcx> WritebackCx<'cx, 'tcx> {
             struct RecursionChecker {
                 def_id: LocalDefId,
             }
-            impl<'tcx> ty::TypeVisitor<'tcx> for RecursionChecker {
+            impl<'tcx> ty::TypeVisitor<TyCtxt<'tcx>> for RecursionChecker {
                 type BreakTy = ();
                 fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
                     if let ty::Alias(ty::Opaque, ty::AliasTy { def_id, .. }) = *t.kind() {
@@ -681,7 +685,7 @@ impl<'cx, 'tcx> WritebackCx<'cx, 'tcx> {
 
     fn resolve<T>(&mut self, x: T, span: &dyn Locatable) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         let mut resolver = Resolver::new(self.fcx, span, self.body);
         let x = x.fold_with(&mut resolver);
@@ -759,8 +763,8 @@ struct EraseEarlyRegions<'tcx> {
     tcx: TyCtxt<'tcx>,
 }
 
-impl<'tcx> TypeFolder<'tcx> for EraseEarlyRegions<'tcx> {
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for EraseEarlyRegions<'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
     fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
@@ -775,8 +779,8 @@ impl<'tcx> TypeFolder<'tcx> for EraseEarlyRegions<'tcx> {
     }
 }
 
-impl<'cx, 'tcx> TypeFolder<'tcx> for Resolver<'cx, 'tcx> {
-    fn tcx<'a>(&'a self) -> TyCtxt<'tcx> {
+impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Resolver<'cx, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
@@ -793,7 +797,7 @@ impl<'cx, 'tcx> TypeFolder<'tcx> for Resolver<'cx, 'tcx> {
                 debug!("Resolver::fold_ty: input type `{:?}` not fully resolvable", t);
                 let e = self.report_error(t);
                 self.replaced_with_error = Some(e);
-                self.tcx().ty_error_with_guaranteed(e)
+                self.interner().ty_error(e)
             }
         }
     }
@@ -810,7 +814,7 @@ impl<'cx, 'tcx> TypeFolder<'tcx> for Resolver<'cx, 'tcx> {
                 debug!("Resolver::fold_const: input const `{:?}` not fully resolvable", ct);
                 let e = self.report_error(ct);
                 self.replaced_with_error = Some(e);
-                self.tcx().const_error_with_guaranteed(ct.ty(), e)
+                self.interner().const_error_with_guaranteed(ct.ty(), e)
             }
         }
     }
diff --git a/compiler/rustc_incremental/Cargo.toml b/compiler/rustc_incremental/Cargo.toml
index 179e85f32..ad8939395 100644
--- a/compiler/rustc_incremental/Cargo.toml
+++ b/compiler/rustc_incremental/Cargo.toml
@@ -6,16 +6,17 @@ edition = "2021"
 [lib]
 
 [dependencies]
-rustc_graphviz = { path = "../rustc_graphviz" }
-tracing = "0.1"
 rand = "0.8.4"
-rustc_middle = { path = "../rustc_middle" }
+rustc_ast = { path = "../rustc_ast" }
 rustc_data_structures = { path = "../rustc_data_structures" }
+rustc_errors = { path = "../rustc_errors" }
+rustc_fs_util = { path = "../rustc_fs_util" }
+rustc_graphviz = { path = "../rustc_graphviz" }
 rustc_hir = { path = "../rustc_hir" }
-rustc_serialize = { path = "../rustc_serialize" }
-rustc_ast = { path = "../rustc_ast" }
 rustc_macros = { path = "../rustc_macros" }
-rustc_span = { path = "../rustc_span" }
-rustc_fs_util = { path = "../rustc_fs_util" }
+rustc_middle = { path = "../rustc_middle" }
+rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
-rustc_errors = { path = "../rustc_errors" }
+rustc_span = { path = "../rustc_span" }
+thin-vec = "0.2.12"
+tracing = "0.1"
diff --git a/compiler/rustc_incremental/locales/en-US.ftl b/compiler/rustc_incremental/locales/en-US.ftl
new file mode 100644
index 000000000..4852ee0d9
--- /dev/null
+++ b/compiler/rustc_incremental/locales/en-US.ftl
@@ -0,0 +1,118 @@
+incremental_unrecognized_depnode = unrecognized `DepNode` variant: {$name}
+
+incremental_missing_depnode = missing `DepNode` variant
+
+incremental_missing_if_this_changed = no `#[rustc_if_this_changed]` annotation detected
+
+incremental_no_path = no path from `{$source}` to `{$target}`
+
+incremental_ok = OK
+
+incremental_unknown_reuse_kind = unknown cgu-reuse-kind `{$kind}` specified
+
+incremental_missing_query_depgraph =
+    found CGU-reuse attribute but `-Zquery-dep-graph` was not specified
+
+incremental_malformed_cgu_name =
+    found malformed codegen unit name `{$user_path}`. codegen units names must always start with the name of the crate (`{$crate_name}` in this case).
+
+incremental_no_module_named =
+    no module named `{$user_path}` (mangled: {$cgu_name}). available modules: {$cgu_names}
+
+incremental_field_associated_value_expected = associated value expected for `{$name}`
+
+incremental_no_field = no field `{$name}`
+
+incremental_assertion_auto =
+    `except` specified DepNodes that can not be affected for \"{$name}\": \"{$e}\"
+
+incremental_undefined_clean_dirty_assertions_item =
+    clean/dirty auto-assertions not yet defined for Node::Item.node={$kind}
+
+incremental_undefined_clean_dirty_assertions =
+    clean/dirty auto-assertions not yet defined for {$kind}
+
+incremental_repeated_depnode_label = dep-node label `{$label}` is repeated
+
+incremental_unrecognized_depnode_label = dep-node label `{$label}` not recognized
+
+incremental_not_dirty = `{$dep_node_str}` should be dirty but is not
+
+incremental_not_clean = `{$dep_node_str}` should be clean but is not
+
+incremental_not_loaded = `{$dep_node_str}` should have been loaded from disk but it was not
+
+incremental_unknown_item = unknown item `{$name}`
+
+incremental_no_cfg = no cfg attribute
+
+incremental_associated_value_expected_for = associated value expected for `{$ident}`
+
+incremental_associated_value_expected = expected an associated value
+
+incremental_unchecked_clean = found unchecked `#[rustc_clean]` attribute
+
+incremental_delete_old = unable to delete old {$name} at `{$path}`: {$err}
+
+incremental_create_new = failed to create {$name} at `{$path}`: {$err}
+
+incremental_write_new = failed to write {$name} to `{$path}`: {$err}
+
+incremental_canonicalize_path = incremental compilation: error canonicalizing path `{$path}`: {$err}
+
+incremental_create_incr_comp_dir =
+    could not create incremental compilation {$tag} directory `{$path}`: {$err}
+
+incremental_create_lock =
+    incremental compilation: could not create session directory lock file: {$lock_err}
+incremental_lock_unsupported =
+    the filesystem for the incremental path at {$session_dir} does not appear to support locking, consider changing the incremental path to a filesystem that supports locking or disable incremental compilation
+incremental_cargo_help_1 =
+    incremental compilation can be disabled by setting the environment variable CARGO_INCREMENTAL=0 (see https://doc.rust-lang.org/cargo/reference/profiles.html#incremental)
+incremental_cargo_help_2 =
+    the entire build directory can be changed to a different filesystem by setting the environment variable CARGO_TARGET_DIR to a different path (see https://doc.rust-lang.org/cargo/reference/config.html#buildtarget-dir)
+
+incremental_delete_lock =
+    error deleting lock file for incremental compilation session directory `{$path}`: {$err}
+
+incremental_hard_link_failed =
+    hard linking files in the incremental compilation cache failed. copying files instead. consider moving the cache directory to a file system which supports hard linking in session dir `{$path}`
+
+incremental_delete_partial = failed to delete partly initialized session dir `{$path}`: {$err}
+
+incremental_delete_full = error deleting incremental compilation session directory `{$path}`: {$err}
+
+incremental_finalize = error finalizing incremental compilation session directory `{$path}`: {$err}
+
+incremental_invalid_gc_failed =
+    failed to garbage collect invalid incremental compilation session directory `{$path}`: {$err}
+
+incremental_finalized_gc_failed =
+    failed to garbage collect finalized incremental compilation session directory `{$path}`: {$err}
+
+incremental_session_gc_failed =
+    failed to garbage collect incremental compilation session directory `{$path}`: {$err}
+
+incremental_assert_not_loaded =
+    we asserted that the incremental cache should not be loaded, but it was loaded
+
+incremental_assert_loaded =
+    we asserted that an existing incremental cache directory should be successfully loaded, but it was not
+
+incremental_delete_incompatible =
+    failed to delete invalidated or incompatible incremental compilation session directory contents `{$path}`: {$err}
+
+incremental_load_dep_graph = could not load dep-graph from `{$path}`: {$err}
+
+incremental_decode_incr_cache = could not decode incremental cache: {$err}
+
+incremental_write_dep_graph = failed to write dependency graph to `{$path}`: {$err}
+
+incremental_move_dep_graph = failed to move dependency graph from `{$from}` to `{$to}`: {$err}
+
+incremental_create_dep_graph = failed to create dependency graph at `{$path}`: {$err}
+
+incremental_copy_workproduct_to_cache =
+    error copying object file `{$from}` to incremental directory as `{$to}`: {$err}
+
+incremental_delete_workproduct = file-system error deleting outdated file `{$path}`: {$err}
diff --git a/compiler/rustc_incremental/src/assert_dep_graph.rs b/compiler/rustc_incremental/src/assert_dep_graph.rs
index 33a9a0cab..22bd12f2e 100644
--- a/compiler/rustc_incremental/src/assert_dep_graph.rs
+++ b/compiler/rustc_incremental/src/assert_dep_graph.rs
@@ -33,12 +33,13 @@
 //! fn baz() { foo(); }
 //! ```
 
+use crate::errors;
 use rustc_ast as ast;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::graph::implementation::{Direction, NodeIndex, INCOMING, OUTGOING};
 use rustc_graphviz as dot;
 use rustc_hir as hir;
-use rustc_hir::def_id::DefId;
+use rustc_hir::def_id::{DefId, LocalDefId, CRATE_DEF_ID};
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_middle::dep_graph::{
     DepGraphQuery, DepKind, DepNode, DepNodeExt, DepNodeFilter, EdgeFilter,
@@ -74,7 +75,7 @@ pub fn assert_dep_graph(tcx: TyCtxt<'_>) {
         let (if_this_changed, then_this_would_need) = {
             let mut visitor =
                 IfThisChanged { tcx, if_this_changed: vec![], then_this_would_need: vec![] };
-            visitor.process_attrs(hir::CRATE_HIR_ID);
+            visitor.process_attrs(CRATE_DEF_ID);
             tcx.hir().visit_all_item_likes_in_crate(&mut visitor);
             (visitor.if_this_changed, visitor.then_this_would_need)
         };
@@ -119,9 +120,9 @@ impl<'tcx> IfThisChanged<'tcx> {
         value
     }
 
-    fn process_attrs(&mut self, hir_id: hir::HirId) {
-        let def_id = self.tcx.hir().local_def_id(hir_id);
+    fn process_attrs(&mut self, def_id: LocalDefId) {
         let def_path_hash = self.tcx.def_path_hash(def_id.to_def_id());
+        let hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id);
         let attrs = self.tcx.hir().attrs(hir_id);
         for attr in attrs {
             if attr.has_name(sym::rustc_if_this_changed) {
@@ -133,12 +134,10 @@ impl<'tcx> IfThisChanged<'tcx> {
                     Some(n) => {
                         match DepNode::from_label_string(self.tcx, n.as_str(), def_path_hash) {
                             Ok(n) => n,
-                            Err(()) => {
-                                self.tcx.sess.span_fatal(
-                                    attr.span,
-                                    &format!("unrecognized DepNode variant {:?}", n),
-                                );
-                            }
+                            Err(()) => self.tcx.sess.emit_fatal(errors::UnrecognizedDepNode {
+                                span: attr.span,
+                                name: n,
+                            }),
                         }
                     }
                 };
@@ -149,16 +148,14 @@ impl<'tcx> IfThisChanged<'tcx> {
                     Some(n) => {
                         match DepNode::from_label_string(self.tcx, n.as_str(), def_path_hash) {
                             Ok(n) => n,
-                            Err(()) => {
-                                self.tcx.sess.span_fatal(
-                                    attr.span,
-                                    &format!("unrecognized DepNode variant {:?}", n),
-                                );
-                            }
+                            Err(()) => self.tcx.sess.emit_fatal(errors::UnrecognizedDepNode {
+                                span: attr.span,
+                                name: n,
+                            }),
                         }
                     }
                     None => {
-                        self.tcx.sess.span_fatal(attr.span, "missing DepNode variant");
+                        self.tcx.sess.emit_fatal(errors::MissingDepNode { span: attr.span });
                     }
                 };
                 self.then_this_would_need.push((
@@ -180,22 +177,22 @@ impl<'tcx> Visitor<'tcx> for IfThisChanged<'tcx> {
     }
 
     fn visit_item(&mut self, item: &'tcx hir::Item<'tcx>) {
-        self.process_attrs(item.hir_id());
+        self.process_attrs(item.owner_id.def_id);
         intravisit::walk_item(self, item);
     }
 
     fn visit_trait_item(&mut self, trait_item: &'tcx hir::TraitItem<'tcx>) {
-        self.process_attrs(trait_item.hir_id());
+        self.process_attrs(trait_item.owner_id.def_id);
         intravisit::walk_trait_item(self, trait_item);
     }
 
     fn visit_impl_item(&mut self, impl_item: &'tcx hir::ImplItem<'tcx>) {
-        self.process_attrs(impl_item.hir_id());
+        self.process_attrs(impl_item.owner_id.def_id);
         intravisit::walk_impl_item(self, impl_item);
     }
 
     fn visit_field_def(&mut self, s: &'tcx hir::FieldDef<'tcx>) {
-        self.process_attrs(s.hir_id);
+        self.process_attrs(s.def_id);
         intravisit::walk_field_def(self, s);
     }
 }
@@ -204,7 +201,7 @@ fn check_paths<'tcx>(tcx: TyCtxt<'tcx>, if_this_changed: &Sources, then_this_wou
     // Return early here so as not to construct the query, which is not cheap.
     if if_this_changed.is_empty() {
         for &(target_span, _, _, _) in then_this_would_need {
-            tcx.sess.span_err(target_span, "no `#[rustc_if_this_changed]` annotation detected");
+            tcx.sess.emit_err(errors::MissingIfThisChanged { span: target_span });
         }
         return;
     }
@@ -213,16 +210,13 @@ fn check_paths<'tcx>(tcx: TyCtxt<'tcx>, if_this_changed: &Sources, then_this_wou
             let dependents = query.transitive_predecessors(source_dep_node);
             for &(target_span, ref target_pass, _, ref target_dep_node) in then_this_would_need {
                 if !dependents.contains(&target_dep_node) {
-                    tcx.sess.span_err(
-                        target_span,
-                        &format!(
-                            "no path from `{}` to `{}`",
-                            tcx.def_path_str(source_def_id),
-                            target_pass
-                        ),
-                    );
+                    tcx.sess.emit_err(errors::NoPath {
+                        span: target_span,
+                        source: tcx.def_path_str(source_def_id),
+                        target: *target_pass,
+                    });
                 } else {
-                    tcx.sess.span_err(target_span, "OK");
+                    tcx.sess.emit_err(errors::Ok { span: target_span });
                 }
             }
         }
diff --git a/compiler/rustc_incremental/src/assert_module_sources.rs b/compiler/rustc_incremental/src/assert_module_sources.rs
index 89d419bc8..c550e553b 100644
--- a/compiler/rustc_incremental/src/assert_module_sources.rs
+++ b/compiler/rustc_incremental/src/assert_module_sources.rs
@@ -18,10 +18,11 @@
 //! the HIR doesn't change as a result of the annotations, which might
 //! perturb the reuse results.
 //!
-//! `#![rustc_expected_cgu_reuse(module="spike", cfg="rpass2", kind="post-lto")]
+//! `#![rustc_expected_cgu_reuse(module="spike", cfg="rpass2", kind="post-lto")]`
 //! allows for doing a more fine-grained check to see if pre- or post-lto data
 //! was re-used.
 
+use crate::errors;
 use rustc_ast as ast;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_hir::def_id::LOCAL_CRATE;
@@ -29,6 +30,7 @@ use rustc_middle::mir::mono::CodegenUnitNameBuilder;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::cgu_reuse_tracker::*;
 use rustc_span::symbol::{sym, Symbol};
+use thin_vec::ThinVec;
 
 #[allow(missing_docs)]
 pub fn assert_module_sources(tcx: TyCtxt<'_>) {
@@ -66,10 +68,9 @@ impl<'tcx> AssertModuleSource<'tcx> {
                 sym::post_dash_lto => (CguReuse::PostLto, ComparisonKind::Exact),
                 sym::any => (CguReuse::PreLto, ComparisonKind::AtLeast),
                 other => {
-                    self.tcx.sess.span_fatal(
-                        attr.span,
-                        &format!("unknown cgu-reuse-kind `{}` specified", other),
-                    );
+                    self.tcx
+                        .sess
+                        .emit_fatal(errors::UnknownReuseKind { span: attr.span, kind: other });
                 }
             }
         } else {
@@ -77,10 +78,7 @@ impl<'tcx> AssertModuleSource<'tcx> {
         };
 
         if !self.tcx.sess.opts.unstable_opts.query_dep_graph {
-            self.tcx.sess.span_fatal(
-                attr.span,
-                "found CGU-reuse attribute but `-Zquery-dep-graph` was not specified",
-            );
+            self.tcx.sess.emit_fatal(errors::MissingQueryDepGraph { span: attr.span });
         }
 
         if !self.check_config(attr) {
@@ -92,13 +90,11 @@ impl<'tcx> AssertModuleSource<'tcx> {
         let crate_name = self.tcx.crate_name(LOCAL_CRATE).to_string();
 
         if !user_path.starts_with(&crate_name) {
-            let msg = format!(
-                "Found malformed codegen unit name `{}`. \
-                Codegen units names must always start with the name of the \
-                crate (`{}` in this case).",
-                user_path, crate_name
-            );
-            self.tcx.sess.span_fatal(attr.span, &msg);
+            self.tcx.sess.emit_fatal(errors::MalformedCguName {
+                span: attr.span,
+                user_path,
+                crate_name,
+            });
         }
 
         // Split of the "special suffix" if there is one.
@@ -125,15 +121,12 @@ impl<'tcx> AssertModuleSource<'tcx> {
             let mut cgu_names: Vec<&str> =
                 self.available_cgus.iter().map(|cgu| cgu.as_str()).collect();
             cgu_names.sort();
-            self.tcx.sess.span_err(
-                attr.span,
-                &format!(
-                    "no module named `{}` (mangled: {}). Available modules: {}",
-                    user_path,
-                    cgu_name,
-                    cgu_names.join(", ")
-                ),
-            );
+            self.tcx.sess.emit_err(errors::NoModuleNamed {
+                span: attr.span,
+                user_path,
+                cgu_name,
+                cgu_names: cgu_names.join(", "),
+            });
         }
 
         self.tcx.sess.cgu_reuse_tracker.set_expectation(
@@ -146,20 +139,20 @@ impl<'tcx> AssertModuleSource<'tcx> {
     }
 
     fn field(&self, attr: &ast::Attribute, name: Symbol) -> Symbol {
-        for item in attr.meta_item_list().unwrap_or_else(Vec::new) {
+        for item in attr.meta_item_list().unwrap_or_else(ThinVec::new) {
             if item.has_name(name) {
                 if let Some(value) = item.value_str() {
                     return value;
                 } else {
-                    self.tcx.sess.span_fatal(
-                        item.span(),
-                        &format!("associated value expected for `{}`", name),
-                    );
+                    self.tcx.sess.emit_fatal(errors::FieldAssociatedValueExpected {
+                        span: item.span(),
+                        name,
+                    });
                 }
             }
         }
 
-        self.tcx.sess.span_fatal(attr.span, &format!("no field `{}`", name));
+        self.tcx.sess.emit_fatal(errors::NoField { span: attr.span, name });
     }
 
     /// Scan for a `cfg="foo"` attribute and check whether we have a
diff --git a/compiler/rustc_incremental/src/errors.rs b/compiler/rustc_incremental/src/errors.rs
new file mode 100644
index 000000000..deb876783
--- /dev/null
+++ b/compiler/rustc_incremental/src/errors.rs
@@ -0,0 +1,364 @@
+use rustc_macros::Diagnostic;
+use rustc_span::{symbol::Ident, Span, Symbol};
+use std::path::{Path, PathBuf};
+
+#[derive(Diagnostic)]
+#[diag(incremental_unrecognized_depnode)]
+pub struct UnrecognizedDepNode {
+    #[primary_span]
+    pub span: Span,
+    pub name: Symbol,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_missing_depnode)]
+pub struct MissingDepNode {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_missing_if_this_changed)]
+pub struct MissingIfThisChanged {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_ok)]
+pub struct Ok {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_no_path)]
+pub struct NoPath {
+    #[primary_span]
+    pub span: Span,
+    pub target: Symbol,
+    pub source: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_unknown_reuse_kind)]
+pub struct UnknownReuseKind {
+    #[primary_span]
+    pub span: Span,
+    pub kind: Symbol,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_missing_query_depgraph)]
+pub struct MissingQueryDepGraph {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_malformed_cgu_name)]
+pub struct MalformedCguName {
+    #[primary_span]
+    pub span: Span,
+    pub user_path: String,
+    pub crate_name: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_no_module_named)]
+pub struct NoModuleNamed<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub user_path: &'a str,
+    pub cgu_name: Symbol,
+    pub cgu_names: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_field_associated_value_expected)]
+pub struct FieldAssociatedValueExpected {
+    #[primary_span]
+    pub span: Span,
+    pub name: Symbol,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_no_field)]
+pub struct NoField {
+    #[primary_span]
+    pub span: Span,
+    pub name: Symbol,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_assertion_auto)]
+pub struct AssertionAuto<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub name: &'a str,
+    pub e: &'a str,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_undefined_clean_dirty_assertions_item)]
+pub struct UndefinedCleanDirtyItem {
+    #[primary_span]
+    pub span: Span,
+    pub kind: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_undefined_clean_dirty_assertions)]
+pub struct UndefinedCleanDirty {
+    #[primary_span]
+    pub span: Span,
+    pub kind: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_repeated_depnode_label)]
+pub struct RepeatedDepNodeLabel<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub label: &'a str,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_unrecognized_depnode_label)]
+pub struct UnrecognizedDepNodeLabel<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub label: &'a str,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_not_dirty)]
+pub struct NotDirty<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub dep_node_str: &'a str,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_not_clean)]
+pub struct NotClean<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub dep_node_str: &'a str,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_not_loaded)]
+pub struct NotLoaded<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub dep_node_str: &'a str,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_unknown_item)]
+pub struct UnknownItem {
+    #[primary_span]
+    pub span: Span,
+    pub name: Symbol,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_no_cfg)]
+pub struct NoCfg {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_associated_value_expected_for)]
+pub struct AssociatedValueExpectedFor {
+    #[primary_span]
+    pub span: Span,
+    pub ident: Ident,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_associated_value_expected)]
+pub struct AssociatedValueExpected {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_unchecked_clean)]
+pub struct UncheckedClean {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_delete_old)]
+pub struct DeleteOld<'a> {
+    pub name: &'a str,
+    pub path: PathBuf,
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_create_new)]
+pub struct CreateNew<'a> {
+    pub name: &'a str,
+    pub path: PathBuf,
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_write_new)]
+pub struct WriteNew<'a> {
+    pub name: &'a str,
+    pub path: PathBuf,
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_canonicalize_path)]
+pub struct CanonicalizePath {
+    pub path: PathBuf,
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_create_incr_comp_dir)]
+pub struct CreateIncrCompDir<'a> {
+    pub tag: &'a str,
+    pub path: &'a Path,
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_create_lock)]
+pub struct CreateLock<'a> {
+    pub lock_err: std::io::Error,
+    pub session_dir: &'a Path,
+    #[note(incremental_lock_unsupported)]
+    pub is_unsupported_lock: Option<()>,
+    #[help(incremental_cargo_help_1)]
+    #[help(incremental_cargo_help_2)]
+    pub is_cargo: Option<()>,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_delete_lock)]
+pub struct DeleteLock<'a> {
+    pub path: &'a Path,
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_hard_link_failed)]
+pub struct HardLinkFailed<'a> {
+    pub path: &'a Path,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_delete_partial)]
+pub struct DeletePartial<'a> {
+    pub path: &'a Path,
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_delete_full)]
+pub struct DeleteFull<'a> {
+    pub path: &'a Path,
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_finalize)]
+pub struct Finalize<'a> {
+    pub path: &'a Path,
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_invalid_gc_failed)]
+pub struct InvalidGcFailed<'a> {
+    pub path: &'a Path,
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_finalized_gc_failed)]
+pub struct FinalizedGcFailed<'a> {
+    pub path: &'a Path,
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_session_gc_failed)]
+pub struct SessionGcFailed<'a> {
+    pub path: &'a Path,
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_assert_not_loaded)]
+pub struct AssertNotLoaded;
+
+#[derive(Diagnostic)]
+#[diag(incremental_assert_loaded)]
+pub struct AssertLoaded;
+
+#[derive(Diagnostic)]
+#[diag(incremental_delete_incompatible)]
+pub struct DeleteIncompatible {
+    pub path: PathBuf,
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_load_dep_graph)]
+pub struct LoadDepGraph {
+    pub path: PathBuf,
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_decode_incr_cache)]
+pub struct DecodeIncrCache {
+    pub err: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_write_dep_graph)]
+pub struct WriteDepGraph<'a> {
+    pub path: &'a Path,
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_move_dep_graph)]
+pub struct MoveDepGraph<'a> {
+    pub from: &'a Path,
+    pub to: &'a Path,
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_create_dep_graph)]
+pub struct CreateDepGraph<'a> {
+    pub path: &'a Path,
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_copy_workproduct_to_cache)]
+pub struct CopyWorkProductToCache<'a> {
+    pub from: &'a Path,
+    pub to: &'a Path,
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(incremental_delete_workproduct)]
+pub struct DeleteWorkProduct<'a> {
+    pub path: &'a Path,
+    pub err: std::io::Error,
+}
diff --git a/compiler/rustc_incremental/src/lib.rs b/compiler/rustc_incremental/src/lib.rs
index 83dd9a67e..511e466c2 100644
--- a/compiler/rustc_incremental/src/lib.rs
+++ b/compiler/rustc_incremental/src/lib.rs
@@ -2,8 +2,11 @@
 
 #![deny(missing_docs)]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
+#![feature(never_type)]
 #![recursion_limit = "256"]
 #![allow(rustc::potential_query_instability)]
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 
 #[macro_use]
 extern crate rustc_middle;
@@ -12,6 +15,7 @@ extern crate tracing;
 
 mod assert_dep_graph;
 pub mod assert_module_sources;
+mod errors;
 mod persist;
 
 use assert_dep_graph::assert_dep_graph;
@@ -27,3 +31,8 @@ pub use persist::save_dep_graph;
 pub use persist::save_work_product_index;
 pub use persist::LoadResult;
 pub use persist::{build_dep_graph, load_dep_graph, DepGraphFuture};
+
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
+use rustc_macros::fluent_messages;
+
+fluent_messages! { "../locales/en-US.ftl" }
diff --git a/compiler/rustc_incremental/src/persist/dirty_clean.rs b/compiler/rustc_incremental/src/persist/dirty_clean.rs
index ed7b272b1..b839416c9 100644
--- a/compiler/rustc_incremental/src/persist/dirty_clean.rs
+++ b/compiler/rustc_incremental/src/persist/dirty_clean.rs
@@ -19,6 +19,7 @@
 //! Errors are reported if we are in the suitable configuration but
 //! the required condition is not met.
 
+use crate::errors;
 use rustc_ast::{self as ast, Attribute, NestedMetaItem};
 use rustc_data_structures::fx::FxHashSet;
 use rustc_hir::def_id::LocalDefId;
@@ -30,6 +31,8 @@ use rustc_middle::hir::nested_filter;
 use rustc_middle::ty::TyCtxt;
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::Span;
+use std::iter::FromIterator;
+use thin_vec::ThinVec;
 
 const LOADED_FROM_DISK: Symbol = sym::loaded_from_disk;
 const EXCEPT: Symbol = sym::except;
@@ -196,11 +199,7 @@ impl<'tcx> DirtyCleanVisitor<'tcx> {
         let loaded_from_disk = self.loaded_from_disk(attr);
         for e in except.iter() {
             if !auto.remove(e) {
-                let msg = format!(
-                    "`except` specified DepNodes that can not be affected for \"{}\": \"{}\"",
-                    name, e
-                );
-                self.tcx.sess.span_fatal(attr.span, &msg);
+                self.tcx.sess.emit_fatal(errors::AssertionAuto { span: attr.span, name, e });
             }
         }
         Assertion { clean: auto, dirty: except, loaded_from_disk }
@@ -208,7 +207,7 @@ impl<'tcx> DirtyCleanVisitor<'tcx> {
 
     /// `loaded_from_disk=` attribute value
     fn loaded_from_disk(&self, attr: &Attribute) -> Labels {
-        for item in attr.meta_item_list().unwrap_or_else(Vec::new) {
+        for item in attr.meta_item_list().unwrap_or_else(ThinVec::new) {
             if item.has_name(LOADED_FROM_DISK) {
                 let value = expect_associated_value(self.tcx, &item);
                 return self.resolve_labels(&item, value);
@@ -220,7 +219,7 @@ impl<'tcx> DirtyCleanVisitor<'tcx> {
 
     /// `except=` attribute value
     fn except(&self, attr: &Attribute) -> Labels {
-        for item in attr.meta_item_list().unwrap_or_else(Vec::new) {
+        for item in attr.meta_item_list().unwrap_or_else(ThinVec::new) {
             if item.has_name(EXCEPT) {
                 let value = expect_associated_value(self.tcx, &item);
                 return self.resolve_labels(&item, value);
@@ -282,14 +281,10 @@ impl<'tcx> DirtyCleanVisitor<'tcx> {
                     // An implementation, eg `impl<A> Trait for Foo { .. }`
                     HirItem::Impl { .. } => ("ItemKind::Impl", LABELS_IMPL),
 
-                    _ => self.tcx.sess.span_fatal(
-                        attr.span,
-                        &format!(
-                            "clean/dirty auto-assertions not yet defined \
-                             for Node::Item.node={:?}",
-                            item.kind
-                        ),
-                    ),
+                    _ => self.tcx.sess.emit_fatal(errors::UndefinedCleanDirtyItem {
+                        span: attr.span,
+                        kind: format!("{:?}", item.kind),
+                    }),
                 }
             }
             HirNode::TraitItem(item) => match item.kind {
@@ -302,10 +297,10 @@ impl<'tcx> DirtyCleanVisitor<'tcx> {
                 ImplItemKind::Const(..) => ("NodeImplConst", LABELS_CONST_IN_IMPL),
                 ImplItemKind::Type(..) => ("NodeImplType", LABELS_CONST_IN_IMPL),
             },
-            _ => self.tcx.sess.span_fatal(
-                attr.span,
-                &format!("clean/dirty auto-assertions not yet defined for {:?}", node),
-            ),
+            _ => self.tcx.sess.emit_fatal(errors::UndefinedCleanDirty {
+                span: attr.span,
+                kind: format!("{:?}", node),
+            }),
         };
         let labels =
             Labels::from_iter(labels.iter().flat_map(|s| s.iter().map(|l| (*l).to_string())));
@@ -318,16 +313,15 @@ impl<'tcx> DirtyCleanVisitor<'tcx> {
             let label = label.trim();
             if DepNode::has_label_string(label) {
                 if out.contains(label) {
-                    self.tcx.sess.span_fatal(
-                        item.span(),
-                        &format!("dep-node label `{}` is repeated", label),
-                    );
+                    self.tcx
+                        .sess
+                        .emit_fatal(errors::RepeatedDepNodeLabel { span: item.span(), label });
                 }
                 out.insert(label.to_string());
             } else {
                 self.tcx
                     .sess
-                    .span_fatal(item.span(), &format!("dep-node label `{}` not recognized", label));
+                    .emit_fatal(errors::UnrecognizedDepNodeLabel { span: item.span(), label });
             }
         }
         out
@@ -348,7 +342,7 @@ impl<'tcx> DirtyCleanVisitor<'tcx> {
             let dep_node_str = self.dep_node_str(&dep_node);
             self.tcx
                 .sess
-                .span_err(item_span, &format!("`{}` should be dirty but is not", dep_node_str));
+                .emit_err(errors::NotDirty { span: item_span, dep_node_str: &dep_node_str });
         }
     }
 
@@ -359,7 +353,7 @@ impl<'tcx> DirtyCleanVisitor<'tcx> {
             let dep_node_str = self.dep_node_str(&dep_node);
             self.tcx
                 .sess
-                .span_err(item_span, &format!("`{}` should be clean but is not", dep_node_str));
+                .emit_err(errors::NotClean { span: item_span, dep_node_str: &dep_node_str });
         }
     }
 
@@ -368,10 +362,9 @@ impl<'tcx> DirtyCleanVisitor<'tcx> {
 
         if !self.tcx.dep_graph.debug_was_loaded_from_disk(dep_node) {
             let dep_node_str = self.dep_node_str(&dep_node);
-            self.tcx.sess.span_err(
-                item_span,
-                &format!("`{}` should have been loaded from disk but it was not", dep_node_str),
-            );
+            self.tcx
+                .sess
+                .emit_err(errors::NotLoaded { span: item_span, dep_node_str: &dep_node_str });
         }
     }
 
@@ -406,18 +399,18 @@ fn check_config(tcx: TyCtxt<'_>, attr: &Attribute) -> bool {
     let config = &tcx.sess.parse_sess.config;
     debug!("check_config: config={:?}", config);
     let mut cfg = None;
-    for item in attr.meta_item_list().unwrap_or_else(Vec::new) {
+    for item in attr.meta_item_list().unwrap_or_else(ThinVec::new) {
         if item.has_name(CFG) {
             let value = expect_associated_value(tcx, &item);
             debug!("check_config: searching for cfg {:?}", value);
             cfg = Some(config.contains(&(value, None)));
         } else if !(item.has_name(EXCEPT) || item.has_name(LOADED_FROM_DISK)) {
-            tcx.sess.span_err(attr.span, &format!("unknown item `{}`", item.name_or_empty()));
+            tcx.sess.emit_err(errors::UnknownItem { span: attr.span, name: item.name_or_empty() });
         }
     }
 
     match cfg {
-        None => tcx.sess.span_fatal(attr.span, "no cfg attribute"),
+        None => tcx.sess.emit_fatal(errors::NoCfg { span: attr.span }),
         Some(c) => c,
     }
 }
@@ -426,13 +419,11 @@ fn expect_associated_value(tcx: TyCtxt<'_>, item: &NestedMetaItem) -> Symbol {
     if let Some(value) = item.value_str() {
         value
     } else {
-        let msg = if let Some(ident) = item.ident() {
-            format!("associated value expected for `{}`", ident)
+        if let Some(ident) = item.ident() {
+            tcx.sess.emit_fatal(errors::AssociatedValueExpectedFor { span: item.span(), ident });
         } else {
-            "expected an associated value".to_string()
-        };
-
-        tcx.sess.span_fatal(item.span(), &msg);
+            tcx.sess.emit_fatal(errors::AssociatedValueExpected { span: item.span() });
+        }
     }
 }
 
@@ -456,7 +447,7 @@ impl<'tcx> FindAllAttrs<'tcx> {
     fn report_unchecked_attrs(&self, mut checked_attrs: FxHashSet<ast::AttrId>) {
         for attr in &self.found_attrs {
             if !checked_attrs.contains(&attr.id) {
-                self.tcx.sess.span_err(attr.span, "found unchecked `#[rustc_clean]` attribute");
+                self.tcx.sess.emit_err(errors::UncheckedClean { span: attr.span });
                 checked_attrs.insert(attr.id);
             }
         }
diff --git a/compiler/rustc_incremental/src/persist/file_format.rs b/compiler/rustc_incremental/src/persist/file_format.rs
index 2dbd4b6bc..dc981c617 100644
--- a/compiler/rustc_incremental/src/persist/file_format.rs
+++ b/compiler/rustc_incremental/src/persist/file_format.rs
@@ -9,15 +9,15 @@
 //! compiler versions don't change frequently for the typical user, being
 //! conservative here practically has no downside.
 
-use std::env;
-use std::fs;
-use std::io::{self, Read};
-use std::path::{Path, PathBuf};
-
+use crate::errors;
 use rustc_data_structures::memmap::Mmap;
 use rustc_serialize::opaque::{FileEncodeResult, FileEncoder};
 use rustc_serialize::Encoder;
 use rustc_session::Session;
+use std::env;
+use std::fs;
+use std::io::{self, Read};
+use std::path::{Path, PathBuf};
 
 /// The first few bytes of files generated by incremental compilation.
 const FILE_MAGIC: &[u8] = b"RSIC";
@@ -60,12 +60,7 @@ where
         }
         Err(err) if err.kind() == io::ErrorKind::NotFound => (),
         Err(err) => {
-            sess.err(&format!(
-                "unable to delete old {} at `{}`: {}",
-                name,
-                path_buf.display(),
-                err
-            ));
+            sess.emit_err(errors::DeleteOld { name, path: path_buf, err });
             return;
         }
     }
@@ -73,7 +68,7 @@ where
     let mut encoder = match FileEncoder::new(&path_buf) {
         Ok(encoder) => encoder,
         Err(err) => {
-            sess.err(&format!("failed to create {} at `{}`: {}", name, path_buf.display(), err));
+            sess.emit_err(errors::CreateNew { name, path: path_buf, err });
             return;
         }
     };
@@ -90,7 +85,7 @@ where
             debug!("save: data written to disk successfully");
         }
         Err(err) => {
-            sess.err(&format!("failed to write {} to `{}`: {}", name, path_buf.display(), err));
+            sess.emit_err(errors::WriteNew { name, path: path_buf, err });
         }
     }
 }
diff --git a/compiler/rustc_incremental/src/persist/fs.rs b/compiler/rustc_incremental/src/persist/fs.rs
index 1fd2b9b0d..73d7e3bec 100644
--- a/compiler/rustc_incremental/src/persist/fs.rs
+++ b/compiler/rustc_incremental/src/persist/fs.rs
@@ -103,6 +103,7 @@
 //! unsupported file system and emit a warning in that case. This is not yet
 //! implemented.
 
+use crate::errors;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::svh::Svh;
 use rustc_data_structures::{base_n, flock};
@@ -225,12 +226,7 @@ pub fn prepare_session_directory(
     let crate_dir = match crate_dir.canonicalize() {
         Ok(v) => v,
         Err(err) => {
-            let reported = sess.err(&format!(
-                "incremental compilation: error canonicalizing path `{}`: {}",
-                crate_dir.display(),
-                err
-            ));
-            return Err(reported);
+            return Err(sess.emit_err(errors::CanonicalizePath { path: crate_dir, err }));
         }
     };
 
@@ -273,14 +269,7 @@ pub fn prepare_session_directory(
             debug!("successfully copied data from: {}", source_directory.display());
 
             if !allows_links {
-                sess.warn(&format!(
-                    "Hard linking files in the incremental \
-                                        compilation cache failed. Copying files \
-                                        instead. Consider moving the cache \
-                                        directory to a file system which supports \
-                                        hard linking in session dir `{}`",
-                    session_dir.display()
-                ));
+                sess.emit_warning(errors::HardLinkFailed { path: &session_dir });
             }
 
             sess.init_incr_comp_session(session_dir, directory_lock, true);
@@ -295,12 +284,7 @@ pub fn prepare_session_directory(
             // Try to remove the session directory we just allocated. We don't
             // know if there's any garbage in it from the failed copy action.
             if let Err(err) = safe_remove_dir_all(&session_dir) {
-                sess.warn(&format!(
-                    "Failed to delete partly initialized \
-                                    session dir `{}`: {}",
-                    session_dir.display(),
-                    err
-                ));
+                sess.emit_warning(errors::DeletePartial { path: &session_dir, err });
             }
 
             delete_session_dir_lock_file(sess, &lock_file_path);
@@ -332,12 +316,7 @@ pub fn finalize_session_directory(sess: &Session, svh: Svh) {
         );
 
         if let Err(err) = safe_remove_dir_all(&*incr_comp_session_dir) {
-            sess.warn(&format!(
-                "Error deleting incremental compilation \
-                                session directory `{}`: {}",
-                incr_comp_session_dir.display(),
-                err
-            ));
+            sess.emit_warning(errors::DeleteFull { path: &incr_comp_session_dir, err });
         }
 
         let lock_file_path = lock_file_path(&*incr_comp_session_dir);
@@ -380,12 +359,7 @@ pub fn finalize_session_directory(sess: &Session, svh: Svh) {
         }
         Err(e) => {
             // Warn about the error. However, no need to abort compilation now.
-            sess.warn(&format!(
-                "Error finalizing incremental compilation \
-                               session directory `{}`: {}",
-                incr_comp_session_dir.display(),
-                e
-            ));
+            sess.emit_warning(errors::Finalize { path: &incr_comp_session_dir, err: e });
 
             debug!("finalize_session_directory() - error, marking as invalid");
             // Drop the file lock, so we can garage collect
@@ -488,16 +462,7 @@ fn create_dir(sess: &Session, path: &Path, dir_tag: &str) -> Result<(), ErrorGua
             debug!("{} directory created successfully", dir_tag);
             Ok(())
         }
-        Err(err) => {
-            let reported = sess.err(&format!(
-                "Could not create incremental compilation {} \
-                               directory `{}`: {}",
-                dir_tag,
-                path.display(),
-                err
-            ));
-            Err(reported)
-        }
+        Err(err) => Err(sess.emit_err(errors::CreateIncrCompDir { tag: dir_tag, path, err })),
     }
 }
 
@@ -518,46 +483,20 @@ fn lock_directory(
         // the lock should be exclusive
         Ok(lock) => Ok((lock, lock_file_path)),
         Err(lock_err) => {
-            let mut err = sess.struct_err(&format!(
-                "incremental compilation: could not create \
-                 session directory lock file: {}",
-                lock_err
-            ));
-            if flock::Lock::error_unsupported(&lock_err) {
-                err.note(&format!(
-                    "the filesystem for the incremental path at {} \
-                     does not appear to support locking, consider changing the \
-                     incremental path to a filesystem that supports locking \
-                     or disable incremental compilation",
-                    session_dir.display()
-                ));
-                if std::env::var_os("CARGO").is_some() {
-                    err.help(
-                        "incremental compilation can be disabled by setting the \
-                         environment variable CARGO_INCREMENTAL=0 (see \
-                         https://doc.rust-lang.org/cargo/reference/profiles.html#incremental)",
-                    );
-                    err.help(
-                        "the entire build directory can be changed to a different \
-                        filesystem by setting the environment variable CARGO_TARGET_DIR \
-                        to a different path (see \
-                        https://doc.rust-lang.org/cargo/reference/config.html#buildtarget-dir)",
-                    );
-                }
-            }
-            Err(err.emit())
+            let is_unsupported_lock = flock::Lock::error_unsupported(&lock_err).then_some(());
+            Err(sess.emit_err(errors::CreateLock {
+                lock_err,
+                session_dir,
+                is_unsupported_lock,
+                is_cargo: std::env::var_os("CARGO").map(|_| ()),
+            }))
         }
     }
 }
 
 fn delete_session_dir_lock_file(sess: &Session, lock_file_path: &Path) {
     if let Err(err) = safe_remove_file(&lock_file_path) {
-        sess.warn(&format!(
-            "Error deleting lock file for incremental \
-                            compilation session directory `{}`: {}",
-            lock_file_path.display(),
-            err
-        ));
+        sess.emit_warning(errors::DeleteLock { path: lock_file_path, err });
     }
 }
 
@@ -774,12 +713,7 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> {
         if !lock_file_to_session_dir.values().any(|dir| *dir == directory_name) {
             let path = crate_directory.join(directory_name);
             if let Err(err) = safe_remove_dir_all(&path) {
-                sess.warn(&format!(
-                    "Failed to garbage collect invalid incremental \
-                                    compilation session directory `{}`: {}",
-                    path.display(),
-                    err
-                ));
+                sess.emit_warning(errors::InvalidGcFailed { path: &path, err });
             }
         }
     }
@@ -885,12 +819,7 @@ pub fn garbage_collect_session_directories(sess: &Session) -> io::Result<()> {
         debug!("garbage_collect_session_directories() - deleting `{}`", path.display());
 
         if let Err(err) = safe_remove_dir_all(&path) {
-            sess.warn(&format!(
-                "Failed to garbage collect finalized incremental \
-                                compilation session directory `{}`: {}",
-                path.display(),
-                err
-            ));
+            sess.emit_warning(errors::FinalizedGcFailed { path: &path, err });
         } else {
             delete_session_dir_lock_file(sess, &lock_file_path(&path));
         }
@@ -907,11 +836,7 @@ fn delete_old(sess: &Session, path: &Path) {
     debug!("garbage_collect_session_directories() - deleting `{}`", path.display());
 
     if let Err(err) = safe_remove_dir_all(&path) {
-        sess.warn(&format!(
-            "Failed to garbage collect incremental compilation session directory `{}`: {}",
-            path.display(),
-            err
-        ));
+        sess.emit_warning(errors::SessionGcFailed { path: &path, err });
     } else {
         delete_session_dir_lock_file(sess, &lock_file_path(&path));
     }
diff --git a/compiler/rustc_incremental/src/persist/load.rs b/compiler/rustc_incremental/src/persist/load.rs
index 1c5fd9169..d5097065d 100644
--- a/compiler/rustc_incremental/src/persist/load.rs
+++ b/compiler/rustc_incremental/src/persist/load.rs
@@ -1,5 +1,6 @@
 //! Code to save/load the dep-graph from files.
 
+use crate::errors;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::memmap::Mmap;
 use rustc_middle::dep_graph::{SerializedDepGraph, WorkProduct, WorkProductId};
@@ -8,7 +9,7 @@ use rustc_serialize::opaque::MemDecoder;
 use rustc_serialize::Decodable;
 use rustc_session::config::IncrementalStateAssertion;
 use rustc_session::Session;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
 use super::data::*;
 use super::file_format;
@@ -27,11 +28,10 @@ pub enum LoadResult<T> {
     },
     /// The file either didn't exist or was produced by an incompatible compiler version.
     DataOutOfDate,
-    /// An error occurred.
-    Error {
-        #[allow(missing_docs)]
-        message: String,
-    },
+    /// Loading the dep graph failed.
+    LoadDepGraph(PathBuf, std::io::Error),
+    /// Decoding loaded incremental cache failed.
+    DecodeIncrCache(Box<dyn std::any::Any + Send>),
 }
 
 impl<T: Default> LoadResult<T> {
@@ -40,36 +40,31 @@ impl<T: Default> LoadResult<T> {
         // Check for errors when using `-Zassert-incremental-state`
         match (sess.opts.assert_incr_state, &self) {
             (Some(IncrementalStateAssertion::NotLoaded), LoadResult::Ok { .. }) => {
-                sess.fatal(
-                    "We asserted that the incremental cache should not be loaded, \
-                         but it was loaded.",
-                );
+                sess.emit_fatal(errors::AssertNotLoaded);
             }
             (
                 Some(IncrementalStateAssertion::Loaded),
-                LoadResult::Error { .. } | LoadResult::DataOutOfDate,
+                LoadResult::LoadDepGraph(..)
+                | LoadResult::DecodeIncrCache(..)
+                | LoadResult::DataOutOfDate,
             ) => {
-                sess.fatal(
-                    "We asserted that an existing incremental cache directory should \
-                         be successfully loaded, but it was not.",
-                );
+                sess.emit_fatal(errors::AssertLoaded);
             }
             _ => {}
         };
 
         match self {
-            LoadResult::Error { message } => {
-                sess.warn(&message);
+            LoadResult::LoadDepGraph(path, err) => {
+                sess.emit_warning(errors::LoadDepGraph { path, err });
+                Default::default()
+            }
+            LoadResult::DecodeIncrCache(err) => {
+                sess.emit_warning(errors::DecodeIncrCache { err: format!("{err:?}") });
                 Default::default()
             }
             LoadResult::DataOutOfDate => {
                 if let Err(err) = delete_all_session_dir_contents(sess) {
-                    sess.err(&format!(
-                        "Failed to delete invalidated or incompatible \
-                         incremental compilation session directory contents `{}`: {}.",
-                        dep_graph_path(sess).display(),
-                        err
-                    ));
+                    sess.emit_err(errors::DeleteIncompatible { path: dep_graph_path(sess), err });
                 }
                 Default::default()
             }
@@ -90,9 +85,7 @@ fn load_data(
             // compiler version. Neither is an error.
             LoadResult::DataOutOfDate
         }
-        Err(err) => LoadResult::Error {
-            message: format!("could not load dep-graph from `{}`: {}", path.display(), err),
-        },
+        Err(err) => LoadResult::LoadDepGraph(path.to_path_buf(), err),
     }
 }
 
@@ -114,9 +107,9 @@ impl<T> MaybeAsync<LoadResult<T>> {
     pub fn open(self) -> LoadResult<T> {
         match self {
             MaybeAsync::Sync(result) => result,
-            MaybeAsync::Async(handle) => handle.join().unwrap_or_else(|e| LoadResult::Error {
-                message: format!("could not decode incremental cache: {:?}", e),
-            }),
+            MaybeAsync::Async(handle) => {
+                handle.join().unwrap_or_else(|e| LoadResult::DecodeIncrCache(e))
+            }
         }
     }
 }
@@ -185,7 +178,8 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture {
 
         match load_data(report_incremental_info, &path, nightly_build) {
             LoadResult::DataOutOfDate => LoadResult::DataOutOfDate,
-            LoadResult::Error { message } => LoadResult::Error { message },
+            LoadResult::LoadDepGraph(path, err) => LoadResult::LoadDepGraph(path, err),
+            LoadResult::DecodeIncrCache(err) => LoadResult::DecodeIncrCache(err),
             LoadResult::Ok { data: (bytes, start_pos) } => {
                 let mut decoder = MemDecoder::new(&bytes, start_pos);
                 let prev_commandline_args_hash = u64::decode(&mut decoder);
diff --git a/compiler/rustc_incremental/src/persist/save.rs b/compiler/rustc_incremental/src/persist/save.rs
index 6e9dcdd98..27be56eac 100644
--- a/compiler/rustc_incremental/src/persist/save.rs
+++ b/compiler/rustc_incremental/src/persist/save.rs
@@ -1,3 +1,4 @@
+use crate::errors;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::join;
 use rustc_middle::dep_graph::{DepGraph, SerializedDepGraph, WorkProduct, WorkProductId};
@@ -59,19 +60,14 @@ pub fn save_dep_graph(tcx: TyCtxt<'_>) {
             move || {
                 sess.time("incr_comp_persist_dep_graph", || {
                     if let Err(err) = tcx.dep_graph.encode(&tcx.sess.prof) {
-                        sess.err(&format!(
-                            "failed to write dependency graph to `{}`: {}",
-                            staging_dep_graph_path.display(),
-                            err
-                        ));
+                        sess.emit_err(errors::WriteDepGraph { path: &staging_dep_graph_path, err });
                     }
                     if let Err(err) = fs::rename(&staging_dep_graph_path, &dep_graph_path) {
-                        sess.err(&format!(
-                            "failed to move dependency graph from `{}` to `{}`: {}",
-                            staging_dep_graph_path.display(),
-                            dep_graph_path.display(),
-                            err
-                        ));
+                        sess.emit_err(errors::MoveDepGraph {
+                            from: &staging_dep_graph_path,
+                            to: &dep_graph_path,
+                            err,
+                        });
                     }
                 });
             },
@@ -163,11 +159,7 @@ pub fn build_dep_graph(
     let mut encoder = match FileEncoder::new(&path_buf) {
         Ok(encoder) => encoder,
         Err(err) => {
-            sess.err(&format!(
-                "failed to create dependency graph at `{}`: {}",
-                path_buf.display(),
-                err
-            ));
+            sess.emit_err(errors::CreateDepGraph { path: &path_buf, err });
             return None;
         }
     };
diff --git a/compiler/rustc_incremental/src/persist/work_product.rs b/compiler/rustc_incremental/src/persist/work_product.rs
index 2f1853c44..dc98fbeb0 100644
--- a/compiler/rustc_incremental/src/persist/work_product.rs
+++ b/compiler/rustc_incremental/src/persist/work_product.rs
@@ -2,6 +2,7 @@
 //!
 //! [work products]: WorkProduct
 
+use crate::errors;
 use crate::persist::fs::*;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_fs_util::link_or_copy;
@@ -28,12 +29,11 @@ pub fn copy_cgu_workproduct_to_incr_comp_cache_dir(
                 let _ = saved_files.insert(ext.to_string(), file_name);
             }
             Err(err) => {
-                sess.warn(&format!(
-                    "error copying object file `{}` to incremental directory as `{}`: {}",
-                    path.display(),
-                    path_in_incr_dir.display(),
-                    err
-                ));
+                sess.emit_warning(errors::CopyWorkProductToCache {
+                    from: &path,
+                    to: &path_in_incr_dir,
+                    err,
+                });
             }
         }
     }
@@ -49,11 +49,7 @@ pub fn delete_workproduct_files(sess: &Session, work_product: &WorkProduct) {
     for (_, path) in &work_product.saved_files {
         let path = in_incr_comp_dir_sess(sess, path);
         if let Err(err) = std_fs::remove_file(&path) {
-            sess.warn(&format!(
-                "file-system error deleting outdated file `{}`: {}",
-                path.display(),
-                err
-            ));
+            sess.emit_warning(errors::DeleteWorkProduct { path: &path, err });
         }
     }
 }
diff --git a/compiler/rustc_index/src/bit_set.rs b/compiler/rustc_index/src/bit_set.rs
index 15179392c..cbf169afb 100644
--- a/compiler/rustc_index/src/bit_set.rs
+++ b/compiler/rustc_index/src/bit_set.rs
@@ -1870,7 +1870,7 @@ impl<R: Idx, C: Idx> SparseBitMatrix<R, C> {
         }
     }
 
-    /// Subtracts `set from `row`. `set` can be either `BitSet` or
+    /// Subtracts `set` from `row`. `set` can be either `BitSet` or
     /// `HybridBitSet`. Has no effect if `row` does not exist.
     ///
     /// Returns true if the row was changed.
diff --git a/compiler/rustc_infer/Cargo.toml b/compiler/rustc_infer/Cargo.toml
index aced787d6..02ac83a5e 100644
--- a/compiler/rustc_infer/Cargo.toml
+++ b/compiler/rustc_infer/Cargo.toml
@@ -15,7 +15,6 @@ rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_serialize = { path = "../rustc_serialize" }
-rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
diff --git a/compiler/rustc_error_messages/locales/en-US/infer.ftl b/compiler/rustc_infer/locales/en-US.ftl
index cc38d71b4..15780898d 100644
--- a/compiler/rustc_error_messages/locales/en-US/infer.ftl
+++ b/compiler/rustc_infer/locales/en-US.ftl
@@ -79,7 +79,7 @@ infer_subtype = ...so that the {$requirement ->
     [if_else_different] `if` and `else` have incompatible types
     [no_else] `if` missing an `else` returns `()`
     [fn_main_correct_type] `main` function has the correct type
-    [fn_start_correct_type] #[start]` function has the correct type
+    [fn_start_correct_type] `#[start]` function has the correct type
     [intristic_correct_type] intrinsic has the correct type
     [method_correct_type] method receiver has the correct type
     *[other] types are compatible
@@ -92,7 +92,7 @@ infer_subtype_2 = ...so that {$requirement ->
     [if_else_different] `if` and `else` have incompatible types
     [no_else] `if` missing an `else` returns `()`
     [fn_main_correct_type] `main` function has the correct type
-    [fn_start_correct_type] #[start]` function has the correct type
+    [fn_start_correct_type] `#[start]` function has the correct type
     [intristic_correct_type] intrinsic has the correct type
     [method_correct_type] method receiver has the correct type
     *[other] types are compatible
@@ -140,6 +140,18 @@ infer_lifetime_param_suggestion_elided = each elided lifetime in input position
 
 infer_region_explanation = {$pref_kind ->
     *[should_not_happen] [{$pref_kind}]
+    [ref_valid_for] ...the reference is valid for
+    [content_valid_for] ...but the borrowed content is only valid for
+    [type_obj_valid_for] object type is valid for
+    [source_pointer_valid_for] source pointer is only valid for
+    [type_satisfy] type must satisfy
+    [type_outlive] type must outlive
+    [lf_param_instantiated_with] lifetime parameter instantiated with
+    [lf_param_must_outlive] but lifetime parameter must outlive
+    [lf_instantiated_with] lifetime instantiated with
+    [lf_must_outlive] but lifetime must outlive
+    [pointer_valid_for] the pointer is valid for
+    [data_valid_for] but the referenced data is only valid for
     [empty] {""}
 }{$pref_kind ->
     [empty] {""}
@@ -147,10 +159,7 @@ infer_region_explanation = {$pref_kind ->
 }{$desc_kind ->
     *[should_not_happen] [{$desc_kind}]
     [restatic] the static lifetime
-    [reempty] the empty lifetime
-    [reemptyuni] the empty lifetime in universe {$desc_arg}
     [revar] lifetime {$desc_arg}
-
     [as_defined] the lifetime `{$desc_arg}` as defined here
     [as_defined_anon] the anonymous lifetime as defined here
     [defined_here] the anonymous lifetime defined here
@@ -160,8 +169,16 @@ infer_region_explanation = {$pref_kind ->
     *[should_not_happen] [{$suff_kind}]
     [empty]{""}
     [continues] ...
+    [req_by_binding] {" "}as required by this binding
 }
 
+infer_outlives_content = lifetime of reference outlives lifetime of borrowed content...
+infer_outlives_bound = lifetime of the source pointer does not outlive lifetime bound of the object type
+infer_fullfill_req_lifetime = the type `{$ty}` does not fulfill the required lifetime
+infer_lf_bound_not_satisfied = lifetime bound not satisfied
+infer_borrowed_too_long = a value of type `{$ty}` is borrowed for too long
+infer_ref_longer_than_data = in type `{$ty}`, reference has a longer lifetime than the data it references
+
 infer_mismatched_static_lifetime = incompatible lifetime on type
 infer_does_not_outlive_static_from_impl = ...does not necessarily outlive the static lifetime introduced by the compatible `impl`
 infer_implicit_static_lifetime_note = this has an implicit `'static` lifetime requirement
@@ -260,7 +277,7 @@ infer_tid_consider_borrowing = consider borrowing this type parameter in the tra
 infer_tid_param_help = the lifetime requirements from the `impl` do not correspond to the requirements in the `trait`
 
 infer_dtcs_has_lifetime_req_label = this has an implicit `'static` lifetime requirement
-infer_dtcs_introduces_requirement = calling this method introduces the `impl`'s 'static` requirement
+infer_dtcs_introduces_requirement = calling this method introduces the `impl`'s `'static` requirement
 infer_dtcs_has_req_note = the used `impl` has a `'static` requirement
 infer_dtcs_suggestion = consider relaxing the implicit `'static` requirement
 
@@ -268,45 +285,66 @@ infer_but_calling_introduces = {$has_param_name ->
     [true] `{$param_name}`
     *[false] `fn` parameter
 } has {$lifetime_kind ->
-    [named] lifetime `{$lifetime}`
-    *[anon] an anonymous lifetime `'_`
-} but calling `{assoc_item}` introduces an implicit `'static` lifetime requirement
+    [true] lifetime `{$lifetime}`
+    *[false] an anonymous lifetime `'_`
+} but calling `{$assoc_item}` introduces an implicit `'static` lifetime requirement
     .label1 = {$has_lifetime ->
-        [named] lifetime `{$lifetime}`
-        *[anon] an anonymous lifetime `'_`
+        [true] lifetime `{$lifetime}`
+        *[false] an anonymous lifetime `'_`
     }
     .label2 = ...is used and required to live as long as `'static` here because of an implicit lifetime bound on the {$has_impl_path ->
-        [named] `impl` of `{$impl_path}`
-        *[anon] inherent `impl`
+        [true] `impl` of `{$impl_path}`
+        *[false] inherent `impl`
     }
 
 infer_but_needs_to_satisfy = {$has_param_name ->
     [true] `{$param_name}`
     *[false] `fn` parameter
 } has {$has_lifetime ->
-    [named] lifetime `{$lifetime}`
-    *[anon] an anonymous lifetime `'_`
+    [true] lifetime `{$lifetime}`
+    *[false] an anonymous lifetime `'_`
 } but it needs to satisfy a `'static` lifetime requirement
     .influencer = this data with {$has_lifetime ->
-        [named] lifetime `{$lifetime}`
-        *[anon] an anonymous lifetime `'_`
+        [true] lifetime `{$lifetime}`
+        *[false] an anonymous lifetime `'_`
     }...
     .require = {$spans_empty ->
         *[true] ...is used and required to live as long as `'static` here
         [false] ...and is required to live as long as `'static` here
     }
     .used_here = ...is used here...
-    .introduced_by_bound = 'static` lifetime requirement introduced by this bound
+    .introduced_by_bound = `'static` lifetime requirement introduced by this bound
 
 infer_more_targeted = {$has_param_name ->
     [true] `{$param_name}`
     *[false] `fn` parameter
 } has {$has_lifetime ->
-    [named] lifetime `{$lifetime}`
-    *[anon] an anonymous lifetime `'_`
+    [true] lifetime `{$lifetime}`
+    *[false] an anonymous lifetime `'_`
 } but calling `{$ident}` introduces an implicit `'static` lifetime requirement
 
 infer_ril_introduced_here = `'static` requirement introduced here
 infer_ril_introduced_by = requirement introduced by this return type
 infer_ril_because_of = because of this returned expression
 infer_ril_static_introduced_by = "`'static` lifetime requirement introduced by the return type
+
+infer_where_remove = remove the `where` clause
+infer_where_copy_predicates = copy the `where` clause predicates from the trait
+
+infer_srs_remove_and_box = consider removing this semicolon and boxing the expressions
+infer_srs_remove = consider removing this semicolon
+infer_srs_add = consider returning the local binding `{$ident}`
+infer_srs_add_one = consider returning one of these bindings
+
+infer_await_both_futures = consider `await`ing on both `Future`s
+infer_await_future = consider `await`ing on the `Future`
+infer_await_note = calling an async function returns a future
+
+infer_prlf_defined_with_sub = the lifetime `{$sub_symbol}` defined here...
+infer_prlf_defined_without_sub = the lifetime defined here...
+infer_prlf_must_oultive_with_sup = ...must outlive the lifetime `{$sup_symbol}` defined here
+infer_prlf_must_oultive_without_sup = ...must outlive the lifetime defined here
+infer_prlf_known_limitation = this is a known limitation that will be removed in the future (see issue #100013 <https://github.com/rust-lang/rust/issues/100013> for more information)
+
+infer_opaque_captures_lifetime = hidden type for `{$opaque_ty}` captures lifetime that does not appear in bounds
+    .label = opaque type defined here
diff --git a/compiler/rustc_infer/src/errors/mod.rs b/compiler/rustc_infer/src/errors/mod.rs
index 033a1842e..6bbd3fd3e 100644
--- a/compiler/rustc_infer/src/errors/mod.rs
+++ b/compiler/rustc_infer/src/errors/mod.rs
@@ -1,6 +1,6 @@
 use hir::GenericParamKind;
 use rustc_errors::{
-    fluent, AddToDiagnostic, Applicability, Diagnostic, DiagnosticMessage, DiagnosticStyledString,
+    AddToDiagnostic, Applicability, Diagnostic, DiagnosticMessage, DiagnosticStyledString,
     IntoDiagnosticArg, MultiSpan, SubdiagnosticMessage,
 };
 use rustc_hir as hir;
@@ -12,9 +12,10 @@ use rustc_span::symbol::kw;
 use rustc_span::Symbol;
 use rustc_span::{symbol::Ident, BytePos, Span};
 
-use crate::infer::error_reporting::nice_region_error::placeholder_error::Highlighted;
+use crate::fluent_generated as fluent;
 use crate::infer::error_reporting::{
     need_type_info::{GeneratorKindAsDiagArg, UnderspecifiedArgKind},
+    nice_region_error::placeholder_error::Highlighted,
     ObligationCauseAsDiagArg,
 };
 
@@ -26,9 +27,9 @@ pub struct OpaqueHiddenTypeDiag {
     #[primary_span]
     #[label]
     pub span: Span,
-    #[note(opaque_type)]
+    #[note(infer_opaque_type)]
     pub opaque_type: Span,
-    #[note(hidden_type)]
+    #[note(infer_hidden_type)]
     pub hidden_type: Span,
 }
 
@@ -768,11 +769,11 @@ impl<'tcx> ActualImplExplNotes<'tcx> {
 pub struct TraitPlaceholderMismatch<'tcx> {
     #[primary_span]
     pub span: Span,
-    #[label(label_satisfy)]
+    #[label(infer_label_satisfy)]
     pub satisfy_span: Option<Span>,
-    #[label(label_where)]
+    #[label(infer_label_where)]
     pub where_span: Option<Span>,
-    #[label(label_dup)]
+    #[label(infer_label_dup)]
     pub dup_span: Option<Span>,
     pub def_id: String,
     pub trait_def_id: String,
@@ -808,11 +809,11 @@ pub struct RelationshipHelp;
 #[diag(infer_trait_impl_diff)]
 pub struct TraitImplDiff {
     #[primary_span]
-    #[label(found)]
+    #[label(infer_found)]
     pub sp: Span,
-    #[label(expected)]
+    #[label(infer_expected)]
     pub trait_sp: Span,
-    #[note(expected_found)]
+    #[note(infer_expected_found)]
     pub note: (),
     #[subdiagnostic]
     pub param_help: ConsiderBorrowingParamHelp,
@@ -852,10 +853,10 @@ impl AddToDiagnostic for DynTraitConstraintSuggestion {
 #[derive(Diagnostic)]
 #[diag(infer_but_calling_introduces, code = "E0772")]
 pub struct ButCallingIntroduces {
-    #[label(label1)]
+    #[label(infer_label1)]
     pub param_ty_span: Span,
     #[primary_span]
-    #[label(label2)]
+    #[label(infer_label2)]
     pub cause_span: Span,
 
     pub has_param_name: bool,
@@ -913,21 +914,246 @@ impl AddToDiagnostic for MoreTargeted {
 pub struct ButNeedsToSatisfy {
     #[primary_span]
     pub sp: Span,
-    #[label(influencer)]
+    #[label(infer_influencer)]
     pub influencer_point: Span,
-    #[label(used_here)]
+    #[label(infer_used_here)]
     pub spans: Vec<Span>,
-    #[label(require)]
+    #[label(infer_require)]
     pub require_span_as_label: Option<Span>,
-    #[note(require)]
+    #[note(infer_require)]
     pub require_span_as_note: Option<Span>,
-    #[note(introduced_by_bound)]
+    #[note(infer_introduced_by_bound)]
     pub bound: Option<Span>,
 
     #[subdiagnostic]
     pub req_introduces_loc: Option<ReqIntroducedLocations>,
 
+    pub has_param_name: bool,
+    pub param_name: String,
     pub spans_empty: bool,
     pub has_lifetime: bool,
     pub lifetime: String,
 }
+
+#[derive(Diagnostic)]
+#[diag(infer_outlives_content, code = "E0312")]
+pub struct OutlivesContent<'a> {
+    #[primary_span]
+    pub span: Span,
+    #[subdiagnostic]
+    pub notes: Vec<note_and_explain::RegionExplanation<'a>>,
+}
+
+#[derive(Diagnostic)]
+#[diag(infer_outlives_bound, code = "E0476")]
+pub struct OutlivesBound<'a> {
+    #[primary_span]
+    pub span: Span,
+    #[subdiagnostic]
+    pub notes: Vec<note_and_explain::RegionExplanation<'a>>,
+}
+
+#[derive(Diagnostic)]
+#[diag(infer_fullfill_req_lifetime, code = "E0477")]
+pub struct FullfillReqLifetime<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub ty: Ty<'a>,
+    #[subdiagnostic]
+    pub note: Option<note_and_explain::RegionExplanation<'a>>,
+}
+
+#[derive(Diagnostic)]
+#[diag(infer_lf_bound_not_satisfied, code = "E0478")]
+pub struct LfBoundNotSatisfied<'a> {
+    #[primary_span]
+    pub span: Span,
+    #[subdiagnostic]
+    pub notes: Vec<note_and_explain::RegionExplanation<'a>>,
+}
+
+#[derive(Diagnostic)]
+#[diag(infer_ref_longer_than_data, code = "E0491")]
+pub struct RefLongerThanData<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub ty: Ty<'a>,
+    #[subdiagnostic]
+    pub notes: Vec<note_and_explain::RegionExplanation<'a>>,
+}
+
+#[derive(Subdiagnostic)]
+pub enum WhereClauseSuggestions {
+    #[suggestion(
+        infer_where_remove,
+        code = "",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
+    Remove {
+        #[primary_span]
+        span: Span,
+    },
+    #[suggestion(
+        infer_where_copy_predicates,
+        code = "{space}where {trait_predicates}",
+        applicability = "machine-applicable",
+        style = "verbose"
+    )]
+    CopyPredicates {
+        #[primary_span]
+        span: Span,
+        space: &'static str,
+        trait_predicates: String,
+    },
+}
+
+#[derive(Subdiagnostic)]
+pub enum SuggestRemoveSemiOrReturnBinding {
+    #[multipart_suggestion(infer_srs_remove_and_box, applicability = "machine-applicable")]
+    RemoveAndBox {
+        #[suggestion_part(code = "Box::new(")]
+        first_lo: Span,
+        #[suggestion_part(code = ")")]
+        first_hi: Span,
+        #[suggestion_part(code = "Box::new(")]
+        second_lo: Span,
+        #[suggestion_part(code = ")")]
+        second_hi: Span,
+        #[suggestion_part(code = "")]
+        sp: Span,
+    },
+    #[suggestion(
+        infer_srs_remove,
+        style = "short",
+        code = "",
+        applicability = "machine-applicable"
+    )]
+    Remove {
+        #[primary_span]
+        sp: Span,
+    },
+    #[suggestion(
+        infer_srs_add,
+        style = "verbose",
+        code = "{code}",
+        applicability = "maybe-incorrect"
+    )]
+    Add {
+        #[primary_span]
+        sp: Span,
+        code: String,
+        ident: Ident,
+    },
+    #[note(infer_srs_add_one)]
+    AddOne {
+        #[primary_span]
+        spans: MultiSpan,
+    },
+}
+
+#[derive(Subdiagnostic)]
+pub enum ConsiderAddingAwait {
+    #[help(infer_await_both_futures)]
+    BothFuturesHelp,
+    #[multipart_suggestion(infer_await_both_futures, applicability = "maybe-incorrect")]
+    BothFuturesSugg {
+        #[suggestion_part(code = ".await")]
+        first: Span,
+        #[suggestion_part(code = ".await")]
+        second: Span,
+    },
+    #[suggestion(
+        infer_await_future,
+        code = ".await",
+        style = "verbose",
+        applicability = "maybe-incorrect"
+    )]
+    FutureSugg {
+        #[primary_span]
+        span: Span,
+    },
+    #[note(infer_await_note)]
+    FutureSuggNote {
+        #[primary_span]
+        span: Span,
+    },
+    #[multipart_suggestion(
+        infer_await_future,
+        style = "verbose",
+        applicability = "maybe-incorrect"
+    )]
+    FutureSuggMultiple {
+        #[suggestion_part(code = ".await")]
+        spans: Vec<Span>,
+    },
+}
+
+#[derive(Diagnostic)]
+pub enum PlaceholderRelationLfNotSatisfied {
+    #[diag(infer_lf_bound_not_satisfied)]
+    HasBoth {
+        #[primary_span]
+        span: Span,
+        #[note(infer_prlf_defined_with_sub)]
+        sub_span: Span,
+        #[note(infer_prlf_must_oultive_with_sup)]
+        sup_span: Span,
+        sub_symbol: Symbol,
+        sup_symbol: Symbol,
+        #[note(infer_prlf_known_limitation)]
+        note: (),
+    },
+    #[diag(infer_lf_bound_not_satisfied)]
+    HasSub {
+        #[primary_span]
+        span: Span,
+        #[note(infer_prlf_defined_with_sub)]
+        sub_span: Span,
+        #[note(infer_prlf_must_oultive_without_sup)]
+        sup_span: Span,
+        sub_symbol: Symbol,
+        #[note(infer_prlf_known_limitation)]
+        note: (),
+    },
+    #[diag(infer_lf_bound_not_satisfied)]
+    HasSup {
+        #[primary_span]
+        span: Span,
+        #[note(infer_prlf_defined_without_sub)]
+        sub_span: Span,
+        #[note(infer_prlf_must_oultive_with_sup)]
+        sup_span: Span,
+        sup_symbol: Symbol,
+        #[note(infer_prlf_known_limitation)]
+        note: (),
+    },
+    #[diag(infer_lf_bound_not_satisfied)]
+    HasNone {
+        #[primary_span]
+        span: Span,
+        #[note(infer_prlf_defined_without_sub)]
+        sub_span: Span,
+        #[note(infer_prlf_must_oultive_without_sup)]
+        sup_span: Span,
+        #[note(infer_prlf_known_limitation)]
+        note: (),
+    },
+    #[diag(infer_lf_bound_not_satisfied)]
+    OnlyPrimarySpan {
+        #[primary_span]
+        span: Span,
+        #[note(infer_prlf_known_limitation)]
+        note: (),
+    },
+}
+
+#[derive(Diagnostic)]
+#[diag(infer_opaque_captures_lifetime, code = "E0700")]
+pub struct OpaqueCapturesLifetime<'tcx> {
+    #[primary_span]
+    pub span: Span,
+    #[label]
+    pub opaque_ty_span: Span,
+    pub opaque_ty: Ty<'tcx>,
+}
diff --git a/compiler/rustc_infer/src/errors/note_and_explain.rs b/compiler/rustc_infer/src/errors/note_and_explain.rs
index 7aaa5ce2f..ef543b1fb 100644
--- a/compiler/rustc_infer/src/errors/note_and_explain.rs
+++ b/compiler/rustc_infer/src/errors/note_and_explain.rs
@@ -1,7 +1,6 @@
+use crate::fluent_generated as fluent;
 use crate::infer::error_reporting::nice_region_error::find_anon_type;
-use rustc_errors::{
-    self, fluent, AddToDiagnostic, Diagnostic, IntoDiagnosticArg, SubdiagnosticMessage,
-};
+use rustc_errors::{self, AddToDiagnostic, Diagnostic, IntoDiagnosticArg, SubdiagnosticMessage};
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_span::{symbol::kw, Span};
 
@@ -31,6 +30,8 @@ impl<'a> DescriptionCtx<'a> {
 
             ty::RePlaceholder(_) => return None,
 
+            ty::ReError(_) => return None,
+
             // FIXME(#13998) RePlaceholder should probably print like
             // ReFree rather than dumping Debug output on the user.
             //
@@ -119,16 +120,42 @@ impl<'a> DescriptionCtx<'a> {
 
 pub enum PrefixKind {
     Empty,
+    RefValidFor,
+    ContentValidFor,
+    TypeObjValidFor,
+    SourcePointerValidFor,
+    TypeSatisfy,
+    TypeOutlive,
+    LfParamInstantiatedWith,
+    LfParamMustOutlive,
+    LfInstantiatedWith,
+    LfMustOutlive,
+    PointerValidFor,
+    DataValidFor,
 }
 
 pub enum SuffixKind {
+    Empty,
     Continues,
+    ReqByBinding,
 }
 
 impl IntoDiagnosticArg for PrefixKind {
     fn into_diagnostic_arg(self) -> rustc_errors::DiagnosticArgValue<'static> {
         let kind = match self {
             Self::Empty => "empty",
+            Self::RefValidFor => "ref_valid_for",
+            Self::ContentValidFor => "content_valid_for",
+            Self::TypeObjValidFor => "type_obj_valid_for",
+            Self::SourcePointerValidFor => "source_pointer_valid_for",
+            Self::TypeSatisfy => "type_satisfy",
+            Self::TypeOutlive => "type_outlive",
+            Self::LfParamInstantiatedWith => "lf_param_instantiated_with",
+            Self::LfParamMustOutlive => "lf_param_must_outlive",
+            Self::LfInstantiatedWith => "lf_instantiated_with",
+            Self::LfMustOutlive => "lf_must_outlive",
+            Self::PointerValidFor => "pointer_valid_for",
+            Self::DataValidFor => "data_valid_for",
         }
         .into();
         rustc_errors::DiagnosticArgValue::Str(kind)
@@ -138,7 +165,9 @@ impl IntoDiagnosticArg for PrefixKind {
 impl IntoDiagnosticArg for SuffixKind {
     fn into_diagnostic_arg(self) -> rustc_errors::DiagnosticArgValue<'static> {
         let kind = match self {
+            Self::Empty => "empty",
             Self::Continues => "continues",
+            Self::ReqByBinding => "req_by_binding",
         }
         .into();
         rustc_errors::DiagnosticArgValue::Str(kind)
@@ -164,17 +193,19 @@ impl RegionExplanation<'_> {
 }
 
 impl AddToDiagnostic for RegionExplanation<'_> {
-    fn add_to_diagnostic_with<F>(self, diag: &mut Diagnostic, _: F)
+    fn add_to_diagnostic_with<F>(self, diag: &mut Diagnostic, f: F)
     where
         F: Fn(&mut Diagnostic, SubdiagnosticMessage) -> SubdiagnosticMessage,
     {
-        if let Some(span) = self.desc.span {
-            diag.span_note(span, fluent::infer_region_explanation);
-        } else {
-            diag.note(fluent::infer_region_explanation);
-        }
-        self.desc.add_to(diag);
         diag.set_arg("pref_kind", self.prefix);
         diag.set_arg("suff_kind", self.suffix);
+        let desc_span = self.desc.span;
+        self.desc.add_to(diag);
+        let msg = f(diag, fluent::infer_region_explanation.into());
+        if let Some(span) = desc_span {
+            diag.span_note(span, msg);
+        } else {
+            diag.note(msg);
+        }
     }
 }
diff --git a/compiler/rustc_infer/src/infer/at.rs b/compiler/rustc_infer/src/infer/at.rs
index d816a9ed3..7d9bae735 100644
--- a/compiler/rustc_infer/src/infer/at.rs
+++ b/compiler/rustc_infer/src/infer/at.rs
@@ -55,7 +55,7 @@ impl<'tcx> InferCtxt<'tcx> {
         cause: &'a ObligationCause<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
     ) -> At<'a, 'tcx> {
-        At { infcx: self, cause, param_env, define_opaque_types: true }
+        At { infcx: self, cause, param_env, define_opaque_types: false }
     }
 
     /// Forks the inference context, creating a new inference context with the same inference
@@ -369,6 +369,34 @@ impl<'tcx> ToTrace<'tcx> for Const<'tcx> {
     }
 }
 
+impl<'tcx> ToTrace<'tcx> for ty::GenericArg<'tcx> {
+    fn to_trace(
+        _: TyCtxt<'tcx>,
+        cause: &ObligationCause<'tcx>,
+        a_is_expected: bool,
+        a: Self,
+        b: Self,
+    ) -> TypeTrace<'tcx> {
+        use GenericArgKind::*;
+        TypeTrace {
+            cause: cause.clone(),
+            values: match (a.unpack(), b.unpack()) {
+                (Lifetime(a), Lifetime(b)) => Regions(ExpectedFound::new(a_is_expected, a, b)),
+                (Type(a), Type(b)) => Terms(ExpectedFound::new(a_is_expected, a.into(), b.into())),
+                (Const(a), Const(b)) => {
+                    Terms(ExpectedFound::new(a_is_expected, a.into(), b.into()))
+                }
+
+                (Lifetime(_), Type(_) | Const(_))
+                | (Type(_), Lifetime(_) | Const(_))
+                | (Const(_), Lifetime(_) | Type(_)) => {
+                    bug!("relating different kinds: {a:?} {b:?}")
+                }
+            },
+        }
+    }
+}
+
 impl<'tcx> ToTrace<'tcx> for ty::Term<'tcx> {
     fn to_trace(
         _: TyCtxt<'tcx>,
diff --git a/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs b/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs
index 091635e6c..678c4a0be 100644
--- a/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs
+++ b/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs
@@ -12,7 +12,7 @@ use crate::infer::InferCtxt;
 use rustc_middle::ty::flags::FlagComputation;
 use rustc_middle::ty::fold::{TypeFoldable, TypeFolder, TypeSuperFoldable};
 use rustc_middle::ty::subst::GenericArg;
-use rustc_middle::ty::{self, BoundVar, InferConst, List, Ty, TyCtxt, TypeFlags};
+use rustc_middle::ty::{self, BoundVar, InferConst, List, Ty, TyCtxt, TypeFlags, TypeVisitableExt};
 use std::sync::atomic::Ordering;
 
 use rustc_data_structures::fx::FxHashMap;
@@ -41,7 +41,7 @@ impl<'tcx> InferCtxt<'tcx> {
         query_state: &mut OriginalQueryValues<'tcx>,
     ) -> Canonical<'tcx, V>
     where
-        V: TypeFoldable<'tcx>,
+        V: TypeFoldable<TyCtxt<'tcx>>,
     {
         self.tcx.sess.perf_stats.queries_canonicalized.fetch_add(1, Ordering::Relaxed);
 
@@ -50,7 +50,7 @@ impl<'tcx> InferCtxt<'tcx> {
 
     /// Like [Self::canonicalize_query], but preserves distinct universes. For
     /// example, canonicalizing `&'?0: Trait<'?1>`, where `'?0` is in `U1` and
-    /// `'?1` is in `U3` would be canonicalized to have ?0` in `U1` and `'?1`
+    /// `'?1` is in `U3` would be canonicalized to have `?0` in `U1` and `'?1`
     /// in `U2`.
     ///
     /// This is used for Chalk integration.
@@ -60,7 +60,7 @@ impl<'tcx> InferCtxt<'tcx> {
         query_state: &mut OriginalQueryValues<'tcx>,
     ) -> Canonical<'tcx, V>
     where
-        V: TypeFoldable<'tcx>,
+        V: TypeFoldable<TyCtxt<'tcx>>,
     {
         self.tcx.sess.perf_stats.queries_canonicalized.fetch_add(1, Ordering::Relaxed);
 
@@ -100,7 +100,7 @@ impl<'tcx> InferCtxt<'tcx> {
     /// [c]: https://rust-lang.github.io/chalk/book/canonical_queries/canonicalization.html#canonicalizing-the-query-result
     pub fn canonicalize_response<V>(&self, value: V) -> Canonical<'tcx, V>
     where
-        V: TypeFoldable<'tcx>,
+        V: TypeFoldable<TyCtxt<'tcx>>,
     {
         let mut query_state = OriginalQueryValues::default();
         Canonicalizer::canonicalize(
@@ -114,7 +114,7 @@ impl<'tcx> InferCtxt<'tcx> {
 
     pub fn canonicalize_user_type_annotation<V>(&self, value: V) -> Canonical<'tcx, V>
     where
-        V: TypeFoldable<'tcx>,
+        V: TypeFoldable<TyCtxt<'tcx>>,
     {
         let mut query_state = OriginalQueryValues::default();
         Canonicalizer::canonicalize(
@@ -136,7 +136,7 @@ impl<'tcx> InferCtxt<'tcx> {
         query_state: &mut OriginalQueryValues<'tcx>,
     ) -> Canonical<'tcx, V>
     where
-        V: TypeFoldable<'tcx>,
+        V: TypeFoldable<TyCtxt<'tcx>>,
     {
         self.tcx.sess.perf_stats.queries_canonicalized.fetch_add(1, Ordering::Relaxed);
 
@@ -203,12 +203,10 @@ impl CanonicalizeMode for CanonicalizeQueryResponse {
                 // rust-lang/rust#57464: `impl Trait` can leak local
                 // scopes (in manner violating typeck). Therefore, use
                 // `delay_span_bug` to allow type error over an ICE.
-                ty::tls::with(|tcx| {
-                    tcx.sess.delay_span_bug(
-                        rustc_span::DUMMY_SP,
-                        &format!("unexpected region in query response: `{:?}`", r),
-                    );
-                });
+                canonicalizer.tcx.sess.delay_span_bug(
+                    rustc_span::DUMMY_SP,
+                    &format!("unexpected region in query response: `{:?}`", r),
+                );
                 r
             }
         }
@@ -328,14 +326,14 @@ struct Canonicalizer<'cx, 'tcx> {
     binder_index: ty::DebruijnIndex,
 }
 
-impl<'cx, 'tcx> TypeFolder<'tcx> for Canonicalizer<'cx, 'tcx> {
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
     fn fold_binder<T>(&mut self, t: ty::Binder<'tcx, T>) -> ty::Binder<'tcx, T>
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         self.binder_index.shift_in(1);
         let t = t.super_fold_with(self);
@@ -365,12 +363,13 @@ impl<'cx, 'tcx> TypeFolder<'tcx> for Canonicalizer<'cx, 'tcx> {
                      opportunistically resolved to {:?}",
                     vid, resolved_vid
                 );
-                let r = self.tcx.reuse_or_mk_region(r, ty::ReVar(resolved_vid));
+                let r = self.tcx.mk_re_var(resolved_vid);
                 self.canonicalize_mode.canonicalize_free_region(self, r)
             }
 
             ty::ReStatic
             | ty::ReEarlyBound(..)
+            | ty::ReError(_)
             | ty::ReFree(_)
             | ty::RePlaceholder(..)
             | ty::ReErased => self.canonicalize_mode.canonicalize_free_region(self, r),
@@ -419,10 +418,15 @@ impl<'cx, 'tcx> TypeFolder<'tcx> for Canonicalizer<'cx, 'tcx> {
                 bug!("encountered a fresh type during canonicalization")
             }
 
-            ty::Placeholder(placeholder) => self.canonicalize_ty_var(
-                CanonicalVarInfo { kind: CanonicalVarKind::PlaceholderTy(placeholder) },
-                t,
-            ),
+            ty::Placeholder(mut placeholder) => {
+                if !self.canonicalize_mode.preserve_universes() {
+                    placeholder.universe = ty::UniverseIndex::ROOT;
+                }
+                self.canonicalize_ty_var(
+                    CanonicalVarInfo { kind: CanonicalVarKind::PlaceholderTy(placeholder) },
+                    t,
+                )
+            }
 
             ty::Bound(debruijn, _) => {
                 if debruijn >= self.binder_index {
@@ -435,6 +439,7 @@ impl<'cx, 'tcx> TypeFolder<'tcx> for Canonicalizer<'cx, 'tcx> {
             ty::Closure(..)
             | ty::Generator(..)
             | ty::GeneratorWitness(..)
+            | ty::GeneratorWitnessMIR(..)
             | ty::Bool
             | ty::Char
             | ty::Int(..)
@@ -525,7 +530,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
         query_state: &mut OriginalQueryValues<'tcx>,
     ) -> Canonical<'tcx, V>
     where
-        V: TypeFoldable<'tcx>,
+        V: TypeFoldable<TyCtxt<'tcx>>,
     {
         let needs_canonical_flags = if canonicalize_region_mode.any() {
             TypeFlags::NEEDS_INFER |
@@ -567,7 +572,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
         debug_assert!(!out_value.needs_infer() && !out_value.has_placeholders());
 
         let canonical_variables =
-            tcx.intern_canonical_var_infos(&canonicalizer.universe_canonicalized_variables());
+            tcx.mk_canonical_var_infos(&canonicalizer.universe_canonicalized_variables());
 
         let max_universe = canonical_variables
             .iter()
@@ -737,8 +742,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
     ) -> ty::Region<'tcx> {
         let var = self.canonical_var(info, r.into());
         let br = ty::BoundRegion { var, kind: ty::BrAnon(var.as_u32(), None) };
-        let region = ty::ReLateBound(self.binder_index, br);
-        self.tcx().mk_region(region)
+        self.interner().mk_re_late_bound(self.binder_index, br)
     }
 
     /// Given a type variable `ty_var` of the given kind, first check
@@ -752,7 +756,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
             self.fold_ty(bound_to)
         } else {
             let var = self.canonical_var(info, ty_var.into());
-            self.tcx().mk_ty(ty::Bound(self.binder_index, var.into()))
+            self.interner().mk_bound(self.binder_index, var.into())
         }
     }
 
@@ -771,7 +775,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
             self.fold_const(bound_to)
         } else {
             let var = self.canonical_var(info, const_var.into());
-            self.tcx().mk_const(
+            self.interner().mk_const(
                 ty::ConstKind::Bound(self.binder_index, var),
                 self.fold_ty(const_var.ty()),
             )
diff --git a/compiler/rustc_infer/src/infer/canonical/mod.rs b/compiler/rustc_infer/src/infer/canonical/mod.rs
index e59715b70..ce230afda 100644
--- a/compiler/rustc_infer/src/infer/canonical/mod.rs
+++ b/compiler/rustc_infer/src/infer/canonical/mod.rs
@@ -26,11 +26,11 @@ use crate::infer::{InferCtxt, RegionVariableOrigin, TypeVariableOrigin, TypeVari
 use rustc_index::vec::IndexVec;
 use rustc_middle::ty::fold::TypeFoldable;
 use rustc_middle::ty::subst::GenericArg;
-use rustc_middle::ty::{self, BoundVar, List};
+use rustc_middle::ty::{self, List, TyCtxt};
 use rustc_span::source_map::Span;
 
 pub use rustc_middle::infer::canonical::*;
-use substitute::CanonicalExt;
+pub use substitute::CanonicalExt;
 
 mod canonicalizer;
 pub mod query_response;
@@ -55,7 +55,7 @@ impl<'tcx> InferCtxt<'tcx> {
         canonical: &Canonical<'tcx, T>,
     ) -> (T, CanonicalVarValues<'tcx>)
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         // For each universe that is referred to in the incoming
         // query, create a universe in our local inference context. In
@@ -87,19 +87,24 @@ impl<'tcx> InferCtxt<'tcx> {
         variables: &List<CanonicalVarInfo<'tcx>>,
         universe_map: impl Fn(ty::UniverseIndex) -> ty::UniverseIndex,
     ) -> CanonicalVarValues<'tcx> {
-        let var_values: IndexVec<BoundVar, GenericArg<'tcx>> = variables
-            .iter()
-            .map(|info| self.instantiate_canonical_var(span, info, &universe_map))
-            .collect();
-
-        CanonicalVarValues { var_values }
+        CanonicalVarValues {
+            var_values: self.tcx.mk_substs_from_iter(
+                variables
+                    .iter()
+                    .map(|info| self.instantiate_canonical_var(span, info, &universe_map)),
+            ),
+        }
     }
 
     /// Given the "info" about a canonical variable, creates a fresh
     /// variable for it. If this is an existentially quantified
     /// variable, then you'll get a new inference variable; if it is a
     /// universally quantified variable, you get a placeholder.
-    fn instantiate_canonical_var(
+    ///
+    /// FIXME(-Ztrait-solver=next): This is public because it's used by the
+    /// new trait solver which has a different canonicalization routine.
+    /// We should somehow deduplicate all of this.
+    pub fn instantiate_canonical_var(
         &self,
         span: Span,
         cv_info: CanonicalVarInfo<'tcx>,
@@ -123,7 +128,7 @@ impl<'tcx> InferCtxt<'tcx> {
             CanonicalVarKind::PlaceholderTy(ty::PlaceholderType { universe, name }) => {
                 let universe_mapped = universe_map(universe);
                 let placeholder_mapped = ty::PlaceholderType { universe: universe_mapped, name };
-                self.tcx.mk_ty(ty::Placeholder(placeholder_mapped)).into()
+                self.tcx.mk_placeholder(placeholder_mapped).into()
             }
 
             CanonicalVarKind::Region(ui) => self
@@ -136,7 +141,7 @@ impl<'tcx> InferCtxt<'tcx> {
             CanonicalVarKind::PlaceholderRegion(ty::PlaceholderRegion { universe, name }) => {
                 let universe_mapped = universe_map(universe);
                 let placeholder_mapped = ty::PlaceholderRegion { universe: universe_mapped, name };
-                self.tcx.mk_region(ty::RePlaceholder(placeholder_mapped)).into()
+                self.tcx.mk_re_placeholder(placeholder_mapped).into()
             }
 
             CanonicalVarKind::Const(ui, ty) => self
diff --git a/compiler/rustc_infer/src/infer/canonical/query_response.rs b/compiler/rustc_infer/src/infer/canonical/query_response.rs
index 3d49182f0..436d29c24 100644
--- a/compiler/rustc_infer/src/infer/canonical/query_response.rs
+++ b/compiler/rustc_infer/src/infer/canonical/query_response.rs
@@ -12,12 +12,12 @@ use crate::infer::canonical::{
     Canonical, CanonicalQueryResponse, CanonicalVarValues, Certainty, OriginalQueryValues,
     QueryOutlivesConstraint, QueryRegionConstraints, QueryResponse,
 };
-use crate::infer::nll_relate::{NormalizationStrategy, TypeRelating, TypeRelatingDelegate};
+use crate::infer::nll_relate::{TypeRelating, TypeRelatingDelegate};
 use crate::infer::region_constraints::{Constraint, RegionConstraintData};
 use crate::infer::{InferCtxt, InferOk, InferResult, NllRegionVariableOrigin};
 use crate::traits::query::{Fallible, NoSolution};
 use crate::traits::{Obligation, ObligationCause, PredicateObligation};
-use crate::traits::{PredicateObligations, TraitEngine};
+use crate::traits::{PredicateObligations, TraitEngine, TraitEngineExt};
 use rustc_data_structures::captures::Captures;
 use rustc_index::vec::Idx;
 use rustc_index::vec::IndexVec;
@@ -27,7 +27,7 @@ use rustc_middle::ty::fold::TypeFoldable;
 use rustc_middle::ty::relate::TypeRelation;
 use rustc_middle::ty::subst::{GenericArg, GenericArgKind};
 use rustc_middle::ty::{self, BoundVar, ToPredicate, Ty, TyCtxt};
-use rustc_span::Span;
+use rustc_span::{Span, Symbol};
 use std::fmt::Debug;
 use std::iter;
 
@@ -59,7 +59,7 @@ impl<'tcx> InferCtxt<'tcx> {
         fulfill_cx: &mut dyn TraitEngine<'tcx>,
     ) -> Fallible<CanonicalQueryResponse<'tcx, T>>
     where
-        T: Debug + TypeFoldable<'tcx>,
+        T: Debug + TypeFoldable<TyCtxt<'tcx>>,
         Canonical<'tcx, QueryResponse<'tcx, T>>: ArenaAllocatable<'tcx>,
     {
         let query_response = self.make_query_response(inference_vars, answer, fulfill_cx)?;
@@ -85,7 +85,7 @@ impl<'tcx> InferCtxt<'tcx> {
         answer: T,
     ) -> Canonical<'tcx, QueryResponse<'tcx, T>>
     where
-        T: Debug + TypeFoldable<'tcx>,
+        T: Debug + TypeFoldable<TyCtxt<'tcx>>,
     {
         self.canonicalize_response(QueryResponse {
             var_values: inference_vars,
@@ -106,7 +106,7 @@ impl<'tcx> InferCtxt<'tcx> {
         fulfill_cx: &mut dyn TraitEngine<'tcx>,
     ) -> Result<QueryResponse<'tcx, T>, NoSolution>
     where
-        T: Debug + TypeFoldable<'tcx>,
+        T: Debug + TypeFoldable<TyCtxt<'tcx>>,
     {
         let tcx = self.tcx;
 
@@ -151,11 +151,21 @@ impl<'tcx> InferCtxt<'tcx> {
         })
     }
 
-    /// FIXME: This method should only be used for canonical queries and therefore be private.
-    ///
-    /// As the new solver does canonicalization slightly differently, this is also used there
-    /// for now. This should hopefully change fairly soon.
-    pub fn take_opaque_types_for_query_response(&self) -> Vec<(Ty<'tcx>, Ty<'tcx>)> {
+    /// Used by the new solver as that one takes the opaque types at the end of a probe
+    /// to deal with multiple candidates without having to recompute them.
+    pub fn clone_opaque_types_for_query_response(&self) -> Vec<(Ty<'tcx>, Ty<'tcx>)> {
+        self.inner
+            .borrow()
+            .opaque_type_storage
+            .opaque_types
+            .iter()
+            .map(|&(k, ref v)| {
+                (self.tcx.mk_opaque(k.def_id.to_def_id(), k.substs), v.hidden_type.ty)
+            })
+            .collect()
+    }
+
+    fn take_opaque_types_for_query_response(&self) -> Vec<(Ty<'tcx>, Ty<'tcx>)> {
         std::mem::take(&mut self.inner.borrow_mut().opaque_type_storage.opaque_types)
             .into_iter()
             .map(|(k, v)| (self.tcx.mk_opaque(k.def_id.to_def_id(), k.substs), v.hidden_type.ty))
@@ -180,7 +190,7 @@ impl<'tcx> InferCtxt<'tcx> {
         query_response: &Canonical<'tcx, QueryResponse<'tcx, R>>,
     ) -> InferResult<'tcx, R>
     where
-        R: Debug + TypeFoldable<'tcx>,
+        R: Debug + TypeFoldable<TyCtxt<'tcx>>,
     {
         let InferOk { value: result_subst, mut obligations } =
             self.query_response_substitution(cause, param_env, original_values, query_response)?;
@@ -242,7 +252,7 @@ impl<'tcx> InferCtxt<'tcx> {
         output_query_region_constraints: &mut QueryRegionConstraints<'tcx>,
     ) -> InferResult<'tcx, R>
     where
-        R: Debug + TypeFoldable<'tcx>,
+        R: Debug + TypeFoldable<TyCtxt<'tcx>>,
     {
         let InferOk { value: result_subst, mut obligations } = self
             .query_response_substitution_guess(cause, param_env, original_values, query_response)?;
@@ -268,14 +278,12 @@ impl<'tcx> InferCtxt<'tcx> {
                 (GenericArgKind::Lifetime(v_o), GenericArgKind::Lifetime(v_r)) => {
                     // To make `v_o = v_r`, we emit `v_o: v_r` and `v_r: v_o`.
                     if v_o != v_r {
-                        output_query_region_constraints.outlives.push((
-                            ty::Binder::dummy(ty::OutlivesPredicate(v_o.into(), v_r)),
-                            constraint_category,
-                        ));
-                        output_query_region_constraints.outlives.push((
-                            ty::Binder::dummy(ty::OutlivesPredicate(v_r.into(), v_o)),
-                            constraint_category,
-                        ));
+                        output_query_region_constraints
+                            .outlives
+                            .push((ty::OutlivesPredicate(v_o.into(), v_r), constraint_category));
+                        output_query_region_constraints
+                            .outlives
+                            .push((ty::OutlivesPredicate(v_r.into(), v_o), constraint_category));
                     }
                 }
 
@@ -318,10 +326,8 @@ impl<'tcx> InferCtxt<'tcx> {
             query_response.value.region_constraints.outlives.iter().filter_map(|&r_c| {
                 let r_c = substitute_value(self.tcx, &result_subst, r_c);
 
-                // Screen out `'a: 'a` cases -- we skip the binder here but
-                // only compare the inner values to one another, so they are still at
-                // consistent binding levels.
-                let ty::OutlivesPredicate(k1, r2) = r_c.0.skip_binder();
+                // Screen out `'a: 'a` cases.
+                let ty::OutlivesPredicate(k1, r2) = r_c.0;
                 if k1 != r2.into() { Some(r_c) } else { None }
             }),
         );
@@ -360,7 +366,7 @@ impl<'tcx> InferCtxt<'tcx> {
         query_response: &Canonical<'tcx, QueryResponse<'tcx, R>>,
     ) -> InferResult<'tcx, CanonicalVarValues<'tcx>>
     where
-        R: Debug + TypeFoldable<'tcx>,
+        R: Debug + TypeFoldable<TyCtxt<'tcx>>,
     {
         debug!(
             "query_response_substitution(original_values={:#?}, query_response={:#?})",
@@ -397,6 +403,7 @@ impl<'tcx> InferCtxt<'tcx> {
     /// will instantiate fresh inference variables for each canonical
     /// variable instead. Therefore, the result of this method must be
     /// properly unified
+    #[instrument(level = "debug", skip(self, cause, param_env))]
     fn query_response_substitution_guess<R>(
         &self,
         cause: &ObligationCause<'tcx>,
@@ -405,13 +412,8 @@ impl<'tcx> InferCtxt<'tcx> {
         query_response: &Canonical<'tcx, QueryResponse<'tcx, R>>,
     ) -> InferResult<'tcx, CanonicalVarValues<'tcx>>
     where
-        R: Debug + TypeFoldable<'tcx>,
+        R: Debug + TypeFoldable<TyCtxt<'tcx>>,
     {
-        debug!(
-            "query_response_substitution_guess(original_values={:#?}, query_response={:#?})",
-            original_values, query_response,
-        );
-
         // For each new universe created in the query result that did
         // not appear in the original query, create a local
         // superuniverse.
@@ -482,11 +484,8 @@ impl<'tcx> InferCtxt<'tcx> {
         // given variable in the loop above, use that. Otherwise, use
         // a fresh inference variable.
         let result_subst = CanonicalVarValues {
-            var_values: query_response
-                .variables
-                .iter()
-                .enumerate()
-                .map(|(index, info)| {
+            var_values: self.tcx.mk_substs_from_iter(
+                query_response.variables.iter().enumerate().map(|(index, info)| {
                     if info.is_existential() {
                         match opt_values[BoundVar::new(index)] {
                             Some(k) => k,
@@ -499,8 +498,8 @@ impl<'tcx> InferCtxt<'tcx> {
                             universe_map[u.as_usize()]
                         })
                     }
-                })
-                .collect(),
+                }),
+            ),
         };
 
         let mut obligations = vec![];
@@ -509,7 +508,9 @@ impl<'tcx> InferCtxt<'tcx> {
         for &(a, b) in &query_response.value.opaque_types {
             let a = substitute_value(self.tcx, &result_subst, a);
             let b = substitute_value(self.tcx, &result_subst, b);
-            obligations.extend(self.at(cause, param_env).eq(a, b)?.obligations);
+            debug!(?a, ?b, "constrain opaque type");
+            obligations
+                .extend(self.at(cause, param_env).define_opaque_types(true).eq(a, b)?.obligations);
         }
 
         Ok(InferOk { value: result_subst, obligations })
@@ -530,7 +531,7 @@ impl<'tcx> InferCtxt<'tcx> {
         query_response: &Canonical<'tcx, QueryResponse<'tcx, R>>,
     ) -> InferResult<'tcx, ()>
     where
-        R: Debug + TypeFoldable<'tcx>,
+        R: Debug + TypeFoldable<TyCtxt<'tcx>>,
     {
         // A closure that yields the result value for the given
         // canonical variable; this is taken from
@@ -562,11 +563,11 @@ impl<'tcx> InferCtxt<'tcx> {
 
     pub fn query_outlives_constraint_to_obligation(
         &self,
-        predicate: QueryOutlivesConstraint<'tcx>,
+        (predicate, _): QueryOutlivesConstraint<'tcx>,
         cause: ObligationCause<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
     ) -> Obligation<'tcx, ty::Predicate<'tcx>> {
-        let ty::OutlivesPredicate(k1, r2) = predicate.0.skip_binder();
+        let ty::OutlivesPredicate(k1, r2) = predicate;
 
         let atom = match k1.unpack() {
             GenericArgKind::Lifetime(r1) => {
@@ -581,7 +582,7 @@ impl<'tcx> InferCtxt<'tcx> {
                 span_bug!(cause.span, "unexpected const outlives {:?}", predicate);
             }
         };
-        let predicate = predicate.0.rebind(atom);
+        let predicate = ty::Binder::dummy(atom);
 
         Obligation::new(self.tcx, cause, param_env, predicate)
     }
@@ -646,31 +647,25 @@ pub fn make_query_region_constraints<'tcx>(
     let outlives: Vec<_> = constraints
         .iter()
         .map(|(k, origin)| {
-            // no bound vars in the code above
-            let constraint = ty::Binder::dummy(match *k {
+            let constraint = match *k {
                 // Swap regions because we are going from sub (<=) to outlives
                 // (>=).
-                Constraint::VarSubVar(v1, v2) => ty::OutlivesPredicate(
-                    tcx.mk_region(ty::ReVar(v2)).into(),
-                    tcx.mk_region(ty::ReVar(v1)),
-                ),
+                Constraint::VarSubVar(v1, v2) => {
+                    ty::OutlivesPredicate(tcx.mk_re_var(v2).into(), tcx.mk_re_var(v1))
+                }
                 Constraint::VarSubReg(v1, r2) => {
-                    ty::OutlivesPredicate(r2.into(), tcx.mk_region(ty::ReVar(v1)))
+                    ty::OutlivesPredicate(r2.into(), tcx.mk_re_var(v1))
                 }
                 Constraint::RegSubVar(r1, v2) => {
-                    ty::OutlivesPredicate(tcx.mk_region(ty::ReVar(v2)).into(), r1)
+                    ty::OutlivesPredicate(tcx.mk_re_var(v2).into(), r1)
                 }
                 Constraint::RegSubReg(r1, r2) => ty::OutlivesPredicate(r2.into(), r1),
-            });
+            };
             (constraint, origin.to_constraint_category())
         })
-        .chain(
-            outlives_obligations
-                // no bound vars in the code above
-                .map(|(ty, r, constraint_category)| {
-                    (ty::Binder::dummy(ty::OutlivesPredicate(ty.into(), r)), constraint_category)
-                }),
-        )
+        .chain(outlives_obligations.map(|(ty, r, constraint_category)| {
+            (ty::OutlivesPredicate(ty.into(), r), constraint_category)
+        }))
         .collect();
 
     QueryRegionConstraints { outlives, member_constraints: member_constraints.clone() }
@@ -696,13 +691,17 @@ impl<'tcx> TypeRelatingDelegate<'tcx> for QueryTypeRelatingDelegate<'_, 'tcx> {
         self.infcx.create_next_universe()
     }
 
-    fn next_existential_region_var(&mut self, from_forall: bool) -> ty::Region<'tcx> {
+    fn next_existential_region_var(
+        &mut self,
+        from_forall: bool,
+        _name: Option<Symbol>,
+    ) -> ty::Region<'tcx> {
         let origin = NllRegionVariableOrigin::Existential { from_forall };
         self.infcx.next_nll_region_var(origin)
     }
 
     fn next_placeholder_region(&mut self, placeholder: ty::PlaceholderRegion) -> ty::Region<'tcx> {
-        self.infcx.tcx.mk_region(ty::RePlaceholder(placeholder))
+        self.infcx.tcx.mk_re_placeholder(placeholder)
     }
 
     fn generalize_existential(&mut self, universe: ty::UniverseIndex) -> ty::Region<'tcx> {
@@ -729,10 +728,6 @@ impl<'tcx> TypeRelatingDelegate<'tcx> for QueryTypeRelatingDelegate<'_, 'tcx> {
         });
     }
 
-    fn normalization() -> NormalizationStrategy {
-        NormalizationStrategy::Eager
-    }
-
     fn forbid_inference_vars() -> bool {
         true
     }
diff --git a/compiler/rustc_infer/src/infer/canonical/substitute.rs b/compiler/rustc_infer/src/infer/canonical/substitute.rs
index 389afe22e..cac3b4072 100644
--- a/compiler/rustc_infer/src/infer/canonical/substitute.rs
+++ b/compiler/rustc_infer/src/infer/canonical/substitute.rs
@@ -11,12 +11,14 @@ use rustc_middle::ty::fold::{FnMutDelegate, TypeFoldable};
 use rustc_middle::ty::subst::GenericArgKind;
 use rustc_middle::ty::{self, TyCtxt};
 
-pub(super) trait CanonicalExt<'tcx, V> {
+/// FIXME(-Ztrait-solver=next): This or public because it is shared with the
+/// new trait solver implementation. We should deduplicate canonicalization.
+pub trait CanonicalExt<'tcx, V> {
     /// Instantiate the wrapped value, replacing each canonical value
     /// with the value given in `var_values`.
     fn substitute(&self, tcx: TyCtxt<'tcx>, var_values: &CanonicalVarValues<'tcx>) -> V
     where
-        V: TypeFoldable<'tcx>;
+        V: TypeFoldable<TyCtxt<'tcx>>;
 
     /// Allows one to apply a substitute to some subset of
     /// `self.value`. Invoke `projection_fn` with `self.value` to get
@@ -31,13 +33,13 @@ pub(super) trait CanonicalExt<'tcx, V> {
         projection_fn: impl FnOnce(&V) -> T,
     ) -> T
     where
-        T: TypeFoldable<'tcx>;
+        T: TypeFoldable<TyCtxt<'tcx>>;
 }
 
 impl<'tcx, V> CanonicalExt<'tcx, V> for Canonical<'tcx, V> {
     fn substitute(&self, tcx: TyCtxt<'tcx>, var_values: &CanonicalVarValues<'tcx>) -> V
     where
-        V: TypeFoldable<'tcx>,
+        V: TypeFoldable<TyCtxt<'tcx>>,
     {
         self.substitute_projected(tcx, var_values, |value| value.clone())
     }
@@ -49,7 +51,7 @@ impl<'tcx, V> CanonicalExt<'tcx, V> for Canonical<'tcx, V> {
         projection_fn: impl FnOnce(&V) -> T,
     ) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         assert_eq!(self.variables.len(), var_values.len());
         let value = projection_fn(&self.value);
@@ -66,22 +68,21 @@ pub(super) fn substitute_value<'tcx, T>(
     value: T,
 ) -> T
 where
-    T: TypeFoldable<'tcx>,
+    T: TypeFoldable<TyCtxt<'tcx>>,
 {
     if var_values.var_values.is_empty() {
         value
     } else {
         let delegate = FnMutDelegate {
-            regions: &mut |br: ty::BoundRegion| match var_values.var_values[br.var].unpack() {
+            regions: &mut |br: ty::BoundRegion| match var_values[br.var].unpack() {
                 GenericArgKind::Lifetime(l) => l,
                 r => bug!("{:?} is a region but value is {:?}", br, r),
             },
-            types: &mut |bound_ty: ty::BoundTy| match var_values.var_values[bound_ty.var].unpack() {
+            types: &mut |bound_ty: ty::BoundTy| match var_values[bound_ty.var].unpack() {
                 GenericArgKind::Type(ty) => ty,
                 r => bug!("{:?} is a type but value is {:?}", bound_ty, r),
             },
-            consts: &mut |bound_ct: ty::BoundVar, _| match var_values.var_values[bound_ct].unpack()
-            {
+            consts: &mut |bound_ct: ty::BoundVar, _| match var_values[bound_ct].unpack() {
                 GenericArgKind::Const(ct) => ct,
                 c => bug!("{:?} is a const but value is {:?}", bound_ct, c),
             },
diff --git a/compiler/rustc_infer/src/infer/combine.rs b/compiler/rustc_infer/src/infer/combine.rs
index 72676b718..33292e871 100644
--- a/compiler/rustc_infer/src/infer/combine.rs
+++ b/compiler/rustc_infer/src/infer/combine.rs
@@ -31,13 +31,18 @@ use super::{InferCtxt, MiscVariable, TypeTrace};
 use crate::traits::{Obligation, PredicateObligations};
 use rustc_data_structures::sso::SsoHashMap;
 use rustc_hir::def_id::DefId;
+use rustc_middle::infer::canonical::OriginalQueryValues;
 use rustc_middle::infer::unify_key::{ConstVarValue, ConstVariableValue};
 use rustc_middle::infer::unify_key::{ConstVariableOrigin, ConstVariableOriginKind};
+use rustc_middle::traits::query::NoSolution;
 use rustc_middle::traits::ObligationCause;
 use rustc_middle::ty::error::{ExpectedFound, TypeError};
 use rustc_middle::ty::relate::{self, Relate, RelateResult, TypeRelation};
 use rustc_middle::ty::subst::SubstsRef;
-use rustc_middle::ty::{self, InferConst, Ty, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{
+    self, AliasKind, FallibleTypeFolder, InferConst, ToPredicate, Ty, TyCtxt, TypeFoldable,
+    TypeSuperFoldable, TypeVisitableExt,
+};
 use rustc_middle::ty::{IntType, UintType};
 use rustc_span::{Span, DUMMY_SP};
 
@@ -71,7 +76,7 @@ impl<'tcx> InferCtxt<'tcx> {
         b: Ty<'tcx>,
     ) -> RelateResult<'tcx, Ty<'tcx>>
     where
-        R: TypeRelation<'tcx>,
+        R: ObligationEmittingRelation<'tcx>,
     {
         let a_is_expected = relation.a_is_expected();
 
@@ -119,6 +124,15 @@ impl<'tcx> InferCtxt<'tcx> {
                 Err(TypeError::Sorts(ty::relate::expected_found(relation, a, b)))
             }
 
+            (ty::Alias(AliasKind::Projection, _), _) if self.tcx.trait_solver_next() => {
+                relation.register_type_equate_obligation(a, b);
+                Ok(b)
+            }
+            (_, ty::Alias(AliasKind::Projection, _)) if self.tcx.trait_solver_next() => {
+                relation.register_type_equate_obligation(b, a);
+                Ok(a)
+            }
+
             _ => ty::relate::super_relate_tys(relation, a, b),
         }
     }
@@ -130,7 +144,7 @@ impl<'tcx> InferCtxt<'tcx> {
         b: ty::Const<'tcx>,
     ) -> RelateResult<'tcx, ty::Const<'tcx>>
     where
-        R: ConstEquateRelation<'tcx>,
+        R: ObligationEmittingRelation<'tcx>,
     {
         debug!("{}.consts({:?}, {:?})", relation.tag(), a, b);
         if a == b {
@@ -140,7 +154,33 @@ impl<'tcx> InferCtxt<'tcx> {
         let a = self.shallow_resolve(a);
         let b = self.shallow_resolve(b);
 
-        let a_is_expected = relation.a_is_expected();
+        // We should never have to relate the `ty` field on `Const` as it is checked elsewhere that consts have the
+        // correct type for the generic param they are an argument for. However there have been a number of cases
+        // historically where asserting that the types are equal has found bugs in the compiler so this is valuable
+        // to check even if it is a bit nasty impl wise :(
+        //
+        // This probe is probably not strictly necessary but it seems better to be safe and not accidentally find
+        // ourselves with a check to find bugs being required for code to compile because it made inference progress.
+        self.probe(|_| {
+            if a.ty() == b.ty() {
+                return;
+            }
+
+            // We don't have access to trait solving machinery in `rustc_infer` so the logic for determining if the
+            // two const param's types are able to be equal has to go through a canonical query with the actual logic
+            // in `rustc_trait_selection`.
+            let canonical = self.canonicalize_query(
+                (relation.param_env(), a.ty(), b.ty()),
+                &mut OriginalQueryValues::default(),
+            );
+
+            if let Err(NoSolution) = self.tcx.check_tys_might_be_eq(canonical) {
+                self.tcx.sess.delay_span_bug(
+                    DUMMY_SP,
+                    &format!("cannot relate consts of different types (a={:?}, b={:?})", a, b,),
+                );
+            }
+        });
 
         match (a.kind(), b.kind()) {
             (
@@ -158,17 +198,17 @@ impl<'tcx> InferCtxt<'tcx> {
             }
 
             (ty::ConstKind::Infer(InferConst::Var(vid)), _) => {
-                return self.unify_const_variable(relation.param_env(), vid, b, a_is_expected);
+                return self.unify_const_variable(vid, b);
             }
 
             (_, ty::ConstKind::Infer(InferConst::Var(vid))) => {
-                return self.unify_const_variable(relation.param_env(), vid, a, !a_is_expected);
+                return self.unify_const_variable(vid, a);
             }
             (ty::ConstKind::Unevaluated(..), _) if self.tcx.lazy_normalization() => {
                 // FIXME(#59490): Need to remove the leak check to accommodate
                 // escaping bound variables here.
                 if !a.has_escaping_bound_vars() && !b.has_escaping_bound_vars() {
-                    relation.const_equate_obligation(a, b);
+                    relation.register_const_equate_obligation(a, b);
                 }
                 return Ok(b);
             }
@@ -176,7 +216,7 @@ impl<'tcx> InferCtxt<'tcx> {
                 // FIXME(#59490): Need to remove the leak check to accommodate
                 // escaping bound variables here.
                 if !a.has_escaping_bound_vars() && !b.has_escaping_bound_vars() {
-                    relation.const_equate_obligation(a, b);
+                    relation.register_const_equate_obligation(a, b);
                 }
                 return Ok(a);
             }
@@ -223,10 +263,8 @@ impl<'tcx> InferCtxt<'tcx> {
     #[instrument(level = "debug", skip(self))]
     fn unify_const_variable(
         &self,
-        param_env: ty::ParamEnv<'tcx>,
         target_vid: ty::ConstVid<'tcx>,
         ct: ty::Const<'tcx>,
-        vid_is_expected: bool,
     ) -> RelateResult<'tcx, ty::Const<'tcx>> {
         let (for_universe, span) = {
             let mut inner = self.inner.borrow_mut();
@@ -239,8 +277,12 @@ impl<'tcx> InferCtxt<'tcx> {
                 ConstVariableValue::Unknown { universe } => (universe, var_value.origin.span),
             }
         };
-        let value = ConstInferUnifier { infcx: self, span, param_env, for_universe, target_vid }
-            .relate(ct, ct)?;
+        let value = ct.try_fold_with(&mut ConstInferUnifier {
+            infcx: self,
+            span,
+            for_universe,
+            target_vid,
+        })?;
 
         self.inner.borrow_mut().const_unification_table().union_value(
             target_vid,
@@ -432,32 +474,18 @@ impl<'infcx, 'tcx> CombineFields<'infcx, 'tcx> {
         Ok(Generalization { ty, needs_wf })
     }
 
-    pub fn add_const_equate_obligation(
-        &mut self,
-        a_is_expected: bool,
-        a: ty::Const<'tcx>,
-        b: ty::Const<'tcx>,
-    ) {
-        let predicate = if a_is_expected {
-            ty::PredicateKind::ConstEquate(a, b)
-        } else {
-            ty::PredicateKind::ConstEquate(b, a)
-        };
-        self.obligations.push(Obligation::new(
-            self.tcx(),
-            self.trace.cause.clone(),
-            self.param_env,
-            ty::Binder::dummy(predicate),
-        ));
+    pub fn register_obligations(&mut self, obligations: PredicateObligations<'tcx>) {
+        self.obligations.extend(obligations.into_iter());
+    }
+
+    pub fn register_predicates(&mut self, obligations: impl IntoIterator<Item: ToPredicate<'tcx>>) {
+        self.obligations.extend(obligations.into_iter().map(|to_pred| {
+            Obligation::new(self.infcx.tcx, self.trace.cause.clone(), self.param_env, to_pred)
+        }))
     }
 
     pub fn mark_ambiguous(&mut self) {
-        self.obligations.push(Obligation::new(
-            self.tcx(),
-            self.trace.cause.clone(),
-            self.param_env,
-            ty::Binder::dummy(ty::PredicateKind::Ambiguous),
-        ));
+        self.register_predicates([ty::Binder::dummy(ty::PredicateKind::Ambiguous)]);
     }
 }
 
@@ -702,6 +730,10 @@ impl<'tcx> TypeRelation<'tcx> for Generalizer<'_, 'tcx> {
                 return Ok(r);
             }
 
+            ty::ReError(_) => {
+                return Ok(r);
+            }
+
             ty::RePlaceholder(..)
             | ty::ReVar(..)
             | ty::ReStatic
@@ -772,11 +804,39 @@ impl<'tcx> TypeRelation<'tcx> for Generalizer<'_, 'tcx> {
     }
 }
 
-pub trait ConstEquateRelation<'tcx>: TypeRelation<'tcx> {
+pub trait ObligationEmittingRelation<'tcx>: TypeRelation<'tcx> {
+    /// Register obligations that must hold in order for this relation to hold
+    fn register_obligations(&mut self, obligations: PredicateObligations<'tcx>);
+
+    /// Register predicates that must hold in order for this relation to hold. Uses
+    /// a default obligation cause, [`ObligationEmittingRelation::register_obligations`] should
+    /// be used if control over the obligaton causes is required.
+    fn register_predicates(&mut self, obligations: impl IntoIterator<Item: ToPredicate<'tcx>>);
+
     /// Register an obligation that both constants must be equal to each other.
     ///
     /// If they aren't equal then the relation doesn't hold.
-    fn const_equate_obligation(&mut self, a: ty::Const<'tcx>, b: ty::Const<'tcx>);
+    fn register_const_equate_obligation(&mut self, a: ty::Const<'tcx>, b: ty::Const<'tcx>) {
+        let (a, b) = if self.a_is_expected() { (a, b) } else { (b, a) };
+
+        self.register_predicates([ty::Binder::dummy(if self.tcx().trait_solver_next() {
+            ty::PredicateKind::AliasEq(a.into(), b.into())
+        } else {
+            ty::PredicateKind::ConstEquate(a, b)
+        })]);
+    }
+
+    /// Register an obligation that both types must be equal to each other.
+    ///
+    /// If they aren't equal then the relation doesn't hold.
+    fn register_type_equate_obligation(&mut self, a: Ty<'tcx>, b: Ty<'tcx>) {
+        let (a, b) = if self.a_is_expected() { (a, b) } else { (b, a) };
+
+        self.register_predicates([ty::Binder::dummy(ty::PredicateKind::AliasEq(
+            a.into(),
+            b.into(),
+        ))]);
+    }
 }
 
 fn int_unification_error<'tcx>(
@@ -800,8 +860,6 @@ struct ConstInferUnifier<'cx, 'tcx> {
 
     span: Span,
 
-    param_env: ty::ParamEnv<'tcx>,
-
     for_universe: ty::UniverseIndex,
 
     /// The vid of the const variable that is in the process of being
@@ -810,61 +868,15 @@ struct ConstInferUnifier<'cx, 'tcx> {
     target_vid: ty::ConstVid<'tcx>,
 }
 
-// We use `TypeRelation` here to propagate `RelateResult` upwards.
-//
-// Both inputs are expected to be the same.
-impl<'tcx> TypeRelation<'tcx> for ConstInferUnifier<'_, 'tcx> {
-    fn tcx(&self) -> TyCtxt<'tcx> {
-        self.infcx.tcx
-    }
-
-    fn intercrate(&self) -> bool {
-        assert!(!self.infcx.intercrate);
-        false
-    }
-
-    fn param_env(&self) -> ty::ParamEnv<'tcx> {
-        self.param_env
-    }
-
-    fn tag(&self) -> &'static str {
-        "ConstInferUnifier"
-    }
-
-    fn a_is_expected(&self) -> bool {
-        true
-    }
-
-    fn mark_ambiguous(&mut self) {
-        bug!()
-    }
+impl<'tcx> FallibleTypeFolder<TyCtxt<'tcx>> for ConstInferUnifier<'_, 'tcx> {
+    type Error = TypeError<'tcx>;
 
-    fn relate_with_variance<T: Relate<'tcx>>(
-        &mut self,
-        _variance: ty::Variance,
-        _info: ty::VarianceDiagInfo<'tcx>,
-        a: T,
-        b: T,
-    ) -> RelateResult<'tcx, T> {
-        // We don't care about variance here.
-        self.relate(a, b)
-    }
-
-    fn binders<T>(
-        &mut self,
-        a: ty::Binder<'tcx, T>,
-        b: ty::Binder<'tcx, T>,
-    ) -> RelateResult<'tcx, ty::Binder<'tcx, T>>
-    where
-        T: Relate<'tcx>,
-    {
-        Ok(a.rebind(self.relate(a.skip_binder(), b.skip_binder())?))
+    fn interner(&self) -> TyCtxt<'tcx> {
+        self.infcx.tcx
     }
 
     #[instrument(level = "debug", skip(self), ret)]
-    fn tys(&mut self, t: Ty<'tcx>, _t: Ty<'tcx>) -> RelateResult<'tcx, Ty<'tcx>> {
-        debug_assert_eq!(t, _t);
-
+    fn try_fold_ty(&mut self, t: Ty<'tcx>) -> Result<Ty<'tcx>, TypeError<'tcx>> {
         match t.kind() {
             &ty::Infer(ty::TyVar(vid)) => {
                 let vid = self.infcx.inner.borrow_mut().type_variables().root_var(vid);
@@ -872,7 +884,7 @@ impl<'tcx> TypeRelation<'tcx> for ConstInferUnifier<'_, 'tcx> {
                 match probe {
                     TypeVariableValue::Known { value: u } => {
                         debug!("ConstOccursChecker: known value {:?}", u);
-                        self.tys(u, u)
+                        u.try_fold_with(self)
                     }
                     TypeVariableValue::Unknown { universe } => {
                         if self.for_universe.can_name(universe) {
@@ -887,27 +899,26 @@ impl<'tcx> TypeRelation<'tcx> for ConstInferUnifier<'_, 'tcx> {
                             .borrow_mut()
                             .type_variables()
                             .new_var(self.for_universe, origin);
-                        Ok(self.tcx().mk_ty_var(new_var_id))
+                        Ok(self.interner().mk_ty_var(new_var_id))
                     }
                 }
             }
             ty::Infer(ty::IntVar(_) | ty::FloatVar(_)) => Ok(t),
-            _ => relate::super_relate_tys(self, t, t),
+            _ => t.try_super_fold_with(self),
         }
     }
 
-    fn regions(
+    #[instrument(level = "debug", skip(self), ret)]
+    fn try_fold_region(
         &mut self,
         r: ty::Region<'tcx>,
-        _r: ty::Region<'tcx>,
-    ) -> RelateResult<'tcx, ty::Region<'tcx>> {
-        debug_assert_eq!(r, _r);
+    ) -> Result<ty::Region<'tcx>, TypeError<'tcx>> {
         debug!("ConstInferUnifier: r={:?}", r);
 
         match *r {
             // Never make variables for regions bound within the type itself,
             // nor for erased regions.
-            ty::ReLateBound(..) | ty::ReErased => {
+            ty::ReLateBound(..) | ty::ReErased | ty::ReError(_) => {
                 return Ok(r);
             }
 
@@ -930,14 +941,8 @@ impl<'tcx> TypeRelation<'tcx> for ConstInferUnifier<'_, 'tcx> {
         }
     }
 
-    #[instrument(level = "debug", skip(self))]
-    fn consts(
-        &mut self,
-        c: ty::Const<'tcx>,
-        _c: ty::Const<'tcx>,
-    ) -> RelateResult<'tcx, ty::Const<'tcx>> {
-        debug_assert_eq!(c, _c);
-
+    #[instrument(level = "debug", skip(self), ret)]
+    fn try_fold_const(&mut self, c: ty::Const<'tcx>) -> Result<ty::Const<'tcx>, TypeError<'tcx>> {
         match c.kind() {
             ty::ConstKind::Infer(InferConst::Var(vid)) => {
                 // Check if the current unification would end up
@@ -958,7 +963,7 @@ impl<'tcx> TypeRelation<'tcx> for ConstInferUnifier<'_, 'tcx> {
                 let var_value =
                     self.infcx.inner.borrow_mut().const_unification_table().probe_value(vid);
                 match var_value.val {
-                    ConstVariableValue::Known { value: u } => self.consts(u, u),
+                    ConstVariableValue::Known { value: u } => u.try_fold_with(self),
                     ConstVariableValue::Unknown { universe } => {
                         if self.for_universe.can_name(universe) {
                             Ok(c)
@@ -972,22 +977,12 @@ impl<'tcx> TypeRelation<'tcx> for ConstInferUnifier<'_, 'tcx> {
                                         },
                                     },
                                 );
-                            Ok(self.tcx().mk_const(new_var_id, c.ty()))
+                            Ok(self.interner().mk_const(new_var_id, c.ty()))
                         }
                     }
                 }
             }
-            ty::ConstKind::Unevaluated(ty::UnevaluatedConst { def, substs }) => {
-                let substs = self.relate_with_variance(
-                    ty::Variance::Invariant,
-                    ty::VarianceDiagInfo::default(),
-                    substs,
-                    substs,
-                )?;
-
-                Ok(self.tcx().mk_const(ty::UnevaluatedConst { def, substs }, c.ty()))
-            }
-            _ => relate::super_relate_consts(self, c, c),
+            _ => c.try_super_fold_with(self),
         }
     }
 }
diff --git a/compiler/rustc_infer/src/infer/equate.rs b/compiler/rustc_infer/src/infer/equate.rs
index 46e7813d9..54a62326e 100644
--- a/compiler/rustc_infer/src/infer/equate.rs
+++ b/compiler/rustc_infer/src/infer/equate.rs
@@ -1,10 +1,12 @@
-use super::combine::{CombineFields, ConstEquateRelation, RelationDir};
+use crate::traits::PredicateObligations;
+
+use super::combine::{CombineFields, ObligationEmittingRelation, RelationDir};
 use super::Subtype;
 
 use rustc_middle::ty::relate::{self, Relate, RelateResult, TypeRelation};
 use rustc_middle::ty::subst::SubstsRef;
 use rustc_middle::ty::TyVar;
-use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitableExt};
 
 use rustc_hir::def_id::DefId;
 
@@ -129,7 +131,7 @@ impl<'tcx> TypeRelation<'tcx> for Equate<'_, '_, 'tcx> {
                 let a_types = infcx.tcx.anonymize_bound_vars(a_types);
                 let b_types = infcx.tcx.anonymize_bound_vars(b_types);
                 if a_types.bound_vars() == b_types.bound_vars() {
-                    let (a_types, b_types) = infcx.replace_bound_vars_with_placeholders(
+                    let (a_types, b_types) = infcx.instantiate_binder_with_placeholders(
                         a_types.map_bound(|a_types| (a_types, b_types.skip_binder())),
                     );
                     for (a, b) in std::iter::zip(a_types, b_types) {
@@ -198,8 +200,12 @@ impl<'tcx> TypeRelation<'tcx> for Equate<'_, '_, 'tcx> {
     }
 }
 
-impl<'tcx> ConstEquateRelation<'tcx> for Equate<'_, '_, 'tcx> {
-    fn const_equate_obligation(&mut self, a: ty::Const<'tcx>, b: ty::Const<'tcx>) {
-        self.fields.add_const_equate_obligation(self.a_is_expected, a, b);
+impl<'tcx> ObligationEmittingRelation<'tcx> for Equate<'_, '_, 'tcx> {
+    fn register_predicates(&mut self, obligations: impl IntoIterator<Item: ty::ToPredicate<'tcx>>) {
+        self.fields.register_predicates(obligations);
+    }
+
+    fn register_obligations(&mut self, obligations: PredicateObligations<'tcx>) {
+        self.fields.register_obligations(obligations);
     }
 }
diff --git a/compiler/rustc_infer/src/infer/error_reporting/mod.rs b/compiler/rustc_infer/src/infer/error_reporting/mod.rs
index 28fd03b87..8a2b800af 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/mod.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/mod.rs
@@ -49,6 +49,7 @@ use super::lexical_region_resolve::RegionResolutionError;
 use super::region_constraints::GenericKind;
 use super::{InferCtxt, RegionVariableOrigin, SubregionOrigin, TypeTrace, ValuePairs};
 
+use crate::errors;
 use crate::infer;
 use crate::infer::error_reporting::nice_region_error::find_anon_type::find_anon_type;
 use crate::infer::ExpectedFound;
@@ -60,17 +61,19 @@ use crate::traits::{
 
 use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_errors::{pluralize, struct_span_err, Diagnostic, ErrorGuaranteed, IntoDiagnosticArg};
-use rustc_errors::{Applicability, DiagnosticBuilder, DiagnosticStyledString, MultiSpan};
+use rustc_errors::{Applicability, DiagnosticBuilder, DiagnosticStyledString};
 use rustc_hir as hir;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId};
+use rustc_hir::intravisit::Visitor;
 use rustc_hir::lang_items::LangItem;
 use rustc_hir::Node;
 use rustc_middle::dep_graph::DepContext;
+use rustc_middle::ty::print::with_forced_trimmed_paths;
 use rustc_middle::ty::relate::{self, RelateResult, TypeRelation};
 use rustc_middle::ty::{
     self, error::TypeError, List, Region, Ty, TyCtxt, TypeFoldable, TypeSuperVisitable,
-    TypeVisitable,
+    TypeVisitable, TypeVisitableExt,
 };
 use rustc_span::{sym, symbol::kw, BytePos, DesugaringKind, Pos, Span};
 use rustc_target::spec::abi;
@@ -79,6 +82,7 @@ use std::path::PathBuf;
 use std::{cmp, fmt, iter};
 
 mod note;
+mod note_and_explain;
 mod suggest;
 
 pub(crate) mod need_type_info;
@@ -126,19 +130,16 @@ pub(super) fn note_and_explain_region<'tcx>(
     alt_span: Option<Span>,
 ) {
     let (description, span) = match *region {
-        ty::ReEarlyBound(_) | ty::ReFree(_) | ty::ReStatic => {
-            msg_span_from_free_region(tcx, region, alt_span)
+        ty::ReEarlyBound(_) | ty::ReFree(_) | ty::RePlaceholder(_) | ty::ReStatic => {
+            msg_span_from_named_region(tcx, region, alt_span)
         }
 
-        ty::RePlaceholder(_) => return,
+        ty::ReError(_) => return,
 
-        // FIXME(#13998) RePlaceholder should probably print like
-        // ReFree rather than dumping Debug output on the user.
-        //
         // We shouldn't really be having unification failures with ReVar
         // and ReLateBound though.
         ty::ReVar(_) | ty::ReLateBound(..) | ty::ReErased => {
-            (format!("lifetime {:?}", region), alt_span)
+            (format!("lifetime `{region}`"), alt_span)
         }
     };
 
@@ -152,12 +153,12 @@ fn explain_free_region<'tcx>(
     region: ty::Region<'tcx>,
     suffix: &str,
 ) {
-    let (description, span) = msg_span_from_free_region(tcx, region, None);
+    let (description, span) = msg_span_from_named_region(tcx, region, None);
 
     label_msg_span(err, prefix, description, span, suffix);
 }
 
-fn msg_span_from_free_region<'tcx>(
+fn msg_span_from_named_region<'tcx>(
     tcx: TyCtxt<'tcx>,
     region: ty::Region<'tcx>,
     alt_span: Option<Span>,
@@ -168,6 +169,18 @@ fn msg_span_from_free_region<'tcx>(
             (msg, Some(span))
         }
         ty::ReStatic => ("the static lifetime".to_owned(), alt_span),
+        ty::RePlaceholder(ty::PlaceholderRegion {
+            name: ty::BoundRegionKind::BrNamed(def_id, name),
+            ..
+        }) => (format!("the lifetime `{name}` as defined here"), Some(tcx.def_span(def_id))),
+        ty::RePlaceholder(ty::PlaceholderRegion {
+            name: ty::BoundRegionKind::BrAnon(_, Some(span)),
+            ..
+        }) => (format!("the anonymous lifetime defined here"), Some(span)),
+        ty::RePlaceholder(ty::PlaceholderRegion {
+            name: ty::BoundRegionKind::BrAnon(_, None),
+            ..
+        }) => (format!("an anonymous lifetime"), None),
         _ => bug!("{:?}", region),
     }
 }
@@ -269,15 +282,13 @@ pub fn unexpected_hidden_region_diagnostic<'tcx>(
     span: Span,
     hidden_ty: Ty<'tcx>,
     hidden_region: ty::Region<'tcx>,
-    opaque_ty: ty::OpaqueTypeKey<'tcx>,
+    opaque_ty_key: ty::OpaqueTypeKey<'tcx>,
 ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
-    let opaque_ty = tcx.mk_opaque(opaque_ty.def_id.to_def_id(), opaque_ty.substs);
-    let mut err = struct_span_err!(
-        tcx.sess,
+    let mut err = tcx.sess.create_err(errors::OpaqueCapturesLifetime {
         span,
-        E0700,
-        "hidden type for `{opaque_ty}` captures lifetime that does not appear in bounds",
-    );
+        opaque_ty: tcx.mk_opaque(opaque_ty_key.def_id.to_def_id(), opaque_ty_key.substs),
+        opaque_ty_span: tcx.def_span(opaque_ty_key.def_id),
+    });
 
     // Explain the region we are capturing.
     match *hidden_region {
@@ -311,6 +322,9 @@ pub fn unexpected_hidden_region_diagnostic<'tcx>(
                 )
             }
         }
+        ty::ReError(_) => {
+            err.delay_as_bug();
+        }
         _ => {
             // Ugh. This is a painful case: the hidden region is not one
             // that we can easily summarize or explain. This can happen
@@ -743,15 +757,16 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     };
                     let msg = "`match` arms have incompatible types";
                     err.span_label(outer, msg);
-                    self.suggest_remove_semi_or_return_binding(
-                        err,
+                    if let Some(subdiag) = self.suggest_remove_semi_or_return_binding(
                         prior_arm_block_id,
                         prior_arm_ty,
                         prior_arm_span,
                         arm_block_id,
                         arm_ty,
                         arm_span,
-                    );
+                    ) {
+                        err.subdiagnostic(subdiag);
+                    }
                     if let Some(ret_sp) = opt_suggest_box_span {
                         // Get return type span and point to it.
                         self.suggest_boxing_for_return_impl_trait(
@@ -776,15 +791,16 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 if let Some(sp) = outer_span {
                     err.span_label(sp, "`if` and `else` have incompatible types");
                 }
-                self.suggest_remove_semi_or_return_binding(
-                    err,
+                if let Some(subdiag) = self.suggest_remove_semi_or_return_binding(
                     Some(then_id),
                     then_ty,
                     then_span,
                     Some(else_id),
                     else_ty,
                     else_span,
-                );
+                ) {
+                    err.subdiagnostic(subdiag);
+                }
                 if let Some(ret_sp) = opt_suggest_box_span {
                     self.suggest_boxing_for_return_impl_trait(
                         err,
@@ -908,7 +924,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
     ) -> Option<()> {
         // FIXME/HACK: Go back to `SubstsRef` to use its inherent methods,
         // ideally that shouldn't be necessary.
-        let sub = self.tcx.intern_substs(sub);
+        let sub = self.tcx.mk_substs(sub);
         for (i, ta) in sub.types().enumerate() {
             if ta == other_ty {
                 self.highlight_outer(&mut t1_out, &mut t2_out, path, sub, i, other_ty);
@@ -1344,8 +1360,8 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
             }
 
             (ty::FnDef(did1, substs1), ty::FnDef(did2, substs2)) => {
-                let sig1 = self.tcx.bound_fn_sig(*did1).subst(self.tcx, substs1);
-                let sig2 = self.tcx.bound_fn_sig(*did2).subst(self.tcx, substs2);
+                let sig1 = self.tcx.fn_sig(*did1).subst(self.tcx, substs1);
+                let sig2 = self.tcx.fn_sig(*did2).subst(self.tcx, substs2);
                 let mut values = self.cmp_fn_sig(&sig1, &sig2);
                 let path1 = format!(" {{{}}}", self.tcx.def_path_str_with_substs(*did1, substs1));
                 let path2 = format!(" {{{}}}", self.tcx.def_path_str_with_substs(*did2, substs2));
@@ -1356,7 +1372,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
             }
 
             (ty::FnDef(did1, substs1), ty::FnPtr(sig2)) => {
-                let sig1 = self.tcx.bound_fn_sig(*did1).subst(self.tcx, substs1);
+                let sig1 = self.tcx.fn_sig(*did1).subst(self.tcx, substs1);
                 let mut values = self.cmp_fn_sig(&sig1, sig2);
                 values.0.push_highlighted(format!(
                     " {{{}}}",
@@ -1366,7 +1382,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
             }
 
             (ty::FnPtr(sig1), ty::FnDef(did2, substs2)) => {
-                let sig2 = self.tcx.bound_fn_sig(*did2).subst(self.tcx, substs2);
+                let sig2 = self.tcx.fn_sig(*did2).subst(self.tcx, substs2);
                 let mut values = self.cmp_fn_sig(sig1, &sig2);
                 values.1.push_normal(format!(
                     " {{{}}}",
@@ -1433,8 +1449,8 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         impl<'tcx> OpaqueTypesVisitor<'tcx> {
             fn visit_expected_found(
                 tcx: TyCtxt<'tcx>,
-                expected: impl TypeVisitable<'tcx>,
-                found: impl TypeVisitable<'tcx>,
+                expected: impl TypeVisitable<TyCtxt<'tcx>>,
+                found: impl TypeVisitable<TyCtxt<'tcx>>,
                 ignore_span: Span,
             ) -> Self {
                 let mut types_visitor = OpaqueTypesVisitor {
@@ -1468,57 +1484,23 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 for (key, values) in types.iter() {
                     let count = values.len();
                     let kind = key.descr();
-                    let mut returned_async_output_error = false;
                     for &sp in values {
-                        if sp.is_desugaring(DesugaringKind::Async) && !returned_async_output_error {
-                            if [sp] != err.span.primary_spans() {
-                                let mut span: MultiSpan = sp.into();
-                                span.push_span_label(
-                                    sp,
-                                    format!(
-                                        "checked the `Output` of this `async fn`, {}{} {}{}",
-                                        if count > 1 { "one of the " } else { "" },
-                                        target,
-                                        kind,
-                                        pluralize!(count),
-                                    ),
-                                );
-                                err.span_note(
-                                    span,
-                                    "while checking the return type of the `async fn`",
-                                );
-                            } else {
-                                err.span_label(
-                                    sp,
-                                    format!(
-                                        "checked the `Output` of this `async fn`, {}{} {}{}",
-                                        if count > 1 { "one of the " } else { "" },
-                                        target,
-                                        kind,
-                                        pluralize!(count),
-                                    ),
-                                );
-                                err.note("while checking the return type of the `async fn`");
-                            }
-                            returned_async_output_error = true;
-                        } else {
-                            err.span_label(
-                                sp,
-                                format!(
-                                    "{}{} {}{}",
-                                    if count == 1 { "the " } else { "one of the " },
-                                    target,
-                                    kind,
-                                    pluralize!(count),
-                                ),
-                            );
-                        }
+                        err.span_label(
+                            sp,
+                            format!(
+                                "{}{} {}{}",
+                                if count == 1 { "the " } else { "one of the " },
+                                target,
+                                kind,
+                                pluralize!(count),
+                            ),
+                        );
                     }
                 }
             }
         }
 
-        impl<'tcx> ty::visit::TypeVisitor<'tcx> for OpaqueTypesVisitor<'tcx> {
+        impl<'tcx> ty::visit::TypeVisitor<TyCtxt<'tcx>> for OpaqueTypesVisitor<'tcx> {
             fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
                 if let Some((kind, def_id)) = TyCategory::from_ty(self.tcx, t) {
                     let span = self.tcx.def_span(def_id);
@@ -1535,7 +1517,11 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     //    |
                     //    = note: expected unit type `()`
                     //                 found closure `[closure@$DIR/issue-20862.rs:2:5: 2:14 x:_]`
-                    if !self.ignore_span.overlaps(span) {
+                    //
+                    // Also ignore opaque `Future`s that come from async fns.
+                    if !self.ignore_span.overlaps(span)
+                        && !span.is_desugaring(DesugaringKind::Async)
+                    {
                         self.types.entry(kind).or_default().insert(span);
                     }
                 }
@@ -1611,16 +1597,31 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 {
                     format!("expected this to be `{}`", expected)
                 } else {
-                    terr.to_string()
+                    terr.to_string(self.tcx).to_string()
                 };
                 label_or_note(sp, &terr);
                 label_or_note(span, &msg);
             } else {
-                label_or_note(span, &terr.to_string());
+                label_or_note(span, &terr.to_string(self.tcx));
                 label_or_note(sp, &msg);
             }
         } else {
-            label_or_note(span, &terr.to_string());
+            if let Some(values) = values
+                && let Some((e, f)) = values.ty()
+                && let TypeError::ArgumentSorts(..) | TypeError::Sorts(_) = terr
+            {
+                let e = self.tcx.erase_regions(e);
+                let f = self.tcx.erase_regions(f);
+                let expected = with_forced_trimmed_paths!(e.sort_string(self.tcx));
+                let found = with_forced_trimmed_paths!(f.sort_string(self.tcx));
+                if expected == found {
+                    label_or_note(span, &terr.to_string(self.tcx));
+                } else {
+                    label_or_note(span, &format!("expected {expected}, found {found}"));
+                }
+            } else {
+                label_or_note(span, &terr.to_string(self.tcx));
+            }
         }
 
         if let Some((expected, found, exp_p, found_p)) = expected_found {
@@ -1688,7 +1689,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                                 format!("{name} is defined in the current crate")
                             } else {
                                 let crate_name = self.tcx.crate_name(defid.krate);
-                                format!("{name} is defined in crate `{crate_name}")
+                                format!("{name} is defined in crate `{crate_name}`")
                             };
                             diagnostic.span_note(def_span, msg);
                         };
@@ -1791,14 +1792,24 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                             }
                         }))
                     {
-                        diag.note_expected_found_extra(
-                            &expected_label,
-                            expected,
-                            &found_label,
-                            found,
-                            &sort_string(values.expected, exp_p),
-                            &sort_string(values.found, found_p),
-                        );
+                        if let Some(ExpectedFound { found: found_ty, .. }) = exp_found {
+                            // `Future` is a special opaque type that the compiler
+                            // will try to hide in some case such as `async fn`, so
+                            // to make an error more use friendly we will
+                            // avoid to suggest a mismatch type with a
+                            // type that the user usually are not usign
+                            // directly such as `impl Future<Output = u8>`.
+                            if !self.tcx.ty_is_opaque_future(found_ty) {
+                                diag.note_expected_found_extra(
+                                    &expected_label,
+                                    expected,
+                                    &found_label,
+                                    found,
+                                    &sort_string(values.expected, exp_p),
+                                    &sort_string(values.found, found_p),
+                                );
+                            }
+                        }
                     }
                 }
                 _ => {
@@ -1841,19 +1852,25 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 self.suggest_as_ref_where_appropriate(span, &exp_found, diag);
                 self.suggest_accessing_field_where_appropriate(cause, &exp_found, diag);
                 self.suggest_await_on_expect_found(cause, span, &exp_found, diag);
+                self.suggest_function_pointers(cause, span, &exp_found, diag);
             }
         }
 
-        // In some (most?) cases cause.body_id points to actual body, but in some cases
-        // it's an actual definition. According to the comments (e.g. in
-        // rustc_hir_analysis/check/compare_impl_item.rs:compare_predicate_entailment) the latter
-        // is relied upon by some other code. This might (or might not) need cleanup.
-        let body_owner_def_id =
-            self.tcx.hir().opt_local_def_id(cause.body_id).unwrap_or_else(|| {
-                self.tcx.hir().body_owner_def_id(hir::BodyId { hir_id: cause.body_id })
-            });
         self.check_and_note_conflicting_crates(diag, terr);
-        self.tcx.note_and_explain_type_err(diag, terr, cause, span, body_owner_def_id.to_def_id());
+
+        self.note_and_explain_type_err(diag, terr, cause, span, cause.body_id.to_def_id());
+        if let Some(exp_found) = exp_found
+            && let exp_found = TypeError::Sorts(exp_found)
+            && exp_found != terr
+        {
+            self.note_and_explain_type_err(
+                diag,
+                exp_found,
+                cause,
+                span,
+                cause.body_id.to_def_id(),
+            );
+        }
 
         if let Some(ValuePairs::PolyTraitRefs(exp_found)) = values
             && let ty::Closure(def_id, _) = exp_found.expected.skip_binder().self_ty().kind()
@@ -1929,7 +1946,8 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                         (ty::Uint(ty::UintTy::U8), ty::Char) => {
                             if let Ok(code) = self.tcx.sess().source_map().span_to_snippet(span)
                                 && let Some(code) = code.strip_prefix('\'').and_then(|s| s.strip_suffix('\''))
-                                && code.chars().next().map_or(false, |c| c.is_ascii())
+                                && !code.starts_with("\\u") // forbid all Unicode escapes
+                                && code.chars().next().map_or(false, |c| c.is_ascii()) // forbids literal Unicode characters beyond ASCII
                             {
                                 err.span_suggestion(
                                     span,
@@ -1976,6 +1994,70 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                         (ty::Bool, ty::Tuple(list)) => if list.len() == 0 {
                             self.suggest_let_for_letchains(&mut err, &trace.cause, span);
                         }
+                        (ty::Array(_, _), ty::Array(_, _)) => 'block: {
+                            let hir = self.tcx.hir();
+                            let TypeError::FixedArraySize(sz) = terr else {
+                                break 'block;
+                            };
+                            let tykind = match hir.find_by_def_id(trace.cause.body_id) {
+                                Some(hir::Node::Item(hir::Item {
+                                    kind: hir::ItemKind::Fn(_, _, body_id),
+                                    ..
+                                })) => {
+                                    let body = hir.body(*body_id);
+                                    struct LetVisitor<'v> {
+                                        span: Span,
+                                        result: Option<&'v hir::Ty<'v>>,
+                                    }
+                                    impl<'v> Visitor<'v> for LetVisitor<'v> {
+                                        fn visit_stmt(&mut self, s: &'v hir::Stmt<'v>) {
+                                            if self.result.is_some() {
+                                                return;
+                                            }
+                                            // Find a local statement where the initializer has
+                                            // the same span as the error and the type is specified.
+                                            if let hir::Stmt {
+                                                kind: hir::StmtKind::Local(hir::Local {
+                                                    init: Some(hir::Expr {
+                                                        span: init_span,
+                                                        ..
+                                                    }),
+                                                    ty: Some(array_ty),
+                                                    ..
+                                                }),
+                                                ..
+                                            } = s
+                                            && init_span == &self.span {
+                                                self.result = Some(*array_ty);
+                                            }
+                                        }
+                                    }
+                                    let mut visitor = LetVisitor {span, result: None};
+                                    visitor.visit_body(body);
+                                    visitor.result.map(|r| &r.peel_refs().kind)
+                                }
+                                Some(hir::Node::Item(hir::Item {
+                                    kind: hir::ItemKind::Const(ty, _),
+                                    ..
+                                })) => {
+                                    Some(&ty.peel_refs().kind)
+                                }
+                                _ => None
+                            };
+
+                            if let Some(tykind) = tykind
+                                && let hir::TyKind::Array(_, length) = tykind
+                                && let hir::ArrayLen::Body(hir::AnonConst { hir_id, .. }) = length
+                                && let Some(span) = self.tcx.hir().opt_span(*hir_id)
+                            {
+                                err.span_suggestion(
+                                    span,
+                                    "consider specifying the actual array length",
+                                    sz.found,
+                                    Applicability::MaybeIncorrect,
+                                );
+                            }
+                        }
                         _ => {}
                     }
                 }
@@ -2123,7 +2205,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
     }
 
     /// Returns a string of the form "expected `{}`, found `{}`".
-    fn expected_found_str<T: fmt::Display + TypeFoldable<'tcx>>(
+    fn expected_found_str<T: fmt::Display + TypeFoldable<TyCtxt<'tcx>>>(
         &self,
         exp_found: ty::error::ExpectedFound<T>,
     ) -> Option<(DiagnosticStyledString, DiagnosticStyledString, Option<PathBuf>, Option<PathBuf>)>
@@ -2552,7 +2634,11 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
             );
 
             err.note_expected_found(&"", sup_expected, &"", sup_found);
-            err.emit();
+            if sub_region.is_error() | sup_region.is_error() {
+                err.delay_as_bug();
+            } else {
+                err.emit();
+            }
             return;
         }
 
@@ -2568,7 +2654,11 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         );
 
         self.note_region_origin(&mut err, &sub_origin);
-        err.emit();
+        if sub_region.is_error() | sup_region.is_error() {
+            err.delay_as_bug();
+        } else {
+            err.emit();
+        }
     }
 
     /// Determine whether an error associated with the given span and definition
@@ -2585,7 +2675,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
     /// with the other type. A TyVar inference type is compatible with any type, and an IntVar or
     /// FloatVar inference type are compatible with themselves or their concrete types (Int and
     /// Float types, respectively). When comparing two ADTs, these rules apply recursively.
-    pub fn same_type_modulo_infer(&self, a: Ty<'tcx>, b: Ty<'tcx>) -> bool {
+    pub fn same_type_modulo_infer<T: relate::Relate<'tcx>>(&self, a: T, b: T) -> bool {
         let (a, b) = self.resolve_vars_if_possible((a, b));
         SameTypeModuloInfer(self).relate(a, b).is_ok()
     }
@@ -2847,6 +2937,7 @@ impl IntoDiagnosticArg for ObligationCauseAsDiagArg<'_> {
 pub enum TyCategory {
     Closure,
     Opaque,
+    OpaqueFuture,
     Generator(hir::GeneratorKind),
     Foreign,
 }
@@ -2856,6 +2947,7 @@ impl TyCategory {
         match self {
             Self::Closure => "closure",
             Self::Opaque => "opaque type",
+            Self::OpaqueFuture => "future",
             Self::Generator(gk) => gk.descr(),
             Self::Foreign => "foreign type",
         }
@@ -2864,7 +2956,11 @@ impl TyCategory {
     pub fn from_ty(tcx: TyCtxt<'_>, ty: Ty<'_>) -> Option<(Self, DefId)> {
         match *ty.kind() {
             ty::Closure(def_id, _) => Some((Self::Closure, def_id)),
-            ty::Alias(ty::Opaque, ty::AliasTy { def_id, .. }) => Some((Self::Opaque, def_id)),
+            ty::Alias(ty::Opaque, ty::AliasTy { def_id, .. }) => {
+                let kind =
+                    if tcx.ty_is_opaque_future(ty) { Self::OpaqueFuture } else { Self::Opaque };
+                Some((kind, def_id))
+            }
             ty::Generator(def_id, ..) => {
                 Some((Self::Generator(tcx.generator_kind(def_id).unwrap()), def_id))
             }
diff --git a/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs b/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs
index b8c843a8a..e242900fd 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs
@@ -122,7 +122,7 @@ impl InferenceDiagnosticsParentData {
             tcx.def_key(parent_def_id).disambiguated_data.data.get_opt_name()?.to_string();
 
         Some(InferenceDiagnosticsParentData {
-            prefix: tcx.def_kind(parent_def_id).descr(parent_def_id),
+            prefix: tcx.def_descr(parent_def_id),
             name: parent_name,
         })
     }
@@ -158,8 +158,12 @@ fn fmt_printer<'a, 'tcx>(infcx: &'a InferCtxt<'tcx>, ns: Namespace) -> FmtPrinte
         if infcx.probe_ty_var(ty_vid).is_ok() {
             warn!("resolved ty var in error message");
         }
-        if let TypeVariableOriginKind::TypeParameterDefinition(name, _) =
-            infcx.inner.borrow_mut().type_variables().var_origin(ty_vid).kind
+
+        let mut infcx_inner = infcx.inner.borrow_mut();
+        let ty_vars = infcx_inner.type_variables();
+        let var_origin = ty_vars.var_origin(ty_vid);
+        if let TypeVariableOriginKind::TypeParameterDefinition(name, _) = var_origin.kind
+            && !var_origin.span.from_expansion()
         {
             Some(name)
         } else {
@@ -254,7 +258,7 @@ impl<'tcx> InferCtxt<'tcx> {
                     if let TypeVariableOriginKind::TypeParameterDefinition(name, def_id) =
                         var_origin.kind
                     {
-                        if name != kw::SelfUpper {
+                        if name != kw::SelfUpper && !var_origin.span.from_expansion() {
                             return InferenceDiagnosticsData {
                                 name: name.to_string(),
                                 span: Some(var_origin.span),
@@ -780,7 +784,7 @@ impl<'a, 'tcx> FindInferSourceVisitor<'a, 'tcx> {
         // The sources are listed in order of preference here.
         let tcx = self.infcx.tcx;
         let ctx = CostCtxt { tcx };
-        let base_cost = match source.kind {
+        match source.kind {
             InferSourceKind::LetBinding { ty, .. } => ctx.ty_cost(ty),
             InferSourceKind::ClosureArg { ty, .. } => ctx.ty_cost(ty),
             InferSourceKind::GenericArg { def_id, generic_args, .. } => {
@@ -797,17 +801,17 @@ impl<'a, 'tcx> FindInferSourceVisitor<'a, 'tcx> {
             InferSourceKind::ClosureReturn { ty, should_wrap_expr, .. } => {
                 30 + ctx.ty_cost(ty) + if should_wrap_expr.is_some() { 10 } else { 0 }
             }
-        };
-
-        let suggestion_may_apply = if source.from_expansion() { 10000 } else { 0 };
-
-        base_cost + suggestion_may_apply
+        }
     }
 
     /// Uses `fn source_cost` to determine whether this inference source is preferable to
     /// previous sources. We generally prefer earlier sources.
     #[instrument(level = "debug", skip(self))]
     fn update_infer_source(&mut self, mut new_source: InferSource<'tcx>) {
+        if new_source.from_expansion() {
+            return;
+        }
+
         let cost = self.source_cost(&new_source) + self.attempt;
         debug!(?cost);
         self.attempt += 1;
@@ -819,6 +823,7 @@ impl<'a, 'tcx> FindInferSourceVisitor<'a, 'tcx> {
             // `let x: _ = iter.collect();`, as this is a very common case.
             *def_id = Some(did);
         }
+
         if cost < self.infer_source_cost {
             self.infer_source_cost = cost;
             self.infer_source = Some(new_source);
@@ -1056,8 +1061,8 @@ impl<'a, 'tcx> FindInferSourceVisitor<'a, 'tcx> {
                 };
 
                 let parent_def_id = generics.parent.unwrap();
-                if tcx.def_kind(parent_def_id) == DefKind::Impl {
-                    let parent_ty = tcx.bound_type_of(parent_def_id).subst(tcx, substs);
+                if let DefKind::Impl { .. } = tcx.def_kind(parent_def_id) {
+                    let parent_ty = tcx.type_of(parent_def_id).subst(tcx, substs);
                     match (parent_ty.kind(), &ty.kind) {
                         (
                             ty::Adt(def, substs),
diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/find_anon_type.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/find_anon_type.rs
index 39f4d5022..fec04af23 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/find_anon_type.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/find_anon_type.rs
@@ -2,7 +2,7 @@ use rustc_hir as hir;
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_middle::hir::map::Map;
 use rustc_middle::hir::nested_filter;
-use rustc_middle::middle::resolve_lifetime as rl;
+use rustc_middle::middle::resolve_bound_vars as rbv;
 use rustc_middle::ty::{self, Region, TyCtxt};
 
 /// This function calls the `visit_ty` method for the parameters
@@ -99,11 +99,11 @@ impl<'tcx> Visitor<'tcx> for FindNestedTypeVisitor<'tcx> {
             hir::TyKind::Ref(ref lifetime, _) => {
                 // the lifetime of the Ref
                 let hir_id = lifetime.hir_id;
-                match (self.tcx.named_region(hir_id), self.bound_region) {
+                match (self.tcx.named_bound_var(hir_id), self.bound_region) {
                     // Find the index of the named region that was part of the
                     // error. We will then search the function parameters for a bound
                     // region at the right depth with the same index
-                    (Some(rl::Region::EarlyBound(id)), ty::BrNamed(def_id, _)) => {
+                    (Some(rbv::ResolvedArg::EarlyBound(id)), ty::BrNamed(def_id, _)) => {
                         debug!("EarlyBound id={:?} def_id={:?}", id, def_id);
                         if id == def_id {
                             self.found_type = Some(arg);
@@ -115,7 +115,7 @@ impl<'tcx> Visitor<'tcx> for FindNestedTypeVisitor<'tcx> {
                     // error. We will then search the function parameters for a bound
                     // region at the right depth with the same index
                     (
-                        Some(rl::Region::LateBound(debruijn_index, _, id)),
+                        Some(rbv::ResolvedArg::LateBound(debruijn_index, _, id)),
                         ty::BrNamed(def_id, _),
                     ) => {
                         debug!(
@@ -131,10 +131,11 @@ impl<'tcx> Visitor<'tcx> for FindNestedTypeVisitor<'tcx> {
 
                     (
                         Some(
-                            rl::Region::Static
-                            | rl::Region::Free(_, _)
-                            | rl::Region::EarlyBound(_)
-                            | rl::Region::LateBound(_, _, _),
+                            rbv::ResolvedArg::StaticLifetime
+                            | rbv::ResolvedArg::Free(_, _)
+                            | rbv::ResolvedArg::EarlyBound(_)
+                            | rbv::ResolvedArg::LateBound(_, _, _)
+                            | rbv::ResolvedArg::Error(_),
                         )
                         | None,
                         _,
@@ -186,9 +187,9 @@ impl<'tcx> Visitor<'tcx> for TyPathVisitor<'tcx> {
     }
 
     fn visit_lifetime(&mut self, lifetime: &hir::Lifetime) {
-        match (self.tcx.named_region(lifetime.hir_id), self.bound_region) {
+        match (self.tcx.named_bound_var(lifetime.hir_id), self.bound_region) {
             // the lifetime of the TyPath!
-            (Some(rl::Region::EarlyBound(id)), ty::BrNamed(def_id, _)) => {
+            (Some(rbv::ResolvedArg::EarlyBound(id)), ty::BrNamed(def_id, _)) => {
                 debug!("EarlyBound id={:?} def_id={:?}", id, def_id);
                 if id == def_id {
                     self.found_it = true;
@@ -196,7 +197,7 @@ impl<'tcx> Visitor<'tcx> for TyPathVisitor<'tcx> {
                 }
             }
 
-            (Some(rl::Region::LateBound(debruijn_index, _, id)), ty::BrNamed(def_id, _)) => {
+            (Some(rbv::ResolvedArg::LateBound(debruijn_index, _, id)), ty::BrNamed(def_id, _)) => {
                 debug!("FindNestedTypeVisitor::visit_ty: LateBound depth = {:?}", debruijn_index,);
                 debug!("id={:?}", id);
                 debug!("def_id={:?}", def_id);
@@ -208,10 +209,11 @@ impl<'tcx> Visitor<'tcx> for TyPathVisitor<'tcx> {
 
             (
                 Some(
-                    rl::Region::Static
-                    | rl::Region::EarlyBound(_)
-                    | rl::Region::LateBound(_, _, _)
-                    | rl::Region::Free(_, _),
+                    rbv::ResolvedArg::StaticLifetime
+                    | rbv::ResolvedArg::EarlyBound(_)
+                    | rbv::ResolvedArg::LateBound(_, _, _)
+                    | rbv::ResolvedArg::Free(_, _)
+                    | rbv::ResolvedArg::Error(_),
                 )
                 | None,
                 _,
diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/mismatched_static_lifetime.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/mismatched_static_lifetime.rs
index 1067ccda2..2c63a3904 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/mismatched_static_lifetime.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/mismatched_static_lifetime.rs
@@ -72,7 +72,7 @@ impl<'a, 'tcx> NiceRegionError<'a, 'tcx> {
             };
 
             // Next, let's figure out the set of trait objects with implicit static bounds
-            let ty = self.tcx().type_of(*impl_def_id);
+            let ty = self.tcx().type_of(*impl_def_id).subst_identity();
             let mut v = super::static_impl_trait::TraitObjectVisitor(FxIndexSet::default());
             v.visit_ty(ty);
             let mut traits = vec![];
diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/placeholder_error.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/placeholder_error.rs
index 99431567e..c1ea0a0d9 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/placeholder_error.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/placeholder_error.rs
@@ -14,7 +14,7 @@ use rustc_hir::def_id::DefId;
 use rustc_middle::ty::error::ExpectedFound;
 use rustc_middle::ty::print::{FmtPrinter, Print, RegionHighlightMode};
 use rustc_middle::ty::subst::SubstsRef;
-use rustc_middle::ty::{self, RePlaceholder, ReVar, Region, TyCtxt};
+use rustc_middle::ty::{self, RePlaceholder, Region, TyCtxt};
 
 use std::fmt;
 
@@ -79,7 +79,7 @@ impl<'tcx> NiceRegionError<'_, 'tcx> {
                 sup_placeholder @ Region(Interned(RePlaceholder(_), _)),
                 _,
             )) => self.try_report_trait_placeholder_mismatch(
-                Some(self.tcx().mk_region(ReVar(*vid))),
+                Some(self.tcx().mk_re_var(*vid)),
                 cause,
                 Some(*sub_placeholder),
                 Some(*sup_placeholder),
@@ -95,7 +95,7 @@ impl<'tcx> NiceRegionError<'_, 'tcx> {
                 _,
                 _,
             )) => self.try_report_trait_placeholder_mismatch(
-                Some(self.tcx().mk_region(ReVar(*vid))),
+                Some(self.tcx().mk_re_var(*vid)),
                 cause,
                 Some(*sub_placeholder),
                 None,
@@ -111,7 +111,7 @@ impl<'tcx> NiceRegionError<'_, 'tcx> {
                 sup_placeholder @ Region(Interned(RePlaceholder(_), _)),
                 _,
             )) => self.try_report_trait_placeholder_mismatch(
-                Some(self.tcx().mk_region(ReVar(*vid))),
+                Some(self.tcx().mk_re_var(*vid)),
                 cause,
                 None,
                 Some(*sup_placeholder),
@@ -127,7 +127,7 @@ impl<'tcx> NiceRegionError<'_, 'tcx> {
                 sup_placeholder @ Region(Interned(RePlaceholder(_), _)),
                 _,
             )) => self.try_report_trait_placeholder_mismatch(
-                Some(self.tcx().mk_region(ReVar(*vid))),
+                Some(self.tcx().mk_re_var(*vid)),
                 cause,
                 None,
                 Some(*sup_placeholder),
@@ -141,7 +141,7 @@ impl<'tcx> NiceRegionError<'_, 'tcx> {
                 SubregionOrigin::Subtype(box TypeTrace { cause, values }),
                 sup_placeholder @ Region(Interned(RePlaceholder(_), _)),
             )) => self.try_report_trait_placeholder_mismatch(
-                Some(self.tcx().mk_region(ReVar(*vid))),
+                Some(self.tcx().mk_re_var(*vid)),
                 cause,
                 None,
                 Some(*sup_placeholder),
diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/placeholder_relation.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/placeholder_relation.rs
index 9534bce54..e8d94f0c0 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/placeholder_relation.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/placeholder_relation.rs
@@ -1,5 +1,8 @@
-use crate::infer::{
-    error_reporting::nice_region_error::NiceRegionError, RegionResolutionError, SubregionOrigin,
+use crate::{
+    errors::PlaceholderRelationLfNotSatisfied,
+    infer::{
+        error_reporting::nice_region_error::NiceRegionError, RegionResolutionError, SubregionOrigin,
+    },
 };
 use rustc_data_structures::intern::Interned;
 use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed};
@@ -16,8 +19,7 @@ impl<'tcx> NiceRegionError<'_, 'tcx> {
                 Region(Interned(RePlaceholder(ty::Placeholder { name: sub_name, .. }), _)),
                 Region(Interned(RePlaceholder(ty::Placeholder { name: sup_name, .. }), _)),
             )) => {
-                let msg = "lifetime bound not satisfied";
-                let mut err = self.tcx().sess.struct_span_err(*span, msg);
+                let span = *span;
                 let (sub_span, sub_symbol) = match sub_name {
                     ty::BrNamed(def_id, symbol) => {
                         (Some(self.tcx().def_span(def_id)), Some(symbol))
@@ -32,41 +34,47 @@ impl<'tcx> NiceRegionError<'_, 'tcx> {
                     ty::BrAnon(_, span) => (*span, None),
                     ty::BrEnv => (None, None),
                 };
-                match (sub_span, sup_span, sub_symbol, sup_symbol) {
-                    (Some(sub_span), Some(sup_span), Some(sub_symbol), Some(sup_symbol)) => {
-                        err.span_note(
+                let diag = match (sub_span, sup_span, sub_symbol, sup_symbol) {
+                    (Some(sub_span), Some(sup_span), Some(&sub_symbol), Some(&sup_symbol)) => {
+                        PlaceholderRelationLfNotSatisfied::HasBoth {
+                            span,
                             sub_span,
-                            format!("the lifetime `{sub_symbol}` defined here..."),
-                        );
-                        err.span_note(
                             sup_span,
-                            format!("...must outlive the lifetime `{sup_symbol}` defined here"),
-                        );
+                            sub_symbol,
+                            sup_symbol,
+                            note: (),
+                        }
                     }
-                    (Some(sub_span), Some(sup_span), _, Some(sup_symbol)) => {
-                        err.span_note(sub_span, "the lifetime defined here...");
-                        err.span_note(
+                    (Some(sub_span), Some(sup_span), _, Some(&sup_symbol)) => {
+                        PlaceholderRelationLfNotSatisfied::HasSup {
+                            span,
+                            sub_span,
                             sup_span,
-                            format!("...must outlive the lifetime `{sup_symbol}` defined here"),
-                        );
+                            sup_symbol,
+                            note: (),
+                        }
                     }
-                    (Some(sub_span), Some(sup_span), Some(sub_symbol), _) => {
-                        err.span_note(
+                    (Some(sub_span), Some(sup_span), Some(&sub_symbol), _) => {
+                        PlaceholderRelationLfNotSatisfied::HasSub {
+                            span,
                             sub_span,
-                            format!("the lifetime `{sub_symbol}` defined here..."),
-                        );
-                        err.span_note(sup_span, "...must outlive the lifetime defined here");
+                            sup_span,
+                            sub_symbol,
+                            note: (),
+                        }
                     }
                     (Some(sub_span), Some(sup_span), _, _) => {
-                        err.span_note(sub_span, "the lifetime defined here...");
-                        err.span_note(sup_span, "...must outlive the lifetime defined here");
+                        PlaceholderRelationLfNotSatisfied::HasNone {
+                            span,
+                            sub_span,
+                            sup_span,
+                            note: (),
+                        }
                     }
-                    _ => {}
-                }
-                err.note("this is a known limitation that will be removed in the future (see issue #100013 <https://github.com/rust-lang/rust/issues/100013> for more information)");
-                Some(err)
+                    _ => PlaceholderRelationLfNotSatisfied::OnlyPrimarySpan { span, note: () },
+                };
+                Some(self.tcx().sess.create_err(diag))
             }
-
             _ => None,
         }
     }
diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/static_impl_trait.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/static_impl_trait.rs
index 49ad3ce50..b06ff10d8 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/static_impl_trait.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/static_impl_trait.rs
@@ -98,6 +98,7 @@ impl<'a, 'tcx> NiceRegionError<'a, 'tcx> {
         let sp = var_origin.span();
         let return_sp = sub_origin.span();
         let param = self.find_param_with_region(*sup_r, *sub_r)?;
+        let simple_ident = param.param.pat.simple_ident();
         let lifetime_name = if sup_r.has_name() { sup_r.to_string() } else { "'_".to_owned() };
 
         let (mention_influencer, influencer_point) =
@@ -187,7 +188,9 @@ impl<'a, 'tcx> NiceRegionError<'a, 'tcx> {
             req_introduces_loc: subdiag,
 
             has_lifetime: sup_r.has_name(),
-            lifetime: sup_r.to_string(),
+            lifetime: lifetime_name.clone(),
+            has_param_name: simple_ident.is_some(),
+            param_name: simple_ident.map(|x| x.to_string()).unwrap_or_default(),
             spans_empty,
             bound,
         };
@@ -536,7 +539,7 @@ impl<'a, 'tcx> NiceRegionError<'a, 'tcx> {
 /// Collect all the trait objects in a type that could have received an implicit `'static` lifetime.
 pub struct TraitObjectVisitor(pub FxIndexSet<DefId>);
 
-impl<'tcx> TypeVisitor<'tcx> for TraitObjectVisitor {
+impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for TraitObjectVisitor {
     fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
         match t.kind() {
             ty::Dynamic(preds, re, _) if re.is_static() => {
diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/trait_impl_difference.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/trait_impl_difference.rs
index 40c0c806e..2875448ee 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/trait_impl_difference.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/trait_impl_difference.rs
@@ -75,7 +75,7 @@ impl<'a, 'tcx> NiceRegionError<'a, 'tcx> {
             }
         }
 
-        impl<'tcx> ty::visit::TypeVisitor<'tcx> for HighlightBuilder<'tcx> {
+        impl<'tcx> ty::visit::TypeVisitor<TyCtxt<'tcx>> for HighlightBuilder<'tcx> {
             fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
                 if !r.has_name() && self.counter <= 3 {
                     self.highlight.highlighting_region(r, self.counter);
diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/util.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/util.rs
index fd26d7d29..db4b8af46 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/util.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/util.rs
@@ -65,7 +65,7 @@ pub fn find_param_with_region<'tcx>(
 
     let owner_id = hir.body_owner(body_id);
     let fn_decl = hir.fn_decl_by_hir_id(owner_id).unwrap();
-    let poly_fn_sig = tcx.fn_sig(id);
+    let poly_fn_sig = tcx.fn_sig(id).subst_identity();
 
     let fn_sig = tcx.liberate_late_bound_regions(id, poly_fn_sig);
     let body = hir.body(body_id);
@@ -90,20 +90,18 @@ pub fn find_param_with_region<'tcx>(
                     r
                 }
             });
-            if found_anon_region {
+            found_anon_region.then(|| {
                 let ty_hir_id = fn_decl.inputs[index].hir_id;
                 let param_ty_span = hir.span(ty_hir_id);
                 let is_first = index == 0;
-                Some(AnonymousParamInfo {
+                AnonymousParamInfo {
                     param,
                     param_ty: new_param_ty,
                     param_ty_span,
                     bound_region,
                     is_first,
-                })
-            } else {
-                None
-            }
+                }
+            })
         })
 }
 
@@ -125,7 +123,7 @@ impl<'a, 'tcx> NiceRegionError<'a, 'tcx> {
         br: ty::BoundRegionKind,
         hir_sig: &hir::FnSig<'_>,
     ) -> Option<Span> {
-        let fn_ty = self.tcx().type_of(scope_def_id);
+        let fn_ty = self.tcx().type_of(scope_def_id).subst_identity();
         if let ty::FnDef(_, _) = fn_ty.kind() {
             let ret_ty = fn_ty.fn_sig(self.tcx()).output();
             let span = hir_sig.decl.output.span();
@@ -145,7 +143,7 @@ impl<'a, 'tcx> NiceRegionError<'a, 'tcx> {
 
     fn includes_region(
         &self,
-        ty: Binder<'tcx, impl TypeVisitable<'tcx>>,
+        ty: Binder<'tcx, impl TypeVisitable<TyCtxt<'tcx>>>,
         region: ty::BoundRegionKind,
     ) -> bool {
         let late_bound_regions = self.tcx().collect_referenced_late_bound_regions(&ty);
diff --git a/compiler/rustc_infer/src/infer/error_reporting/note.rs b/compiler/rustc_infer/src/infer/error_reporting/note.rs
index b18cbd404..7ffe1fd20 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/note.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/note.rs
@@ -1,9 +1,12 @@
-use crate::errors::RegionOriginNote;
+use crate::errors::{
+    note_and_explain, FullfillReqLifetime, LfBoundNotSatisfied, OutlivesBound, OutlivesContent,
+    RefLongerThanData, RegionOriginNote, WhereClauseSuggestions,
+};
+use crate::fluent_generated as fluent;
 use crate::infer::error_reporting::{note_and_explain_region, TypeErrCtxt};
 use crate::infer::{self, SubregionOrigin};
 use rustc_errors::{
-    fluent, struct_span_err, AddToDiagnostic, Applicability, Diagnostic, DiagnosticBuilder,
-    ErrorGuaranteed,
+    AddToDiagnostic, Diagnostic, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic,
 };
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_middle::traits::ObligationCauseCode;
@@ -78,7 +81,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         sub: Region<'tcx>,
         sup: Region<'tcx>,
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
-        match origin {
+        let mut err = match origin {
             infer::Subtype(box trace) => {
                 let terr = TypeError::RegionsDoesNotOutlive(sup, sub);
                 let mut err = self.report_and_explain_type_error(trace, terr);
@@ -119,130 +122,105 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 err
             }
             infer::Reborrow(span) => {
-                let mut err = struct_span_err!(
-                    self.tcx.sess,
-                    span,
-                    E0312,
-                    "lifetime of reference outlives lifetime of borrowed content..."
-                );
-                note_and_explain_region(
+                let reference_valid = note_and_explain::RegionExplanation::new(
                     self.tcx,
-                    &mut err,
-                    "...the reference is valid for ",
                     sub,
-                    "...",
                     None,
+                    note_and_explain::PrefixKind::RefValidFor,
+                    note_and_explain::SuffixKind::Continues,
                 );
-                note_and_explain_region(
+                let content_valid = note_and_explain::RegionExplanation::new(
                     self.tcx,
-                    &mut err,
-                    "...but the borrowed content is only valid for ",
                     sup,
-                    "",
                     None,
+                    note_and_explain::PrefixKind::ContentValidFor,
+                    note_and_explain::SuffixKind::Empty,
                 );
-                err
+                OutlivesContent {
+                    span,
+                    notes: reference_valid.into_iter().chain(content_valid).collect(),
+                }
+                .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic)
             }
             infer::RelateObjectBound(span) => {
-                let mut err = struct_span_err!(
-                    self.tcx.sess,
-                    span,
-                    E0476,
-                    "lifetime of the source pointer does not outlive lifetime bound of the \
-                     object type"
-                );
-                note_and_explain_region(
+                let object_valid = note_and_explain::RegionExplanation::new(
                     self.tcx,
-                    &mut err,
-                    "object type is valid for ",
                     sub,
-                    "",
                     None,
+                    note_and_explain::PrefixKind::TypeObjValidFor,
+                    note_and_explain::SuffixKind::Empty,
                 );
-                note_and_explain_region(
+                let pointer_valid = note_and_explain::RegionExplanation::new(
                     self.tcx,
-                    &mut err,
-                    "source pointer is only valid for ",
                     sup,
-                    "",
                     None,
+                    note_and_explain::PrefixKind::SourcePointerValidFor,
+                    note_and_explain::SuffixKind::Empty,
                 );
-                err
+                OutlivesBound {
+                    span,
+                    notes: object_valid.into_iter().chain(pointer_valid).collect(),
+                }
+                .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic)
             }
             infer::RelateParamBound(span, ty, opt_span) => {
-                let mut err = struct_span_err!(
-                    self.tcx.sess,
-                    span,
-                    E0477,
-                    "the type `{}` does not fulfill the required lifetime",
-                    self.ty_to_string(ty)
+                let prefix = match *sub {
+                    ty::ReStatic => note_and_explain::PrefixKind::TypeSatisfy,
+                    _ => note_and_explain::PrefixKind::TypeOutlive,
+                };
+                let suffix = if opt_span.is_some() {
+                    note_and_explain::SuffixKind::ReqByBinding
+                } else {
+                    note_and_explain::SuffixKind::Empty
+                };
+                let note = note_and_explain::RegionExplanation::new(
+                    self.tcx, sub, opt_span, prefix, suffix,
                 );
-                match *sub {
-                    ty::ReStatic => note_and_explain_region(
-                        self.tcx,
-                        &mut err,
-                        "type must satisfy ",
-                        sub,
-                        if opt_span.is_some() { " as required by this binding" } else { "" },
-                        opt_span,
-                    ),
-                    _ => note_and_explain_region(
-                        self.tcx,
-                        &mut err,
-                        "type must outlive ",
-                        sub,
-                        if opt_span.is_some() { " as required by this binding" } else { "" },
-                        opt_span,
-                    ),
-                }
-                err
+                FullfillReqLifetime { span, ty: self.resolve_vars_if_possible(ty), note }
+                    .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic)
             }
             infer::RelateRegionParamBound(span) => {
-                let mut err =
-                    struct_span_err!(self.tcx.sess, span, E0478, "lifetime bound not satisfied");
-                note_and_explain_region(
+                let param_instantiated = note_and_explain::RegionExplanation::new(
                     self.tcx,
-                    &mut err,
-                    "lifetime parameter instantiated with ",
                     sup,
-                    "",
                     None,
+                    note_and_explain::PrefixKind::LfParamInstantiatedWith,
+                    note_and_explain::SuffixKind::Empty,
                 );
-                note_and_explain_region(
+                let param_must_outlive = note_and_explain::RegionExplanation::new(
                     self.tcx,
-                    &mut err,
-                    "but lifetime parameter must outlive ",
                     sub,
-                    "",
                     None,
+                    note_and_explain::PrefixKind::LfParamMustOutlive,
+                    note_and_explain::SuffixKind::Empty,
                 );
-                err
+                LfBoundNotSatisfied {
+                    span,
+                    notes: param_instantiated.into_iter().chain(param_must_outlive).collect(),
+                }
+                .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic)
             }
             infer::ReferenceOutlivesReferent(ty, span) => {
-                let mut err = struct_span_err!(
-                    self.tcx.sess,
-                    span,
-                    E0491,
-                    "in type `{}`, reference has a longer lifetime than the data it references",
-                    self.ty_to_string(ty)
-                );
-                note_and_explain_region(
+                let pointer_valid = note_and_explain::RegionExplanation::new(
                     self.tcx,
-                    &mut err,
-                    "the pointer is valid for ",
                     sub,
-                    "",
                     None,
+                    note_and_explain::PrefixKind::PointerValidFor,
+                    note_and_explain::SuffixKind::Empty,
                 );
-                note_and_explain_region(
+                let data_valid = note_and_explain::RegionExplanation::new(
                     self.tcx,
-                    &mut err,
-                    "but the referenced data is only valid for ",
                     sup,
-                    "",
                     None,
+                    note_and_explain::PrefixKind::DataValidFor,
+                    note_and_explain::SuffixKind::Empty,
                 );
-                err
+                RefLongerThanData {
+                    span,
+                    ty: self.resolve_vars_if_possible(ty),
+                    notes: pointer_valid.into_iter().chain(data_valid).collect(),
+                }
+                .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic)
             }
             infer::CompareImplItemObligation { span, impl_item_def_id, trait_item_def_id } => {
                 let mut err = self.report_extra_impl_obligation(
@@ -279,27 +257,31 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 err
             }
             infer::AscribeUserTypeProvePredicate(span) => {
-                let mut err =
-                    struct_span_err!(self.tcx.sess, span, E0478, "lifetime bound not satisfied");
-                note_and_explain_region(
+                let instantiated = note_and_explain::RegionExplanation::new(
                     self.tcx,
-                    &mut err,
-                    "lifetime instantiated with ",
                     sup,
-                    "",
                     None,
+                    note_and_explain::PrefixKind::LfInstantiatedWith,
+                    note_and_explain::SuffixKind::Empty,
                 );
-                note_and_explain_region(
+                let must_outlive = note_and_explain::RegionExplanation::new(
                     self.tcx,
-                    &mut err,
-                    "but lifetime must outlive ",
                     sub,
-                    "",
                     None,
+                    note_and_explain::PrefixKind::LfMustOutlive,
+                    note_and_explain::SuffixKind::Empty,
                 );
-                err
+                LfBoundNotSatisfied {
+                    span,
+                    notes: instantiated.into_iter().chain(must_outlive).collect(),
+                }
+                .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic)
             }
+        };
+        if sub.is_error() || sup.is_error() {
+            err.delay_as_bug();
         }
+        err
     }
 
     pub fn suggest_copy_trait_method_bounds(
@@ -343,22 +325,17 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
 
         let Some(generics) = self.tcx.hir().get_generics(impl_item_def_id) else { return; };
 
-        if trait_predicates.is_empty() {
-            err.span_suggestion_verbose(
-                generics.where_clause_span,
-                "remove the `where` clause",
-                String::new(),
-                Applicability::MachineApplicable,
-            );
+        let suggestion = if trait_predicates.is_empty() {
+            WhereClauseSuggestions::Remove { span: generics.where_clause_span }
         } else {
             let space = if generics.where_clause_span.is_empty() { " " } else { "" };
-            err.span_suggestion_verbose(
-                generics.where_clause_span,
-                "copy the `where` clause predicates from the trait",
-                format!("{space}where {}", trait_predicates.join(", ")),
-                Applicability::MachineApplicable,
-            );
-        }
+            WhereClauseSuggestions::CopyPredicates {
+                span: generics.where_clause_span,
+                space,
+                trait_predicates: trait_predicates.join(", "),
+            }
+        };
+        err.subdiagnostic(suggestion);
     }
 
     pub(super) fn report_placeholder_failure(
diff --git a/compiler/rustc_infer/src/infer/error_reporting/note_and_explain.rs b/compiler/rustc_infer/src/infer/error_reporting/note_and_explain.rs
new file mode 100644
index 000000000..b33729d0b
--- /dev/null
+++ b/compiler/rustc_infer/src/infer/error_reporting/note_and_explain.rs
@@ -0,0 +1,700 @@
+use super::TypeErrCtxt;
+use rustc_errors::Applicability::{MachineApplicable, MaybeIncorrect};
+use rustc_errors::{pluralize, Diagnostic, MultiSpan};
+use rustc_hir::{self as hir, def::DefKind};
+use rustc_middle::traits::ObligationCauseCode;
+use rustc_middle::ty::error::ExpectedFound;
+use rustc_middle::ty::print::Printer;
+use rustc_middle::{
+    traits::ObligationCause,
+    ty::{self, error::TypeError, print::FmtPrinter, suggest_constraining_type_param, Ty},
+};
+use rustc_span::{def_id::DefId, sym, BytePos, Span, Symbol};
+
+impl<'tcx> TypeErrCtxt<'_, 'tcx> {
+    pub fn note_and_explain_type_err(
+        &self,
+        diag: &mut Diagnostic,
+        err: TypeError<'tcx>,
+        cause: &ObligationCause<'tcx>,
+        sp: Span,
+        body_owner_def_id: DefId,
+    ) {
+        use ty::error::TypeError::*;
+        debug!("note_and_explain_type_err err={:?} cause={:?}", err, cause);
+
+        let tcx = self.tcx;
+
+        match err {
+            ArgumentSorts(values, _) | Sorts(values) => {
+                match (values.expected.kind(), values.found.kind()) {
+                    (ty::Closure(..), ty::Closure(..)) => {
+                        diag.note("no two closures, even if identical, have the same type");
+                        diag.help("consider boxing your closure and/or using it as a trait object");
+                    }
+                    (ty::Alias(ty::Opaque, ..), ty::Alias(ty::Opaque, ..)) => {
+                        // Issue #63167
+                        diag.note("distinct uses of `impl Trait` result in different opaque types");
+                    }
+                    (ty::Float(_), ty::Infer(ty::IntVar(_)))
+                        if let Ok(
+                            // Issue #53280
+                            snippet,
+                        ) = tcx.sess.source_map().span_to_snippet(sp) =>
+                    {
+                        if snippet.chars().all(|c| c.is_digit(10) || c == '-' || c == '_') {
+                            diag.span_suggestion(
+                                sp,
+                                "use a float literal",
+                                format!("{}.0", snippet),
+                                MachineApplicable,
+                            );
+                        }
+                    }
+                    (ty::Param(expected), ty::Param(found)) => {
+                        let generics = tcx.generics_of(body_owner_def_id);
+                        let e_span = tcx.def_span(generics.type_param(expected, tcx).def_id);
+                        if !sp.contains(e_span) {
+                            diag.span_label(e_span, "expected type parameter");
+                        }
+                        let f_span = tcx.def_span(generics.type_param(found, tcx).def_id);
+                        if !sp.contains(f_span) {
+                            diag.span_label(f_span, "found type parameter");
+                        }
+                        diag.note(
+                            "a type parameter was expected, but a different one was found; \
+                             you might be missing a type parameter or trait bound",
+                        );
+                        diag.note(
+                            "for more information, visit \
+                             https://doc.rust-lang.org/book/ch10-02-traits.html\
+                             #traits-as-parameters",
+                        );
+                    }
+                    (ty::Alias(ty::Projection, _), ty::Alias(ty::Projection, _)) => {
+                        diag.note("an associated type was expected, but a different one was found");
+                    }
+                    (ty::Param(p), ty::Alias(ty::Projection, proj)) | (ty::Alias(ty::Projection, proj), ty::Param(p))
+                        if tcx.def_kind(proj.def_id) != DefKind::ImplTraitPlaceholder =>
+                    {
+                        let p_def_id = tcx
+                            .generics_of(body_owner_def_id)
+                            .type_param(p, tcx)
+                            .def_id;
+                        let p_span = tcx.def_span(p_def_id);
+                        if !sp.contains(p_span) {
+                            diag.span_label(p_span, "this type parameter");
+                        }
+                        let hir = tcx.hir();
+                        let mut note = true;
+                        let parent = p_def_id
+                            .as_local()
+                            .and_then(|id| {
+                                let local_id = hir.local_def_id_to_hir_id(id);
+                                let generics = tcx.hir().find_parent(local_id)?.generics()?;
+                                Some((id, generics))
+                            });
+                        if let Some((local_id, generics)) = parent
+                        {
+                            // Synthesize the associated type restriction `Add<Output = Expected>`.
+                            // FIXME: extract this logic for use in other diagnostics.
+                            let (trait_ref, assoc_substs) = proj.trait_ref_and_own_substs(tcx);
+                            let item_name = tcx.item_name(proj.def_id);
+                            let item_args = self.format_generic_args(assoc_substs);
+
+                            // Here, we try to see if there's an existing
+                            // trait implementation that matches the one that
+                            // we're suggesting to restrict. If so, find the
+                            // "end", whether it be at the end of the trait
+                            // or the end of the generic arguments.
+                            let mut matching_span = None;
+                            let mut matched_end_of_args = false;
+                            for bound in generics.bounds_for_param(local_id) {
+                                let potential_spans = bound
+                                    .bounds
+                                    .iter()
+                                    .find_map(|bound| {
+                                        let bound_trait_path = bound.trait_ref()?.path;
+                                        let def_id = bound_trait_path.res.opt_def_id()?;
+                                        let generic_args = bound_trait_path.segments.iter().last().map(|path| path.args());
+                                        (def_id == trait_ref.def_id).then_some((bound_trait_path.span, generic_args))
+                                    });
+
+                                if let Some((end_of_trait, end_of_args)) = potential_spans {
+                                    let args_span = end_of_args.and_then(|args| args.span());
+                                    matched_end_of_args = args_span.is_some();
+                                    matching_span = args_span
+                                        .or_else(|| Some(end_of_trait))
+                                        .map(|span| span.shrink_to_hi());
+                                    break;
+                                }
+                            }
+
+                            if matched_end_of_args {
+                                // Append suggestion to the end of our args
+                                let path = format!(", {}{} = {}",item_name, item_args, p);
+                                note = !suggest_constraining_type_param(
+                                    tcx,
+                                    generics,
+                                    diag,
+                                    &format!("{}", proj.self_ty()),
+                                    &path,
+                                    None,
+                                    matching_span,
+                                );
+                            } else {
+                                // Suggest adding a bound to an existing trait
+                                // or if the trait doesn't exist, add the trait
+                                // and the suggested bounds.
+                                let path = format!("<{}{} = {}>", item_name, item_args, p);
+                                note = !suggest_constraining_type_param(
+                                    tcx,
+                                    generics,
+                                    diag,
+                                    &format!("{}", proj.self_ty()),
+                                    &path,
+                                    None,
+                                    matching_span,
+                                );
+                            }
+                        }
+                        if note {
+                            diag.note("you might be missing a type parameter or trait bound");
+                        }
+                    }
+                    (ty::Param(p), ty::Dynamic(..) | ty::Alias(ty::Opaque, ..))
+                    | (ty::Dynamic(..) | ty::Alias(ty::Opaque, ..), ty::Param(p)) => {
+                        let generics = tcx.generics_of(body_owner_def_id);
+                        let p_span = tcx.def_span(generics.type_param(p, tcx).def_id);
+                        if !sp.contains(p_span) {
+                            diag.span_label(p_span, "this type parameter");
+                        }
+                        diag.help("type parameters must be constrained to match other types");
+                        if tcx.sess.teach(&diag.get_code().unwrap()) {
+                            diag.help(
+                                "given a type parameter `T` and a method `foo`:
+```
+trait Trait<T> { fn foo(&self) -> T; }
+```
+the only ways to implement method `foo` are:
+- constrain `T` with an explicit type:
+```
+impl Trait<String> for X {
+    fn foo(&self) -> String { String::new() }
+}
+```
+- add a trait bound to `T` and call a method on that trait that returns `Self`:
+```
+impl<T: std::default::Default> Trait<T> for X {
+    fn foo(&self) -> T { <T as std::default::Default>::default() }
+}
+```
+- change `foo` to return an argument of type `T`:
+```
+impl<T> Trait<T> for X {
+    fn foo(&self, x: T) -> T { x }
+}
+```",
+                            );
+                        }
+                        diag.note(
+                            "for more information, visit \
+                             https://doc.rust-lang.org/book/ch10-02-traits.html\
+                             #traits-as-parameters",
+                        );
+                    }
+                    (ty::Param(p), ty::Closure(..) | ty::Generator(..)) => {
+                        let generics = tcx.generics_of(body_owner_def_id);
+                        let p_span = tcx.def_span(generics.type_param(p, tcx).def_id);
+                        if !sp.contains(p_span) {
+                            diag.span_label(p_span, "this type parameter");
+                        }
+                        diag.help(&format!(
+                            "every closure has a distinct type and so could not always match the \
+                             caller-chosen type of parameter `{}`",
+                            p
+                        ));
+                    }
+                    (ty::Param(p), _) | (_, ty::Param(p)) => {
+                        let generics = tcx.generics_of(body_owner_def_id);
+                        let p_span = tcx.def_span(generics.type_param(p, tcx).def_id);
+                        if !sp.contains(p_span) {
+                            diag.span_label(p_span, "this type parameter");
+                        }
+                    }
+                    (ty::Alias(ty::Projection, proj_ty), _) if tcx.def_kind(proj_ty.def_id) != DefKind::ImplTraitPlaceholder => {
+                        self.expected_projection(
+                            diag,
+                            proj_ty,
+                            values,
+                            body_owner_def_id,
+                            cause.code(),
+                        );
+                    }
+                    (_, ty::Alias(ty::Projection, proj_ty)) if tcx.def_kind(proj_ty.def_id) != DefKind::ImplTraitPlaceholder => {
+                        let msg = format!(
+                            "consider constraining the associated type `{}` to `{}`",
+                            values.found, values.expected,
+                        );
+                        if !(self.suggest_constraining_opaque_associated_type(
+                            diag,
+                            &msg,
+                            proj_ty,
+                            values.expected,
+                        ) || self.suggest_constraint(
+                            diag,
+                            &msg,
+                            body_owner_def_id,
+                            proj_ty,
+                            values.expected,
+                        )) {
+                            diag.help(&msg);
+                            diag.note(
+                                "for more information, visit \
+                                https://doc.rust-lang.org/book/ch19-03-advanced-traits.html",
+                            );
+                        }
+                    }
+                    (ty::FnPtr(_), ty::FnDef(def, _))
+                    if let hir::def::DefKind::Fn = tcx.def_kind(def) => {
+                        diag.note(
+                            "when the arguments and return types match, functions can be coerced \
+                             to function pointers",
+                        );
+                    }
+                    _ => {}
+                }
+                debug!(
+                    "note_and_explain_type_err expected={:?} ({:?}) found={:?} ({:?})",
+                    values.expected,
+                    values.expected.kind(),
+                    values.found,
+                    values.found.kind(),
+                );
+            }
+            CyclicTy(ty) => {
+                // Watch out for various cases of cyclic types and try to explain.
+                if ty.is_closure() || ty.is_generator() {
+                    diag.note(
+                        "closures cannot capture themselves or take themselves as argument;\n\
+                         this error may be the result of a recent compiler bug-fix,\n\
+                         see issue #46062 <https://github.com/rust-lang/rust/issues/46062>\n\
+                         for more information",
+                    );
+                }
+            }
+            TargetFeatureCast(def_id) => {
+                let target_spans = tcx.get_attrs(def_id, sym::target_feature).map(|attr| attr.span);
+                diag.note(
+                    "functions with `#[target_feature]` can only be coerced to `unsafe` function pointers"
+                );
+                diag.span_labels(target_spans, "`#[target_feature]` added here");
+            }
+            _ => {}
+        }
+    }
+
+    fn suggest_constraint(
+        &self,
+        diag: &mut Diagnostic,
+        msg: &str,
+        body_owner_def_id: DefId,
+        proj_ty: &ty::AliasTy<'tcx>,
+        ty: Ty<'tcx>,
+    ) -> bool {
+        let tcx = self.tcx;
+        let assoc = tcx.associated_item(proj_ty.def_id);
+        let (trait_ref, assoc_substs) = proj_ty.trait_ref_and_own_substs(tcx);
+        if let Some(item) = tcx.hir().get_if_local(body_owner_def_id) {
+            if let Some(hir_generics) = item.generics() {
+                // Get the `DefId` for the type parameter corresponding to `A` in `<A as T>::Foo`.
+                // This will also work for `impl Trait`.
+                let def_id = if let ty::Param(param_ty) = proj_ty.self_ty().kind() {
+                    let generics = tcx.generics_of(body_owner_def_id);
+                    generics.type_param(param_ty, tcx).def_id
+                } else {
+                    return false;
+                };
+                let Some(def_id) = def_id.as_local() else {
+                    return false;
+                };
+
+                // First look in the `where` clause, as this might be
+                // `fn foo<T>(x: T) where T: Trait`.
+                for pred in hir_generics.bounds_for_param(def_id) {
+                    if self.constrain_generic_bound_associated_type_structured_suggestion(
+                        diag,
+                        &trait_ref,
+                        pred.bounds,
+                        assoc,
+                        assoc_substs,
+                        ty,
+                        msg,
+                        false,
+                    ) {
+                        return true;
+                    }
+                }
+            }
+        }
+        false
+    }
+
+    /// An associated type was expected and a different type was found.
+    ///
+    /// We perform a few different checks to see what we can suggest:
+    ///
+    ///  - In the current item, look for associated functions that return the expected type and
+    ///    suggest calling them. (Not a structured suggestion.)
+    ///  - If any of the item's generic bounds can be constrained, we suggest constraining the
+    ///    associated type to the found type.
+    ///  - If the associated type has a default type and was expected inside of a `trait`, we
+    ///    mention that this is disallowed.
+    ///  - If all other things fail, and the error is not because of a mismatch between the `trait`
+    ///    and the `impl`, we provide a generic `help` to constrain the assoc type or call an assoc
+    ///    fn that returns the type.
+    fn expected_projection(
+        &self,
+        diag: &mut Diagnostic,
+        proj_ty: &ty::AliasTy<'tcx>,
+        values: ExpectedFound<Ty<'tcx>>,
+        body_owner_def_id: DefId,
+        cause_code: &ObligationCauseCode<'_>,
+    ) {
+        let tcx = self.tcx;
+
+        let msg = format!(
+            "consider constraining the associated type `{}` to `{}`",
+            values.expected, values.found
+        );
+        let body_owner = tcx.hir().get_if_local(body_owner_def_id);
+        let current_method_ident = body_owner.and_then(|n| n.ident()).map(|i| i.name);
+
+        // We don't want to suggest calling an assoc fn in a scope where that isn't feasible.
+        let callable_scope = matches!(
+            body_owner,
+            Some(
+                hir::Node::Item(hir::Item { kind: hir::ItemKind::Fn(..), .. })
+                    | hir::Node::TraitItem(hir::TraitItem { kind: hir::TraitItemKind::Fn(..), .. })
+                    | hir::Node::ImplItem(hir::ImplItem { kind: hir::ImplItemKind::Fn(..), .. }),
+            )
+        );
+        let impl_comparison =
+            matches!(cause_code, ObligationCauseCode::CompareImplItemObligation { .. });
+        let assoc = tcx.associated_item(proj_ty.def_id);
+        if !callable_scope || impl_comparison {
+            // We do not want to suggest calling functions when the reason of the
+            // type error is a comparison of an `impl` with its `trait` or when the
+            // scope is outside of a `Body`.
+        } else {
+            // If we find a suitable associated function that returns the expected type, we don't
+            // want the more general suggestion later in this method about "consider constraining
+            // the associated type or calling a method that returns the associated type".
+            let point_at_assoc_fn = self.point_at_methods_that_satisfy_associated_type(
+                diag,
+                assoc.container_id(tcx),
+                current_method_ident,
+                proj_ty.def_id,
+                values.expected,
+            );
+            // Possibly suggest constraining the associated type to conform to the
+            // found type.
+            if self.suggest_constraint(diag, &msg, body_owner_def_id, proj_ty, values.found)
+                || point_at_assoc_fn
+            {
+                return;
+            }
+        }
+
+        self.suggest_constraining_opaque_associated_type(diag, &msg, proj_ty, values.found);
+
+        if self.point_at_associated_type(diag, body_owner_def_id, values.found) {
+            return;
+        }
+
+        if !impl_comparison {
+            // Generic suggestion when we can't be more specific.
+            if callable_scope {
+                diag.help(&format!(
+                    "{} or calling a method that returns `{}`",
+                    msg, values.expected
+                ));
+            } else {
+                diag.help(&msg);
+            }
+            diag.note(
+                "for more information, visit \
+                 https://doc.rust-lang.org/book/ch19-03-advanced-traits.html",
+            );
+        }
+        if tcx.sess.teach(&diag.get_code().unwrap()) {
+            diag.help(
+                "given an associated type `T` and a method `foo`:
+```
+trait Trait {
+type T;
+fn foo(&self) -> Self::T;
+}
+```
+the only way of implementing method `foo` is to constrain `T` with an explicit associated type:
+```
+impl Trait for X {
+type T = String;
+fn foo(&self) -> Self::T { String::new() }
+}
+```",
+            );
+        }
+    }
+
+    /// When the expected `impl Trait` is not defined in the current item, it will come from
+    /// a return type. This can occur when dealing with `TryStream` (#71035).
+    fn suggest_constraining_opaque_associated_type(
+        &self,
+        diag: &mut Diagnostic,
+        msg: &str,
+        proj_ty: &ty::AliasTy<'tcx>,
+        ty: Ty<'tcx>,
+    ) -> bool {
+        let tcx = self.tcx;
+
+        let assoc = tcx.associated_item(proj_ty.def_id);
+        if let ty::Alias(ty::Opaque, ty::AliasTy { def_id, .. }) = *proj_ty.self_ty().kind() {
+            let opaque_local_def_id = def_id.as_local();
+            let opaque_hir_ty = if let Some(opaque_local_def_id) = opaque_local_def_id {
+                match &tcx.hir().expect_item(opaque_local_def_id).kind {
+                    hir::ItemKind::OpaqueTy(opaque_hir_ty) => opaque_hir_ty,
+                    _ => bug!("The HirId comes from a `ty::Opaque`"),
+                }
+            } else {
+                return false;
+            };
+
+            let (trait_ref, assoc_substs) = proj_ty.trait_ref_and_own_substs(tcx);
+
+            self.constrain_generic_bound_associated_type_structured_suggestion(
+                diag,
+                &trait_ref,
+                opaque_hir_ty.bounds,
+                assoc,
+                assoc_substs,
+                ty,
+                msg,
+                true,
+            )
+        } else {
+            false
+        }
+    }
+
+    fn point_at_methods_that_satisfy_associated_type(
+        &self,
+        diag: &mut Diagnostic,
+        assoc_container_id: DefId,
+        current_method_ident: Option<Symbol>,
+        proj_ty_item_def_id: DefId,
+        expected: Ty<'tcx>,
+    ) -> bool {
+        let tcx = self.tcx;
+
+        let items = tcx.associated_items(assoc_container_id);
+        // Find all the methods in the trait that could be called to construct the
+        // expected associated type.
+        // FIXME: consider suggesting the use of associated `const`s.
+        let methods: Vec<(Span, String)> = items
+            .in_definition_order()
+            .filter(|item| {
+                ty::AssocKind::Fn == item.kind
+                    && Some(item.name) != current_method_ident
+                    && !tcx.is_doc_hidden(item.def_id)
+            })
+            .filter_map(|item| {
+                let method = tcx.fn_sig(item.def_id).subst_identity();
+                match *method.output().skip_binder().kind() {
+                    ty::Alias(ty::Projection, ty::AliasTy { def_id: item_def_id, .. })
+                        if item_def_id == proj_ty_item_def_id =>
+                    {
+                        Some((
+                            tcx.def_span(item.def_id),
+                            format!("consider calling `{}`", tcx.def_path_str(item.def_id)),
+                        ))
+                    }
+                    _ => None,
+                }
+            })
+            .collect();
+        if !methods.is_empty() {
+            // Use a single `help:` to show all the methods in the trait that can
+            // be used to construct the expected associated type.
+            let mut span: MultiSpan =
+                methods.iter().map(|(sp, _)| *sp).collect::<Vec<Span>>().into();
+            let msg = format!(
+                "{some} method{s} {are} available that return{r} `{ty}`",
+                some = if methods.len() == 1 { "a" } else { "some" },
+                s = pluralize!(methods.len()),
+                are = pluralize!("is", methods.len()),
+                r = if methods.len() == 1 { "s" } else { "" },
+                ty = expected
+            );
+            for (sp, label) in methods.into_iter() {
+                span.push_span_label(sp, label);
+            }
+            diag.span_help(span, &msg);
+            return true;
+        }
+        false
+    }
+
+    fn point_at_associated_type(
+        &self,
+        diag: &mut Diagnostic,
+        body_owner_def_id: DefId,
+        found: Ty<'tcx>,
+    ) -> bool {
+        let tcx = self.tcx;
+
+        let Some(hir_id) = body_owner_def_id.as_local() else {
+            return false;
+        };
+        let hir_id = tcx.hir().local_def_id_to_hir_id(hir_id);
+        // When `body_owner` is an `impl` or `trait` item, look in its associated types for
+        // `expected` and point at it.
+        let parent_id = tcx.hir().get_parent_item(hir_id);
+        let item = tcx.hir().find_by_def_id(parent_id.def_id);
+
+        debug!("expected_projection parent item {:?}", item);
+
+        let param_env = tcx.param_env(body_owner_def_id);
+
+        match item {
+            Some(hir::Node::Item(hir::Item { kind: hir::ItemKind::Trait(.., items), .. })) => {
+                // FIXME: account for `#![feature(specialization)]`
+                for item in &items[..] {
+                    match item.kind {
+                        hir::AssocItemKind::Type => {
+                            // FIXME: account for returning some type in a trait fn impl that has
+                            // an assoc type as a return type (#72076).
+                            if let hir::Defaultness::Default { has_value: true } =
+                                tcx.impl_defaultness(item.id.owner_id)
+                            {
+                                let assoc_ty = tcx.type_of(item.id.owner_id).subst_identity();
+                                if self.infcx.can_eq(param_env, assoc_ty, found) {
+                                    diag.span_label(
+                                        item.span,
+                                        "associated type defaults can't be assumed inside the \
+                                            trait defining them",
+                                    );
+                                    return true;
+                                }
+                            }
+                        }
+                        _ => {}
+                    }
+                }
+            }
+            Some(hir::Node::Item(hir::Item {
+                kind: hir::ItemKind::Impl(hir::Impl { items, .. }),
+                ..
+            })) => {
+                for item in &items[..] {
+                    if let hir::AssocItemKind::Type = item.kind {
+                        let assoc_ty = tcx.type_of(item.id.owner_id).subst_identity();
+
+                        if self.infcx.can_eq(param_env, assoc_ty, found) {
+                            diag.span_label(item.span, "expected this associated type");
+                            return true;
+                        }
+                    }
+                }
+            }
+            _ => {}
+        }
+        false
+    }
+
+    /// Given a slice of `hir::GenericBound`s, if any of them corresponds to the `trait_ref`
+    /// requirement, provide a structured suggestion to constrain it to a given type `ty`.
+    ///
+    /// `is_bound_surely_present` indicates whether we know the bound we're looking for is
+    /// inside `bounds`. If that's the case then we can consider `bounds` containing only one
+    /// trait bound as the one we're looking for. This can help in cases where the associated
+    /// type is defined on a supertrait of the one present in the bounds.
+    fn constrain_generic_bound_associated_type_structured_suggestion(
+        &self,
+        diag: &mut Diagnostic,
+        trait_ref: &ty::TraitRef<'tcx>,
+        bounds: hir::GenericBounds<'_>,
+        assoc: ty::AssocItem,
+        assoc_substs: &[ty::GenericArg<'tcx>],
+        ty: Ty<'tcx>,
+        msg: &str,
+        is_bound_surely_present: bool,
+    ) -> bool {
+        // FIXME: we would want to call `resolve_vars_if_possible` on `ty` before suggesting.
+
+        let trait_bounds = bounds.iter().filter_map(|bound| match bound {
+            hir::GenericBound::Trait(ptr, hir::TraitBoundModifier::None) => Some(ptr),
+            _ => None,
+        });
+
+        let matching_trait_bounds = trait_bounds
+            .clone()
+            .filter(|ptr| ptr.trait_ref.trait_def_id() == Some(trait_ref.def_id))
+            .collect::<Vec<_>>();
+
+        let span = match &matching_trait_bounds[..] {
+            &[ptr] => ptr.span,
+            &[] if is_bound_surely_present => match &trait_bounds.collect::<Vec<_>>()[..] {
+                &[ptr] => ptr.span,
+                _ => return false,
+            },
+            _ => return false,
+        };
+
+        self.constrain_associated_type_structured_suggestion(
+            diag,
+            span,
+            assoc,
+            assoc_substs,
+            ty,
+            msg,
+        )
+    }
+
+    /// Given a span corresponding to a bound, provide a structured suggestion to set an
+    /// associated type to a given type `ty`.
+    fn constrain_associated_type_structured_suggestion(
+        &self,
+        diag: &mut Diagnostic,
+        span: Span,
+        assoc: ty::AssocItem,
+        assoc_substs: &[ty::GenericArg<'tcx>],
+        ty: Ty<'tcx>,
+        msg: &str,
+    ) -> bool {
+        let tcx = self.tcx;
+
+        if let Ok(has_params) =
+            tcx.sess.source_map().span_to_snippet(span).map(|snippet| snippet.ends_with('>'))
+        {
+            let (span, sugg) = if has_params {
+                let pos = span.hi() - BytePos(1);
+                let span = Span::new(pos, pos, span.ctxt(), span.parent());
+                (span, format!(", {} = {}", assoc.ident(tcx), ty))
+            } else {
+                let item_args = self.format_generic_args(assoc_substs);
+                (span.shrink_to_hi(), format!("<{}{} = {}>", assoc.ident(tcx), item_args, ty))
+            };
+            diag.span_suggestion_verbose(span, msg, sugg, MaybeIncorrect);
+            return true;
+        }
+        false
+    }
+
+    pub fn format_generic_args(&self, args: &[ty::GenericArg<'tcx>]) -> String {
+        FmtPrinter::new(self.tcx, hir::def::Namespace::TypeNS)
+            .path_generic_args(Ok, args)
+            .expect("could not write to `String`.")
+            .into_buffer()
+    }
+}
diff --git a/compiler/rustc_infer/src/infer/error_reporting/suggest.rs b/compiler/rustc_infer/src/infer/error_reporting/suggest.rs
index 5b02956a1..55dcfd05e 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/suggest.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/suggest.rs
@@ -8,24 +8,25 @@ use rustc_middle::traits::{
     StatementAsExpression,
 };
 use rustc_middle::ty::print::with_no_trimmed_paths;
-use rustc_middle::ty::{self as ty, Ty, TypeVisitable};
+use rustc_middle::ty::{self as ty, IsSuggestable, Ty, TypeVisitableExt};
 use rustc_span::{sym, BytePos, Span};
 
-use crate::errors::SuggAddLetForLetChains;
+use crate::errors::{
+    ConsiderAddingAwait, SuggAddLetForLetChains, SuggestRemoveSemiOrReturnBinding,
+};
 
 use super::TypeErrCtxt;
 
 impl<'tcx> TypeErrCtxt<'_, 'tcx> {
     pub(super) fn suggest_remove_semi_or_return_binding(
         &self,
-        err: &mut Diagnostic,
         first_id: Option<hir::HirId>,
         first_ty: Ty<'tcx>,
         first_span: Span,
         second_id: Option<hir::HirId>,
         second_ty: Ty<'tcx>,
         second_span: Span,
-    ) {
+    ) -> Option<SuggestRemoveSemiOrReturnBinding> {
         let remove_semicolon = [
             (first_id, self.resolve_vars_if_possible(second_ty)),
             (second_id, self.resolve_vars_if_possible(first_ty)),
@@ -37,35 +38,29 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         });
         match remove_semicolon {
             Some((sp, StatementAsExpression::NeedsBoxing)) => {
-                err.multipart_suggestion(
-                    "consider removing this semicolon and boxing the expressions",
-                    vec![
-                        (first_span.shrink_to_lo(), "Box::new(".to_string()),
-                        (first_span.shrink_to_hi(), ")".to_string()),
-                        (second_span.shrink_to_lo(), "Box::new(".to_string()),
-                        (second_span.shrink_to_hi(), ")".to_string()),
-                        (sp, String::new()),
-                    ],
-                    Applicability::MachineApplicable,
-                );
+                Some(SuggestRemoveSemiOrReturnBinding::RemoveAndBox {
+                    first_lo: first_span.shrink_to_lo(),
+                    first_hi: first_span.shrink_to_hi(),
+                    second_lo: second_span.shrink_to_lo(),
+                    second_hi: second_span.shrink_to_hi(),
+                    sp,
+                })
             }
             Some((sp, StatementAsExpression::CorrectType)) => {
-                err.span_suggestion_short(
-                    sp,
-                    "consider removing this semicolon",
-                    "",
-                    Applicability::MachineApplicable,
-                );
+                Some(SuggestRemoveSemiOrReturnBinding::Remove { sp })
             }
             None => {
+                let mut ret = None;
                 for (id, ty) in [(first_id, second_ty), (second_id, first_ty)] {
                     if let Some(id) = id
                         && let hir::Node::Block(blk) = self.tcx.hir().get(id)
-                        && self.consider_returning_binding(blk, ty, err)
+                        && let Some(diag) = self.consider_returning_binding_diag(blk, ty)
                     {
+                        ret = Some(diag);
                         break;
                     }
                 }
+                ret
             }
         }
     }
@@ -198,7 +193,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
             return;
         }
 
-        match (
+        let subdiag = match (
             self.get_impl_future_output_ty(exp_found.expected),
             self.get_impl_future_output_ty(exp_found.found),
         ) {
@@ -207,82 +202,68 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
             {
                 ObligationCauseCode::IfExpression(box IfExpressionCause { then_id, .. }) => {
                     let then_span = self.find_block_span_from_hir_id(*then_id);
-                    diag.multipart_suggestion(
-                        "consider `await`ing on both `Future`s",
-                        vec![
-                            (then_span.shrink_to_hi(), ".await".to_string()),
-                            (exp_span.shrink_to_hi(), ".await".to_string()),
-                        ],
-                        Applicability::MaybeIncorrect,
-                    );
+                    Some(ConsiderAddingAwait::BothFuturesSugg {
+                        first: then_span.shrink_to_hi(),
+                        second: exp_span.shrink_to_hi(),
+                    })
                 }
                 ObligationCauseCode::MatchExpressionArm(box MatchExpressionArmCause {
                     prior_arms,
                     ..
                 }) => {
                     if let [.., arm_span] = &prior_arms[..] {
-                        diag.multipart_suggestion(
-                            "consider `await`ing on both `Future`s",
-                            vec![
-                                (arm_span.shrink_to_hi(), ".await".to_string()),
-                                (exp_span.shrink_to_hi(), ".await".to_string()),
-                            ],
-                            Applicability::MaybeIncorrect,
-                        );
+                        Some(ConsiderAddingAwait::BothFuturesSugg {
+                            first: arm_span.shrink_to_hi(),
+                            second: exp_span.shrink_to_hi(),
+                        })
                     } else {
-                        diag.help("consider `await`ing on both `Future`s");
+                        Some(ConsiderAddingAwait::BothFuturesHelp)
                     }
                 }
-                _ => {
-                    diag.help("consider `await`ing on both `Future`s");
-                }
+                _ => Some(ConsiderAddingAwait::BothFuturesHelp),
             },
             (_, Some(ty)) if self.same_type_modulo_infer(exp_found.expected, ty) => {
-                diag.span_suggestion_verbose(
-                    exp_span.shrink_to_hi(),
-                    "consider `await`ing on the `Future`",
-                    ".await",
-                    Applicability::MaybeIncorrect,
-                );
+                // FIXME: Seems like we can't have a suggestion and a note with different spans in a single subdiagnostic
+                diag.subdiagnostic(ConsiderAddingAwait::FutureSugg {
+                    span: exp_span.shrink_to_hi(),
+                });
+                Some(ConsiderAddingAwait::FutureSuggNote { span: exp_span })
             }
             (Some(ty), _) if self.same_type_modulo_infer(ty, exp_found.found) => match cause.code()
             {
                 ObligationCauseCode::Pattern { span: Some(then_span), .. } => {
-                    diag.span_suggestion_verbose(
-                        then_span.shrink_to_hi(),
-                        "consider `await`ing on the `Future`",
-                        ".await",
-                        Applicability::MaybeIncorrect,
-                    );
+                    Some(ConsiderAddingAwait::FutureSugg { span: then_span.shrink_to_hi() })
                 }
                 ObligationCauseCode::IfExpression(box IfExpressionCause { then_id, .. }) => {
                     let then_span = self.find_block_span_from_hir_id(*then_id);
-                    diag.span_suggestion_verbose(
-                        then_span.shrink_to_hi(),
-                        "consider `await`ing on the `Future`",
-                        ".await",
-                        Applicability::MaybeIncorrect,
-                    );
+                    Some(ConsiderAddingAwait::FutureSugg { span: then_span.shrink_to_hi() })
                 }
                 ObligationCauseCode::MatchExpressionArm(box MatchExpressionArmCause {
                     ref prior_arms,
                     ..
-                }) => {
-                    diag.multipart_suggestion_verbose(
-                        "consider `await`ing on the `Future`",
-                        prior_arms
-                            .iter()
-                            .map(|arm| (arm.shrink_to_hi(), ".await".to_string()))
-                            .collect(),
-                        Applicability::MaybeIncorrect,
-                    );
-                }
-                _ => {}
+                }) => Some({
+                    ConsiderAddingAwait::FutureSuggMultiple {
+                        spans: prior_arms.iter().map(|arm| arm.shrink_to_hi()).collect(),
+                    }
+                }),
+                _ => None,
             },
-            _ => {}
+            _ => None,
+        };
+        if let Some(subdiag) = subdiag {
+            diag.subdiagnostic(subdiag);
         }
     }
 
+    pub fn suggest_await_on_future(&self, diag: &mut Diagnostic, sp: Span) {
+        diag.span_suggestion_verbose(
+            sp.shrink_to_hi(),
+            "consider `await`ing on the `Future`",
+            ".await",
+            Applicability::MaybeIncorrect,
+        );
+    }
+
     pub(super) fn suggest_accessing_field_where_appropriate(
         &self,
         cause: &ObligationCause<'tcx>,
@@ -351,6 +332,118 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         }
     }
 
+    pub(super) fn suggest_function_pointers(
+        &self,
+        cause: &ObligationCause<'tcx>,
+        span: Span,
+        exp_found: &ty::error::ExpectedFound<Ty<'tcx>>,
+        diag: &mut Diagnostic,
+    ) {
+        debug!("suggest_function_pointers(cause={:?}, exp_found={:?})", cause, exp_found);
+        let ty::error::ExpectedFound { expected, found } = exp_found;
+        let expected_inner = expected.peel_refs();
+        let found_inner = found.peel_refs();
+        if !expected_inner.is_fn() || !found_inner.is_fn() {
+            return;
+        }
+        match (&expected_inner.kind(), &found_inner.kind()) {
+            (ty::FnPtr(sig), ty::FnDef(did, substs)) => {
+                let expected_sig = &(self.normalize_fn_sig)(*sig);
+                let found_sig =
+                    &(self.normalize_fn_sig)(self.tcx.fn_sig(*did).subst(self.tcx, substs));
+
+                let fn_name = self.tcx.def_path_str_with_substs(*did, substs);
+
+                if !self.same_type_modulo_infer(*found_sig, *expected_sig)
+                    || !sig.is_suggestable(self.tcx, true)
+                    || ty::util::is_intrinsic(self.tcx, *did)
+                {
+                    return;
+                }
+
+                let (msg, sug) = match (expected.is_ref(), found.is_ref()) {
+                    (true, false) => {
+                        let msg = "consider using a reference";
+                        let sug = format!("&{fn_name}");
+                        (msg, sug)
+                    }
+                    (false, true) => {
+                        let msg = "consider removing the reference";
+                        let sug = format!("{fn_name}");
+                        (msg, sug)
+                    }
+                    (true, true) => {
+                        diag.note("fn items are distinct from fn pointers");
+                        let msg = "consider casting to a fn pointer";
+                        let sug = format!("&({fn_name} as {sig})");
+                        (msg, sug)
+                    }
+                    (false, false) => {
+                        diag.note("fn items are distinct from fn pointers");
+                        let msg = "consider casting to a fn pointer";
+                        let sug = format!("{fn_name} as {sig}");
+                        (msg, sug)
+                    }
+                };
+                diag.span_suggestion_verbose(span, msg, sug, Applicability::MaybeIncorrect);
+            }
+            (ty::FnDef(did1, substs1), ty::FnDef(did2, substs2)) => {
+                let expected_sig =
+                    &(self.normalize_fn_sig)(self.tcx.fn_sig(*did1).subst(self.tcx, substs1));
+                let found_sig =
+                    &(self.normalize_fn_sig)(self.tcx.fn_sig(*did2).subst(self.tcx, substs2));
+
+                if self.same_type_modulo_infer(*expected_sig, *found_sig) {
+                    diag.note("different fn items have unique types, even if their signatures are the same");
+                }
+
+                if !self.same_type_modulo_infer(*found_sig, *expected_sig)
+                    || !found_sig.is_suggestable(self.tcx, true)
+                    || !expected_sig.is_suggestable(self.tcx, true)
+                    || ty::util::is_intrinsic(self.tcx, *did1)
+                    || ty::util::is_intrinsic(self.tcx, *did2)
+                {
+                    return;
+                }
+
+                let fn_name = self.tcx.def_path_str_with_substs(*did2, substs2);
+                let sug = if found.is_ref() {
+                    format!("&({fn_name} as {found_sig})")
+                } else {
+                    format!("{fn_name} as {found_sig}")
+                };
+
+                let msg = format!(
+                    "consider casting both fn items to fn pointers using `as {expected_sig}`"
+                );
+
+                diag.span_suggestion_hidden(span, msg, sug, Applicability::MaybeIncorrect);
+            }
+            (ty::FnDef(did, substs), ty::FnPtr(sig)) => {
+                let expected_sig =
+                    &(self.normalize_fn_sig)(self.tcx.fn_sig(*did).subst(self.tcx, substs));
+                let found_sig = &(self.normalize_fn_sig)(*sig);
+
+                if !self.same_type_modulo_infer(*found_sig, *expected_sig) {
+                    return;
+                }
+
+                let fn_name = self.tcx.def_path_str_with_substs(*did, substs);
+
+                let casting = if expected.is_ref() {
+                    format!("&({fn_name} as {found_sig})")
+                } else {
+                    format!("{fn_name} as {found_sig}")
+                };
+
+                diag.help(&format!("consider casting the fn item to a fn pointer: `{}`", casting));
+            }
+            _ => {
+                return;
+            }
+        };
+    }
+
     pub fn should_suggest_as_ref(&self, expected: Ty<'tcx>, found: Ty<'tcx>) -> Option<&str> {
         if let (ty::Adt(exp_def, exp_substs), ty::Ref(_, found_ty, _)) =
             (expected.kind(), found.kind())
@@ -411,8 +504,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         span: Span,
     ) {
         let hir = self.tcx.hir();
-        let fn_hir_id = hir.parent_id(cause.body_id);
-        if let Some(node) = self.tcx.hir().find(fn_hir_id) &&
+        if let Some(node) = self.tcx.hir().find_by_def_id(cause.body_id) &&
             let hir::Node::Item(hir::Item {
                     kind: hir::ItemKind::Fn(_sig, _, body_id), ..
                 }) = node {
@@ -549,16 +641,15 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
 
     /// Suggest returning a local binding with a compatible type if the block
     /// has no return expression.
-    pub fn consider_returning_binding(
+    pub fn consider_returning_binding_diag(
         &self,
         blk: &'tcx hir::Block<'tcx>,
         expected_ty: Ty<'tcx>,
-        err: &mut Diagnostic,
-    ) -> bool {
+    ) -> Option<SuggestRemoveSemiOrReturnBinding> {
         let blk = blk.innermost_block();
         // Do not suggest if we have a tail expr.
         if blk.expr.is_some() {
-            return false;
+            return None;
         }
         let mut shadowed = FxIndexSet::default();
         let mut candidate_idents = vec![];
@@ -627,7 +718,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         match &candidate_idents[..] {
             [(ident, _ty)] => {
                 let sm = self.tcx.sess.source_map();
-                if let Some(stmt) = blk.stmts.last() {
+                let (span, sugg) = if let Some(stmt) = blk.stmts.last() {
                     let stmt_span = sm.stmt_span(stmt.span, blk.span);
                     let sugg = if sm.is_multiline(blk.span)
                         && let Some(spacing) = sm.indentation_before(stmt_span)
@@ -636,12 +727,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     } else {
                         format!(" {ident}")
                     };
-                    err.span_suggestion_verbose(
-                        stmt_span.shrink_to_hi(),
-                        format!("consider returning the local binding `{ident}`"),
-                        sugg,
-                        Applicability::MaybeIncorrect,
-                    );
+                    (stmt_span.shrink_to_hi(), sugg)
                 } else {
                     let sugg = if sm.is_multiline(blk.span)
                         && let Some(spacing) = sm.indentation_before(blk.span.shrink_to_lo())
@@ -651,21 +737,34 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                         format!(" {ident} ")
                     };
                     let left_span = sm.span_through_char(blk.span, '{').shrink_to_hi();
-                    err.span_suggestion_verbose(
+                    (
                         sm.span_extend_while(left_span, |c| c.is_whitespace()).unwrap_or(left_span),
-                        format!("consider returning the local binding `{ident}`"),
                         sugg,
-                        Applicability::MaybeIncorrect,
-                    );
-                }
-                true
+                    )
+                };
+                Some(SuggestRemoveSemiOrReturnBinding::Add { sp: span, code: sugg, ident: *ident })
             }
             values if (1..3).contains(&values.len()) => {
                 let spans = values.iter().map(|(ident, _)| ident.span).collect::<Vec<_>>();
-                err.span_note(spans, "consider returning one of these bindings");
+                Some(SuggestRemoveSemiOrReturnBinding::AddOne { spans: spans.into() })
+            }
+            _ => None,
+        }
+    }
+
+    pub fn consider_returning_binding(
+        &self,
+        blk: &'tcx hir::Block<'tcx>,
+        expected_ty: Ty<'tcx>,
+        err: &mut Diagnostic,
+    ) -> bool {
+        let diag = self.consider_returning_binding_diag(blk, expected_ty);
+        match diag {
+            Some(diag) => {
+                err.subdiagnostic(diag);
                 true
             }
-            _ => false,
+            None => false,
         }
     }
 }
diff --git a/compiler/rustc_infer/src/infer/freshen.rs b/compiler/rustc_infer/src/infer/freshen.rs
index 8f53b1ccd..f09f93abf 100644
--- a/compiler/rustc_infer/src/infer/freshen.rs
+++ b/compiler/rustc_infer/src/infer/freshen.rs
@@ -34,7 +34,7 @@ use super::InferCtxt;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_middle::infer::unify_key::ToType;
 use rustc_middle::ty::fold::TypeFolder;
-use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable, TypeSuperFoldable, TypeVisitable};
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable, TypeSuperFoldable, TypeVisitableExt};
 use std::collections::hash_map::Entry;
 
 pub struct TypeFreshener<'a, 'tcx> {
@@ -58,14 +58,9 @@ impl<'a, 'tcx> TypeFreshener<'a, 'tcx> {
         }
     }
 
-    fn freshen_ty<F>(
-        &mut self,
-        opt_ty: Option<Ty<'tcx>>,
-        key: ty::InferTy,
-        freshener: F,
-    ) -> Ty<'tcx>
+    fn freshen_ty<F>(&mut self, opt_ty: Option<Ty<'tcx>>, key: ty::InferTy, mk_fresh: F) -> Ty<'tcx>
     where
-        F: FnOnce(u32) -> ty::InferTy,
+        F: FnOnce(u32) -> Ty<'tcx>,
     {
         if let Some(ty) = opt_ty {
             return ty.fold_with(self);
@@ -76,7 +71,7 @@ impl<'a, 'tcx> TypeFreshener<'a, 'tcx> {
             Entry::Vacant(entry) => {
                 let index = self.ty_freshen_count;
                 self.ty_freshen_count += 1;
-                let t = self.infcx.tcx.mk_ty_infer(freshener(index));
+                let t = mk_fresh(index);
                 entry.insert(t);
                 t
             }
@@ -110,8 +105,8 @@ impl<'a, 'tcx> TypeFreshener<'a, 'tcx> {
     }
 }
 
-impl<'a, 'tcx> TypeFolder<'tcx> for TypeFreshener<'a, 'tcx> {
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for TypeFreshener<'a, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.infcx.tcx
     }
 
@@ -126,92 +121,36 @@ impl<'a, 'tcx> TypeFolder<'tcx> for TypeFreshener<'a, 'tcx> {
             | ty::ReFree(_)
             | ty::ReVar(_)
             | ty::RePlaceholder(..)
+            | ty::ReError(_)
             | ty::ReErased => {
                 // replace all free regions with 'erased
-                self.tcx().lifetimes.re_erased
+                self.interner().lifetimes.re_erased
             }
             ty::ReStatic => {
                 if self.keep_static {
                     r
                 } else {
-                    self.tcx().lifetimes.re_erased
+                    self.interner().lifetimes.re_erased
                 }
             }
         }
     }
 
+    #[inline]
     fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
         if !t.needs_infer() && !t.has_erasable_regions() {
-            return t;
-        }
+            t
+        } else {
+            match *t.kind() {
+                ty::Infer(v) => self.fold_infer_ty(v).unwrap_or(t),
 
-        let tcx = self.infcx.tcx;
+                // This code is hot enough that a non-debug assertion here makes a noticeable
+                // difference on benchmarks like `wg-grammar`.
+                #[cfg(debug_assertions)]
+                ty::Placeholder(..) | ty::Bound(..) => bug!("unexpected type {:?}", t),
 
-        match *t.kind() {
-            ty::Infer(ty::TyVar(v)) => {
-                let opt_ty = self.infcx.inner.borrow_mut().type_variables().probe(v).known();
-                self.freshen_ty(opt_ty, ty::TyVar(v), ty::FreshTy)
+                _ => t.super_fold_with(self),
             }
-
-            ty::Infer(ty::IntVar(v)) => self.freshen_ty(
-                self.infcx
-                    .inner
-                    .borrow_mut()
-                    .int_unification_table()
-                    .probe_value(v)
-                    .map(|v| v.to_type(tcx)),
-                ty::IntVar(v),
-                ty::FreshIntTy,
-            ),
-
-            ty::Infer(ty::FloatVar(v)) => self.freshen_ty(
-                self.infcx
-                    .inner
-                    .borrow_mut()
-                    .float_unification_table()
-                    .probe_value(v)
-                    .map(|v| v.to_type(tcx)),
-                ty::FloatVar(v),
-                ty::FreshFloatTy,
-            ),
-
-            ty::Infer(ty::FreshTy(ct) | ty::FreshIntTy(ct) | ty::FreshFloatTy(ct)) => {
-                if ct >= self.ty_freshen_count {
-                    bug!(
-                        "Encountered a freshend type with id {} \
-                          but our counter is only at {}",
-                        ct,
-                        self.ty_freshen_count
-                    );
-                }
-                t
-            }
-
-            ty::Generator(..)
-            | ty::Bool
-            | ty::Char
-            | ty::Int(..)
-            | ty::Uint(..)
-            | ty::Float(..)
-            | ty::Adt(..)
-            | ty::Str
-            | ty::Error(_)
-            | ty::Array(..)
-            | ty::Slice(..)
-            | ty::RawPtr(..)
-            | ty::Ref(..)
-            | ty::FnDef(..)
-            | ty::FnPtr(_)
-            | ty::Dynamic(..)
-            | ty::Never
-            | ty::Tuple(..)
-            | ty::Alias(..)
-            | ty::Foreign(..)
-            | ty::Param(..)
-            | ty::Closure(..)
-            | ty::GeneratorWitness(..) => t.super_fold_with(self),
-
-            ty::Placeholder(..) | ty::Bound(..) => bug!("unexpected type {:?}", t),
         }
     }
 
@@ -252,3 +191,54 @@ impl<'a, 'tcx> TypeFolder<'tcx> for TypeFreshener<'a, 'tcx> {
         }
     }
 }
+
+impl<'a, 'tcx> TypeFreshener<'a, 'tcx> {
+    // This is separate from `fold_ty` to keep that method small and inlinable.
+    #[inline(never)]
+    fn fold_infer_ty(&mut self, v: ty::InferTy) -> Option<Ty<'tcx>> {
+        match v {
+            ty::TyVar(v) => {
+                let opt_ty = self.infcx.inner.borrow_mut().type_variables().probe(v).known();
+                Some(self.freshen_ty(opt_ty, ty::TyVar(v), |n| self.infcx.tcx.mk_fresh_ty(n)))
+            }
+
+            ty::IntVar(v) => Some(
+                self.freshen_ty(
+                    self.infcx
+                        .inner
+                        .borrow_mut()
+                        .int_unification_table()
+                        .probe_value(v)
+                        .map(|v| v.to_type(self.infcx.tcx)),
+                    ty::IntVar(v),
+                    |n| self.infcx.tcx.mk_fresh_int_ty(n),
+                ),
+            ),
+
+            ty::FloatVar(v) => Some(
+                self.freshen_ty(
+                    self.infcx
+                        .inner
+                        .borrow_mut()
+                        .float_unification_table()
+                        .probe_value(v)
+                        .map(|v| v.to_type(self.infcx.tcx)),
+                    ty::FloatVar(v),
+                    |n| self.infcx.tcx.mk_fresh_float_ty(n),
+                ),
+            ),
+
+            ty::FreshTy(ct) | ty::FreshIntTy(ct) | ty::FreshFloatTy(ct) => {
+                if ct >= self.ty_freshen_count {
+                    bug!(
+                        "Encountered a freshend type with id {} \
+                          but our counter is only at {}",
+                        ct,
+                        self.ty_freshen_count
+                    );
+                }
+                None
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_infer/src/infer/fudge.rs b/compiler/rustc_infer/src/infer/fudge.rs
index 6dd6c4e1f..86c2c2be4 100644
--- a/compiler/rustc_infer/src/infer/fudge.rs
+++ b/compiler/rustc_infer/src/infer/fudge.rs
@@ -98,7 +98,7 @@ impl<'tcx> InferCtxt<'tcx> {
     pub fn fudge_inference_if_ok<T, E, F>(&self, f: F) -> Result<T, E>
     where
         F: FnOnce() -> Result<T, E>,
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         let variable_lengths = self.variable_lengths();
         let (mut fudger, value) = self.probe(|_| {
@@ -175,8 +175,8 @@ pub struct InferenceFudger<'a, 'tcx> {
     const_vars: (Range<ConstVid<'tcx>>, Vec<ConstVariableOrigin>),
 }
 
-impl<'a, 'tcx> TypeFolder<'tcx> for InferenceFudger<'a, 'tcx> {
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for InferenceFudger<'a, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.infcx.tcx
     }
 
diff --git a/compiler/rustc_infer/src/infer/glb.rs b/compiler/rustc_infer/src/infer/glb.rs
index 21b68ce99..49df393d8 100644
--- a/compiler/rustc_infer/src/infer/glb.rs
+++ b/compiler/rustc_infer/src/infer/glb.rs
@@ -1,14 +1,13 @@
 //! Greatest lower bound. See [`lattice`].
 
-use super::combine::CombineFields;
+use super::combine::{CombineFields, ObligationEmittingRelation};
 use super::lattice::{self, LatticeDir};
 use super::InferCtxt;
 use super::Subtype;
 
-use crate::infer::combine::ConstEquateRelation;
-use crate::traits::{ObligationCause, PredicateObligation};
+use crate::traits::{ObligationCause, PredicateObligations};
 use rustc_middle::ty::relate::{Relate, RelateResult, TypeRelation};
-use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitableExt};
 
 /// "Greatest lower bound" (common subtype)
 pub struct Glb<'combine, 'infcx, 'tcx> {
@@ -79,7 +78,8 @@ impl<'tcx> TypeRelation<'tcx> for Glb<'_, '_, 'tcx> {
         debug!("{}.regions({:?}, {:?})", self.tag(), a, b);
 
         let origin = Subtype(Box::new(self.fields.trace.clone()));
-        Ok(self.fields.infcx.inner.borrow_mut().unwrap_region_constraints().glb_regions(
+        // GLB(&'static u8, &'a u8) == &RegionLUB('static, 'a) u8 == &'static u8
+        Ok(self.fields.infcx.inner.borrow_mut().unwrap_region_constraints().lub_regions(
             self.tcx(),
             origin,
             a,
@@ -135,10 +135,6 @@ impl<'combine, 'infcx, 'tcx> LatticeDir<'infcx, 'tcx> for Glb<'combine, 'infcx,
         &self.fields.trace.cause
     }
 
-    fn add_obligations(&mut self, obligations: Vec<PredicateObligation<'tcx>>) {
-        self.fields.obligations.extend(obligations)
-    }
-
     fn relate_bound(&mut self, v: Ty<'tcx>, a: Ty<'tcx>, b: Ty<'tcx>) -> RelateResult<'tcx, ()> {
         let mut sub = self.fields.sub(self.a_is_expected);
         sub.relate(v, a)?;
@@ -151,8 +147,12 @@ impl<'combine, 'infcx, 'tcx> LatticeDir<'infcx, 'tcx> for Glb<'combine, 'infcx,
     }
 }
 
-impl<'tcx> ConstEquateRelation<'tcx> for Glb<'_, '_, 'tcx> {
-    fn const_equate_obligation(&mut self, a: ty::Const<'tcx>, b: ty::Const<'tcx>) {
-        self.fields.add_const_equate_obligation(self.a_is_expected, a, b);
+impl<'tcx> ObligationEmittingRelation<'tcx> for Glb<'_, '_, 'tcx> {
+    fn register_predicates(&mut self, obligations: impl IntoIterator<Item: ty::ToPredicate<'tcx>>) {
+        self.fields.register_predicates(obligations);
+    }
+
+    fn register_obligations(&mut self, obligations: PredicateObligations<'tcx>) {
+        self.fields.register_obligations(obligations);
     }
 }
diff --git a/compiler/rustc_infer/src/infer/higher_ranked/mod.rs b/compiler/rustc_infer/src/infer/higher_ranked/mod.rs
index 817ae10c7..d1897cf24 100644
--- a/compiler/rustc_infer/src/infer/higher_ranked/mod.rs
+++ b/compiler/rustc_infer/src/infer/higher_ranked/mod.rs
@@ -6,7 +6,7 @@ use super::{HigherRankedType, InferCtxt};
 use crate::infer::CombinedSnapshot;
 use rustc_middle::ty::fold::FnMutDelegate;
 use rustc_middle::ty::relate::{Relate, RelateResult, TypeRelation};
-use rustc_middle::ty::{self, Binder, TypeFoldable};
+use rustc_middle::ty::{self, Binder, TyCtxt, TypeFoldable};
 
 impl<'a, 'tcx> CombineFields<'a, 'tcx> {
     /// Checks whether `for<..> sub <: for<..> sup` holds.
@@ -38,13 +38,13 @@ impl<'a, 'tcx> CombineFields<'a, 'tcx> {
         // First, we instantiate each bound region in the supertype with a
         // fresh placeholder region. Note that this automatically creates
         // a new universe if needed.
-        let sup_prime = self.infcx.replace_bound_vars_with_placeholders(sup);
+        let sup_prime = self.infcx.instantiate_binder_with_placeholders(sup);
 
         // Next, we instantiate each bound region in the subtype
         // with a fresh region variable. These region variables --
         // but no other pre-existing region variables -- can name
         // the placeholders.
-        let sub_prime = self.infcx.replace_bound_vars_with_fresh_vars(span, HigherRankedType, sub);
+        let sub_prime = self.infcx.instantiate_binder_with_fresh_vars(span, HigherRankedType, sub);
 
         debug!("a_prime={:?}", sub_prime);
         debug!("b_prime={:?}", sup_prime);
@@ -70,9 +70,9 @@ impl<'tcx> InferCtxt<'tcx> {
     ///
     /// [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/traits/hrtb.html
     #[instrument(level = "debug", skip(self), ret)]
-    pub fn replace_bound_vars_with_placeholders<T>(&self, binder: ty::Binder<'tcx, T>) -> T
+    pub fn instantiate_binder_with_placeholders<T>(&self, binder: ty::Binder<'tcx, T>) -> T
     where
-        T: TypeFoldable<'tcx> + Copy,
+        T: TypeFoldable<TyCtxt<'tcx>> + Copy,
     {
         if let Some(inner) = binder.no_bound_vars() {
             return inner;
@@ -82,16 +82,16 @@ impl<'tcx> InferCtxt<'tcx> {
 
         let delegate = FnMutDelegate {
             regions: &mut |br: ty::BoundRegion| {
-                self.tcx.mk_region(ty::RePlaceholder(ty::PlaceholderRegion {
+                self.tcx.mk_re_placeholder(ty::PlaceholderRegion {
                     universe: next_universe,
                     name: br.kind,
-                }))
+                })
             },
             types: &mut |bound_ty: ty::BoundTy| {
-                self.tcx.mk_ty(ty::Placeholder(ty::PlaceholderType {
+                self.tcx.mk_placeholder(ty::PlaceholderType {
                     universe: next_universe,
-                    name: bound_ty.var,
-                }))
+                    name: bound_ty.kind,
+                })
             },
             consts: &mut |bound_var: ty::BoundVar, ty| {
                 self.tcx
diff --git a/compiler/rustc_infer/src/infer/lattice.rs b/compiler/rustc_infer/src/infer/lattice.rs
index 4dbb4b4d7..f377ac1d1 100644
--- a/compiler/rustc_infer/src/infer/lattice.rs
+++ b/compiler/rustc_infer/src/infer/lattice.rs
@@ -17,11 +17,12 @@
 //!
 //! [lattices]: https://en.wikipedia.org/wiki/Lattice_(order)
 
+use super::combine::ObligationEmittingRelation;
 use super::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
 use super::InferCtxt;
 
-use crate::traits::{ObligationCause, PredicateObligation};
-use rustc_middle::ty::relate::{RelateResult, TypeRelation};
+use crate::traits::ObligationCause;
+use rustc_middle::ty::relate::RelateResult;
 use rustc_middle::ty::TyVar;
 use rustc_middle::ty::{self, Ty};
 
@@ -30,13 +31,11 @@ use rustc_middle::ty::{self, Ty};
 ///
 /// GLB moves "down" the lattice (to smaller values); LUB moves
 /// "up" the lattice (to bigger values).
-pub trait LatticeDir<'f, 'tcx>: TypeRelation<'tcx> {
+pub trait LatticeDir<'f, 'tcx>: ObligationEmittingRelation<'tcx> {
     fn infcx(&self) -> &'f InferCtxt<'tcx>;
 
     fn cause(&self) -> &ObligationCause<'tcx>;
 
-    fn add_obligations(&mut self, obligations: Vec<PredicateObligation<'tcx>>);
-
     fn define_opaque_types(&self) -> bool;
 
     // Relates the type `v` to `a` and `b` such that `v` represents
@@ -113,7 +112,7 @@ where
         | (_, &ty::Alias(ty::Opaque, ty::AliasTy { def_id, .. }))
             if this.define_opaque_types() && def_id.is_local() =>
         {
-            this.add_obligations(
+            this.register_obligations(
                 infcx
                     .handle_opaque_type(a, b, this.a_is_expected(), this.cause(), this.param_env())?
                     .obligations,
diff --git a/compiler/rustc_infer/src/infer/lexical_region_resolve/mod.rs b/compiler/rustc_infer/src/infer/lexical_region_resolve/mod.rs
index ce8aec804..2c4803550 100644
--- a/compiler/rustc_infer/src/infer/lexical_region_resolve/mod.rs
+++ b/compiler/rustc_infer/src/infer/lexical_region_resolve/mod.rs
@@ -17,7 +17,7 @@ use rustc_index::vec::{Idx, IndexVec};
 use rustc_middle::ty::fold::TypeFoldable;
 use rustc_middle::ty::PlaceholderRegion;
 use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_middle::ty::{ReEarlyBound, ReErased, ReFree, ReStatic};
+use rustc_middle::ty::{ReEarlyBound, ReErased, ReError, ReFree, ReStatic};
 use rustc_middle::ty::{ReLateBound, RePlaceholder, ReVar};
 use rustc_middle::ty::{Region, RegionVid};
 use rustc_span::Span;
@@ -70,7 +70,7 @@ pub enum RegionResolutionError<'tcx> {
     /// `o` requires that `a <= b`, but this does not hold
     ConcreteFailure(SubregionOrigin<'tcx>, Region<'tcx>, Region<'tcx>),
 
-    /// `GenericBoundFailure(p, s, a)
+    /// `GenericBoundFailure(p, s, a)`:
     ///
     /// The parameter/associated-type `p` must be known to outlive the lifetime
     /// `a` (but none of the known bounds are sufficient).
@@ -216,6 +216,8 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
                 Ok(self.tcx().lifetimes.re_static)
             }
 
+            ReError(_) => Ok(a_region),
+
             ReEarlyBound(_) | ReFree(_) => {
                 // All empty regions are less than early-bound, free,
                 // and scope regions.
@@ -380,7 +382,7 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
                     // name the placeholder, then the placeholder is
                     // larger; otherwise, the only ancestor is `'static`.
                     Err(placeholder) if empty_ui.can_name(placeholder.universe) => {
-                        self.tcx().mk_region(RePlaceholder(placeholder))
+                        self.tcx().mk_re_placeholder(placeholder)
                     }
                     Err(_) => self.tcx().lifetimes.re_static,
                 };
@@ -436,6 +438,10 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
             }
             (VarValue::Value(a), VarValue::Empty(_)) => {
                 match *a {
+                    // this is always on an error path,
+                    // so it doesn't really matter if it's shorter or longer than an empty region
+                    ReError(_) => false,
+
                     ReLateBound(..) | ReErased => {
                         bug!("cannot relate region: {:?}", a);
                     }
@@ -465,6 +471,10 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
             }
             (VarValue::Empty(a_ui), VarValue::Value(b)) => {
                 match *b {
+                    // this is always on an error path,
+                    // so it doesn't really matter if it's shorter or longer than an empty region
+                    ReError(_) => false,
+
                     ReLateBound(..) | ReErased => {
                         bug!("cannot relate region: {:?}", b);
                     }
@@ -546,6 +556,10 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
                 );
             }
 
+            (ReError(_), _) => a,
+
+            (_, ReError(_)) => b,
+
             (ReStatic, _) | (_, ReStatic) => {
                 // nothing lives longer than `'static`
                 self.tcx().lifetimes.re_static
@@ -1018,7 +1032,7 @@ impl<'tcx> fmt::Debug for RegionAndOrigin<'tcx> {
 impl<'tcx> LexicalRegionResolutions<'tcx> {
     fn normalize<T>(&self, tcx: TyCtxt<'tcx>, value: T) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         tcx.fold_regions(value, |r, _db| self.resolve_region(tcx, r))
     }
diff --git a/compiler/rustc_infer/src/infer/lub.rs b/compiler/rustc_infer/src/infer/lub.rs
index c07ac1d3a..c871ccb21 100644
--- a/compiler/rustc_infer/src/infer/lub.rs
+++ b/compiler/rustc_infer/src/infer/lub.rs
@@ -1,14 +1,13 @@
 //! Least upper bound. See [`lattice`].
 
-use super::combine::CombineFields;
+use super::combine::{CombineFields, ObligationEmittingRelation};
 use super::lattice::{self, LatticeDir};
 use super::InferCtxt;
 use super::Subtype;
 
-use crate::infer::combine::ConstEquateRelation;
-use crate::traits::{ObligationCause, PredicateObligation};
+use crate::traits::{ObligationCause, PredicateObligations};
 use rustc_middle::ty::relate::{Relate, RelateResult, TypeRelation};
-use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitableExt};
 
 /// "Least upper bound" (common supertype)
 pub struct Lub<'combine, 'infcx, 'tcx> {
@@ -79,7 +78,8 @@ impl<'tcx> TypeRelation<'tcx> for Lub<'_, '_, 'tcx> {
         debug!("{}.regions({:?}, {:?})", self.tag(), a, b);
 
         let origin = Subtype(Box::new(self.fields.trace.clone()));
-        Ok(self.fields.infcx.inner.borrow_mut().unwrap_region_constraints().lub_regions(
+        // LUB(&'static u8, &'a u8) == &RegionGLB('static, 'a) u8 == &'a u8
+        Ok(self.fields.infcx.inner.borrow_mut().unwrap_region_constraints().glb_regions(
             self.tcx(),
             origin,
             a,
@@ -126,12 +126,6 @@ impl<'tcx> TypeRelation<'tcx> for Lub<'_, '_, 'tcx> {
     }
 }
 
-impl<'tcx> ConstEquateRelation<'tcx> for Lub<'_, '_, 'tcx> {
-    fn const_equate_obligation(&mut self, a: ty::Const<'tcx>, b: ty::Const<'tcx>) {
-        self.fields.add_const_equate_obligation(self.a_is_expected, a, b);
-    }
-}
-
 impl<'combine, 'infcx, 'tcx> LatticeDir<'infcx, 'tcx> for Lub<'combine, 'infcx, 'tcx> {
     fn infcx(&self) -> &'infcx InferCtxt<'tcx> {
         self.fields.infcx
@@ -141,10 +135,6 @@ impl<'combine, 'infcx, 'tcx> LatticeDir<'infcx, 'tcx> for Lub<'combine, 'infcx,
         &self.fields.trace.cause
     }
 
-    fn add_obligations(&mut self, obligations: Vec<PredicateObligation<'tcx>>) {
-        self.fields.obligations.extend(obligations)
-    }
-
     fn relate_bound(&mut self, v: Ty<'tcx>, a: Ty<'tcx>, b: Ty<'tcx>) -> RelateResult<'tcx, ()> {
         let mut sub = self.fields.sub(self.a_is_expected);
         sub.relate(a, v)?;
@@ -156,3 +146,13 @@ impl<'combine, 'infcx, 'tcx> LatticeDir<'infcx, 'tcx> for Lub<'combine, 'infcx,
         self.fields.define_opaque_types
     }
 }
+
+impl<'tcx> ObligationEmittingRelation<'tcx> for Lub<'_, '_, 'tcx> {
+    fn register_predicates(&mut self, obligations: impl IntoIterator<Item: ty::ToPredicate<'tcx>>) {
+        self.fields.register_predicates(obligations);
+    }
+
+    fn register_obligations(&mut self, obligations: PredicateObligations<'tcx>) {
+        self.fields.register_obligations(obligations)
+    }
+}
diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs
index f0e42c1fc..bd1f96635 100644
--- a/compiler/rustc_infer/src/infer/mod.rs
+++ b/compiler/rustc_infer/src/infer/mod.rs
@@ -4,6 +4,7 @@ pub use self::LateBoundRegionConversionTime::*;
 pub use self::RegionVariableOrigin::*;
 pub use self::SubregionOrigin::*;
 pub use self::ValuePairs::*;
+pub use combine::ObligationEmittingRelation;
 
 use self::opaque_types::OpaqueTypeStorage;
 pub(crate) use self::undo_log::{InferCtxtUndoLogs, Snapshot, UndoLog};
@@ -28,9 +29,9 @@ use rustc_middle::ty::fold::BoundVarReplacerDelegate;
 use rustc_middle::ty::fold::{TypeFoldable, TypeFolder, TypeSuperFoldable};
 use rustc_middle::ty::relate::RelateResult;
 use rustc_middle::ty::subst::{GenericArg, GenericArgKind, InternalSubsts, SubstsRef};
-use rustc_middle::ty::visit::TypeVisitable;
+use rustc_middle::ty::visit::{TypeVisitable, TypeVisitableExt};
 pub use rustc_middle::ty::IntVarValue;
-use rustc_middle::ty::{self, GenericParamDefKind, InferConst, Ty, TyCtxt};
+use rustc_middle::ty::{self, GenericParamDefKind, InferConst, InferTy, Ty, TyCtxt};
 use rustc_middle::ty::{ConstVid, FloatVid, IntVid, TyVid};
 use rustc_span::symbol::Symbol;
 use rustc_span::Span;
@@ -155,7 +156,7 @@ pub struct InferCtxtInner<'tcx> {
     undo_log: InferCtxtUndoLogs<'tcx>,
 
     /// Caches for opaque type inference.
-    pub opaque_type_storage: OpaqueTypeStorage<'tcx>,
+    opaque_type_storage: OpaqueTypeStorage<'tcx>,
 }
 
 impl<'tcx> InferCtxtInner<'tcx> {
@@ -194,41 +195,17 @@ impl<'tcx> InferCtxtInner<'tcx> {
     }
 
     #[inline]
-    fn int_unification_table(
-        &mut self,
-    ) -> ut::UnificationTable<
-        ut::InPlace<
-            ty::IntVid,
-            &mut ut::UnificationStorage<ty::IntVid>,
-            &mut InferCtxtUndoLogs<'tcx>,
-        >,
-    > {
+    fn int_unification_table(&mut self) -> UnificationTable<'_, 'tcx, ty::IntVid> {
         self.int_unification_storage.with_log(&mut self.undo_log)
     }
 
     #[inline]
-    fn float_unification_table(
-        &mut self,
-    ) -> ut::UnificationTable<
-        ut::InPlace<
-            ty::FloatVid,
-            &mut ut::UnificationStorage<ty::FloatVid>,
-            &mut InferCtxtUndoLogs<'tcx>,
-        >,
-    > {
+    fn float_unification_table(&mut self) -> UnificationTable<'_, 'tcx, ty::FloatVid> {
         self.float_unification_storage.with_log(&mut self.undo_log)
     }
 
     #[inline]
-    fn const_unification_table(
-        &mut self,
-    ) -> ut::UnificationTable<
-        ut::InPlace<
-            ty::ConstVid<'tcx>,
-            &mut ut::UnificationStorage<ty::ConstVid<'tcx>>,
-            &mut InferCtxtUndoLogs<'tcx>,
-        >,
-    > {
+    fn const_unification_table(&mut self) -> UnificationTable<'_, 'tcx, ty::ConstVid<'tcx>> {
         self.const_unification_storage.with_log(&mut self.undo_log)
     }
 
@@ -263,7 +240,7 @@ pub struct InferCtxt<'tcx> {
     /// short lived InferCtxt within queries. The opaque type obligations are forwarded
     /// to the outside until the end up in an `InferCtxt` for typeck or borrowck.
     ///
-    /// It is default value is `DefiningAnchor::Error`, this way it is easier to catch errors that
+    /// Its default value is `DefiningAnchor::Error`, this way it is easier to catch errors that
     /// might come up during inference or typeck.
     pub defining_use_anchor: DefiningAnchor,
 
@@ -616,7 +593,7 @@ impl<'tcx> InferCtxtBuilder<'tcx> {
         canonical: &Canonical<'tcx, T>,
     ) -> (InferCtxt<'tcx>, T, CanonicalVarValues<'tcx>)
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         let infcx = self.build();
         let (value, subst) = infcx.instantiate_canonical_with_fresh_inference_vars(span, canonical);
@@ -696,7 +673,7 @@ impl<'tcx> InferCtxt<'tcx> {
         self.in_snapshot.get()
     }
 
-    pub fn freshen<T: TypeFoldable<'tcx>>(&self, t: T) -> T {
+    pub fn freshen<T: TypeFoldable<TyCtxt<'tcx>>>(&self, t: T) -> T {
         t.fold_with(&mut self.freshener())
     }
 
@@ -879,30 +856,20 @@ impl<'tcx> InferCtxt<'tcx> {
         self.inner.borrow_mut().unwrap_region_constraints().add_given(sub, sup);
     }
 
-    pub fn can_sub<T>(&self, param_env: ty::ParamEnv<'tcx>, a: T, b: T) -> UnitResult<'tcx>
+    pub fn can_sub<T>(&self, param_env: ty::ParamEnv<'tcx>, a: T, b: T) -> bool
     where
         T: at::ToTrace<'tcx>,
     {
         let origin = &ObligationCause::dummy();
-        self.probe(|_| {
-            self.at(origin, param_env).sub(a, b).map(|InferOk { obligations: _, .. }| {
-                // Ignore obligations, since we are unrolling
-                // everything anyway.
-            })
-        })
+        self.probe(|_| self.at(origin, param_env).sub(a, b).is_ok())
     }
 
-    pub fn can_eq<T>(&self, param_env: ty::ParamEnv<'tcx>, a: T, b: T) -> UnitResult<'tcx>
+    pub fn can_eq<T>(&self, param_env: ty::ParamEnv<'tcx>, a: T, b: T) -> bool
     where
         T: at::ToTrace<'tcx>,
     {
         let origin = &ObligationCause::dummy();
-        self.probe(|_| {
-            self.at(origin, param_env).eq(a, b).map(|InferOk { obligations: _, .. }| {
-                // Ignore obligations, since we are unrolling
-                // everything anyway.
-            })
-        })
+        self.probe(|_| self.at(origin, param_env).eq(a, b).is_ok())
     }
 
     #[instrument(skip(self), level = "debug")]
@@ -995,7 +962,7 @@ impl<'tcx> InferCtxt<'tcx> {
 
         Ok(self.commit_if_ok(|_snapshot| {
             let ty::SubtypePredicate { a_is_expected, a, b } =
-                self.replace_bound_vars_with_placeholders(predicate);
+                self.instantiate_binder_with_placeholders(predicate);
 
             let ok = self.at(cause, param_env).sub_exp(a_is_expected, a, b)?;
 
@@ -1008,7 +975,7 @@ impl<'tcx> InferCtxt<'tcx> {
         cause: &traits::ObligationCause<'tcx>,
         predicate: ty::PolyRegionOutlivesPredicate<'tcx>,
     ) {
-        let ty::OutlivesPredicate(r_a, r_b) = self.replace_bound_vars_with_placeholders(predicate);
+        let ty::OutlivesPredicate(r_a, r_b) = self.instantiate_binder_with_placeholders(predicate);
         let origin =
             SubregionOrigin::from_obligation_cause(cause, || RelateRegionParamBound(cause.span));
         self.sub_regions(origin, r_b, r_a); // `b : a` ==> `a <= b`
@@ -1102,7 +1069,7 @@ impl<'tcx> InferCtxt<'tcx> {
     ) -> ty::Region<'tcx> {
         let region_var =
             self.inner.borrow_mut().unwrap_region_constraints().new_region_var(universe, origin);
-        self.tcx.mk_region(ty::ReVar(region_var))
+        self.tcx.mk_re_var(region_var)
     }
 
     /// Return the universe that the region `r` was created in. For
@@ -1120,11 +1087,13 @@ impl<'tcx> InferCtxt<'tcx> {
     }
 
     /// Just a convenient wrapper of `next_region_var` for using during NLL.
+    #[instrument(skip(self), level = "debug")]
     pub fn next_nll_region_var(&self, origin: NllRegionVariableOrigin) -> ty::Region<'tcx> {
         self.next_region_var(RegionVariableOrigin::Nll(origin))
     }
 
     /// Just a convenient wrapper of `next_region_var` for using during NLL.
+    #[instrument(skip(self), level = "debug")]
     pub fn next_nll_region_var_in_universe(
         &self,
         origin: NllRegionVariableOrigin,
@@ -1175,7 +1144,15 @@ impl<'tcx> InferCtxt<'tcx> {
                         origin,
                         val: ConstVariableValue::Unknown { universe: self.universe() },
                     });
-                self.tcx.mk_const(const_var_id, self.tcx.type_of(param.def_id)).into()
+                self.tcx
+                    .mk_const(
+                        const_var_id,
+                        self.tcx
+                            .type_of(param.def_id)
+                            .no_bound_vars()
+                            .expect("const parameter types cannot be generic"),
+                    )
+                    .into()
             }
         }
     }
@@ -1370,7 +1347,7 @@ impl<'tcx> InferCtxt<'tcx> {
     /// will be resolving them as well, e.g. in a loop).
     pub fn shallow_resolve<T>(&self, value: T) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         value.fold_with(&mut ShallowResolver { infcx: self })
     }
@@ -1387,10 +1364,10 @@ impl<'tcx> InferCtxt<'tcx> {
     /// at will.
     pub fn resolve_vars_if_possible<T>(&self, value: T) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
-        if !value.needs_infer() {
-            return value; // Avoid duplicated subst-folding.
+        if !value.has_non_region_infer() {
+            return value;
         }
         let mut r = resolve::OpportunisticVarResolver::new(self);
         value.fold_with(&mut r)
@@ -1398,7 +1375,7 @@ impl<'tcx> InferCtxt<'tcx> {
 
     pub fn resolve_numeric_literals_with_default<T>(&self, value: T) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         if !value.needs_infer() {
             return value; // Avoid duplicated subst-folding.
@@ -1413,7 +1390,7 @@ impl<'tcx> InferCtxt<'tcx> {
         value: &T,
     ) -> Option<(ty::Term<'tcx>, Option<Span>)>
     where
-        T: TypeVisitable<'tcx>,
+        T: TypeVisitable<TyCtxt<'tcx>>,
     {
         value.visit_with(&mut resolve::UnresolvedTypeOrConstFinder::new(self)).break_value()
     }
@@ -1428,17 +1405,14 @@ impl<'tcx> InferCtxt<'tcx> {
         }
     }
 
-    pub fn fully_resolve<T: TypeFoldable<'tcx>>(&self, value: T) -> FixupResult<'tcx, T> {
-        /*!
-         * Attempts to resolve all type/region/const variables in
-         * `value`. Region inference must have been run already (e.g.,
-         * by calling `resolve_regions_and_report_errors`). If some
-         * variable was never unified, an `Err` results.
-         *
-         * This method is idempotent, but it not typically not invoked
-         * except during the writeback phase.
-         */
-
+    /// Attempts to resolve all type/region/const variables in
+    /// `value`. Region inference must have been run already (e.g.,
+    /// by calling `resolve_regions_and_report_errors`). If some
+    /// variable was never unified, an `Err` results.
+    ///
+    /// This method is idempotent, but it not typically not invoked
+    /// except during the writeback phase.
+    pub fn fully_resolve<T: TypeFoldable<TyCtxt<'tcx>>>(&self, value: T) -> FixupResult<'tcx, T> {
         let value = resolve::fully_resolve(self, value);
         assert!(
             value.as_ref().map_or(true, |value| !value.needs_infer()),
@@ -1447,14 +1421,21 @@ impl<'tcx> InferCtxt<'tcx> {
         value
     }
 
-    pub fn replace_bound_vars_with_fresh_vars<T>(
+    // Instantiates the bound variables in a given binder with fresh inference
+    // variables in the current universe.
+    //
+    // Use this method if you'd like to find some substitution of the binder's
+    // variables (e.g. during a method call). If there isn't a [`LateBoundRegionConversionTime`]
+    // that corresponds to your use case, consider whether or not you should
+    // use [`InferCtxt::instantiate_binder_with_placeholders`] instead.
+    pub fn instantiate_binder_with_fresh_vars<T>(
         &self,
         span: Span,
         lbrct: LateBoundRegionConversionTime,
         value: ty::Binder<'tcx, T>,
     ) -> T
     where
-        T: TypeFoldable<'tcx> + Copy,
+        T: TypeFoldable<TyCtxt<'tcx>> + Copy,
     {
         if let Some(inner) = value.no_bound_vars() {
             return inner;
@@ -1746,7 +1727,6 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
     // in this case. The typechecker should only ever report type errors involving mismatched
     // types using one of these methods, and should not call span_err directly for such
     // errors.
-
     pub fn type_error_struct_with_diag<M>(
         &self,
         sp: Span,
@@ -1844,8 +1824,8 @@ struct InferenceLiteralEraser<'tcx> {
     tcx: TyCtxt<'tcx>,
 }
 
-impl<'tcx> TypeFolder<'tcx> for InferenceLiteralEraser<'tcx> {
-    fn tcx(&self) -> TyCtxt<'tcx> {
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for InferenceLiteralEraser<'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
@@ -1862,17 +1842,41 @@ struct ShallowResolver<'a, 'tcx> {
     infcx: &'a InferCtxt<'tcx>,
 }
 
-impl<'a, 'tcx> TypeFolder<'tcx> for ShallowResolver<'a, 'tcx> {
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for ShallowResolver<'a, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.infcx.tcx
     }
 
     /// If `ty` is a type variable of some kind, resolve it one level
     /// (but do not resolve types found in the result). If `typ` is
     /// not a type variable, just return it unmodified.
+    #[inline]
     fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
-        match *ty.kind() {
-            ty::Infer(ty::TyVar(v)) => {
+        if let ty::Infer(v) = ty.kind() { self.fold_infer_ty(*v).unwrap_or(ty) } else { ty }
+    }
+
+    fn fold_const(&mut self, ct: ty::Const<'tcx>) -> ty::Const<'tcx> {
+        if let ty::ConstKind::Infer(InferConst::Var(vid)) = ct.kind() {
+            self.infcx
+                .inner
+                .borrow_mut()
+                .const_unification_table()
+                .probe_value(vid)
+                .val
+                .known()
+                .unwrap_or(ct)
+        } else {
+            ct
+        }
+    }
+}
+
+impl<'a, 'tcx> ShallowResolver<'a, 'tcx> {
+    // This is separate from `fold_ty` to keep that method small and inlinable.
+    #[inline(never)]
+    fn fold_infer_ty(&mut self, v: InferTy) -> Option<Ty<'tcx>> {
+        match v {
+            ty::TyVar(v) => {
                 // Not entirely obvious: if `typ` is a type variable,
                 // it can be resolved to an int/float variable, which
                 // can then be recursively resolved, hence the
@@ -1886,41 +1890,26 @@ impl<'a, 'tcx> TypeFolder<'tcx> for ShallowResolver<'a, 'tcx> {
                 // Note: if these two lines are combined into one we get
                 // dynamic borrow errors on `self.inner`.
                 let known = self.infcx.inner.borrow_mut().type_variables().probe(v).known();
-                known.map_or(ty, |t| self.fold_ty(t))
+                known.map(|t| self.fold_ty(t))
             }
 
-            ty::Infer(ty::IntVar(v)) => self
+            ty::IntVar(v) => self
                 .infcx
                 .inner
                 .borrow_mut()
                 .int_unification_table()
                 .probe_value(v)
-                .map_or(ty, |v| v.to_type(self.infcx.tcx)),
+                .map(|v| v.to_type(self.infcx.tcx)),
 
-            ty::Infer(ty::FloatVar(v)) => self
+            ty::FloatVar(v) => self
                 .infcx
                 .inner
                 .borrow_mut()
                 .float_unification_table()
                 .probe_value(v)
-                .map_or(ty, |v| v.to_type(self.infcx.tcx)),
-
-            _ => ty,
-        }
-    }
+                .map(|v| v.to_type(self.infcx.tcx)),
 
-    fn fold_const(&mut self, ct: ty::Const<'tcx>) -> ty::Const<'tcx> {
-        if let ty::ConstKind::Infer(InferConst::Var(vid)) = ct.kind() {
-            self.infcx
-                .inner
-                .borrow_mut()
-                .const_unification_table()
-                .probe_value(vid)
-                .val
-                .known()
-                .unwrap_or(ct)
-        } else {
-            ct
+            ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_) => None,
         }
     }
 }
@@ -2044,24 +2033,24 @@ fn replace_param_and_infer_substs_with_placeholder<'tcx>(
 ) -> SubstsRef<'tcx> {
     struct ReplaceParamAndInferWithPlaceholder<'tcx> {
         tcx: TyCtxt<'tcx>,
-        idx: usize,
+        idx: u32,
     }
 
-    impl<'tcx> TypeFolder<'tcx> for ReplaceParamAndInferWithPlaceholder<'tcx> {
-        fn tcx(&self) -> TyCtxt<'tcx> {
+    impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ReplaceParamAndInferWithPlaceholder<'tcx> {
+        fn interner(&self) -> TyCtxt<'tcx> {
             self.tcx
         }
 
         fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
             if let ty::Infer(_) = t.kind() {
-                self.tcx.mk_ty(ty::Placeholder(ty::PlaceholderType {
+                self.tcx.mk_placeholder(ty::PlaceholderType {
                     universe: ty::UniverseIndex::ROOT,
-                    name: ty::BoundVar::from_usize({
+                    name: ty::BoundTyKind::Anon({
                         let idx = self.idx;
                         self.idx += 1;
                         idx
                     }),
-                }))
+                })
             } else {
                 t.super_fold_with(self)
             }
@@ -2077,7 +2066,7 @@ fn replace_param_and_infer_substs_with_placeholder<'tcx>(
                 self.tcx.mk_const(
                     ty::PlaceholderConst {
                         universe: ty::UniverseIndex::ROOT,
-                        name: ty::BoundVar::from_usize({
+                        name: ty::BoundVar::from_u32({
                             let idx = self.idx;
                             self.idx += 1;
                             idx
diff --git a/compiler/rustc_infer/src/infer/nll_relate/mod.rs b/compiler/rustc_infer/src/infer/nll_relate/mod.rs
index f235cb5ab..573cd91a2 100644
--- a/compiler/rustc_infer/src/infer/nll_relate/mod.rs
+++ b/compiler/rustc_infer/src/infer/nll_relate/mod.rs
@@ -21,26 +21,21 @@
 //!   thing we relate in chalk are basically domain goals and their
 //!   constituents)
 
-use crate::infer::combine::ConstEquateRelation;
 use crate::infer::InferCtxt;
 use crate::infer::{ConstVarValue, ConstVariableValue};
 use crate::infer::{TypeVariableOrigin, TypeVariableOriginKind};
-use crate::traits::{Obligation, PredicateObligation};
+use crate::traits::{Obligation, PredicateObligations};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_middle::traits::ObligationCause;
 use rustc_middle::ty::error::TypeError;
 use rustc_middle::ty::relate::{self, Relate, RelateResult, TypeRelation};
-use rustc_middle::ty::visit::{TypeSuperVisitable, TypeVisitable, TypeVisitor};
+use rustc_middle::ty::visit::{TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor};
 use rustc_middle::ty::{self, InferConst, Ty, TyCtxt};
-use rustc_span::Span;
+use rustc_span::{Span, Symbol};
 use std::fmt::Debug;
 use std::ops::ControlFlow;
 
-#[derive(PartialEq)]
-pub enum NormalizationStrategy {
-    Lazy,
-    Eager,
-}
+use super::combine::ObligationEmittingRelation;
 
 pub struct TypeRelating<'me, 'tcx, D>
 where
@@ -55,7 +50,7 @@ where
     ///
     /// - Covariant means `a <: b`.
     /// - Contravariant means `b <: a`.
-    /// - Invariant means `a == b.
+    /// - Invariant means `a == b`.
     /// - Bivariant means that it doesn't matter.
     ambient_variance: ty::Variance,
 
@@ -92,7 +87,7 @@ pub trait TypeRelatingDelegate<'tcx> {
         info: ty::VarianceDiagInfo<'tcx>,
     );
 
-    fn register_obligations(&mut self, obligations: Vec<PredicateObligation<'tcx>>);
+    fn register_obligations(&mut self, obligations: PredicateObligations<'tcx>);
 
     /// Creates a new universe index. Used when instantiating placeholders.
     fn create_next_universe(&mut self) -> ty::UniverseIndex;
@@ -105,7 +100,11 @@ pub trait TypeRelatingDelegate<'tcx> {
     /// we will invoke this method to instantiate `'a` with an
     /// inference variable (though `'b` would be instantiated first,
     /// as a placeholder).
-    fn next_existential_region_var(&mut self, was_placeholder: bool) -> ty::Region<'tcx>;
+    fn next_existential_region_var(
+        &mut self,
+        was_placeholder: bool,
+        name: Option<Symbol>,
+    ) -> ty::Region<'tcx>;
 
     /// Creates a new region variable representing a
     /// higher-ranked region that is instantiated universally.
@@ -125,9 +124,6 @@ pub trait TypeRelatingDelegate<'tcx> {
     /// relation stating that `'?0: 'a`).
     fn generalize_existential(&mut self, universe: ty::UniverseIndex) -> ty::Region<'tcx>;
 
-    /// Define the normalization strategy to use, eager or lazy.
-    fn normalization() -> NormalizationStrategy;
-
     /// Enables some optimizations if we do not expect inference variables
     /// in the RHS of the relation.
     fn forbid_inference_vars() -> bool;
@@ -196,7 +192,7 @@ where
                     let placeholder = ty::PlaceholderRegion { universe, name: br.kind };
                     delegate.next_placeholder_region(placeholder)
                 } else {
-                    delegate.next_existential_region_var(true)
+                    delegate.next_existential_region_var(true, br.kind.get_name())
                 }
             }
         };
@@ -265,38 +261,6 @@ where
         self.delegate.push_outlives(sup, sub, info);
     }
 
-    /// Relate a projection type and some value type lazily. This will always
-    /// succeed, but we push an additional `ProjectionEq` goal depending
-    /// on the value type:
-    /// - if the value type is any type `T` which is not a projection, we push
-    ///   `ProjectionEq(projection = T)`.
-    /// - if the value type is another projection `other_projection`, we create
-    ///   a new inference variable `?U` and push the two goals
-    ///   `ProjectionEq(projection = ?U)`, `ProjectionEq(other_projection = ?U)`.
-    fn relate_projection_ty(
-        &mut self,
-        projection_ty: ty::AliasTy<'tcx>,
-        value_ty: Ty<'tcx>,
-    ) -> Ty<'tcx> {
-        use rustc_span::DUMMY_SP;
-
-        match *value_ty.kind() {
-            ty::Alias(ty::Projection, other_projection_ty) => {
-                let var = self.infcx.next_ty_var(TypeVariableOrigin {
-                    kind: TypeVariableOriginKind::MiscVariable,
-                    span: DUMMY_SP,
-                });
-                // FIXME(lazy-normalization): This will always ICE, because the recursive
-                // call will end up in the _ arm below.
-                self.relate_projection_ty(projection_ty, var);
-                self.relate_projection_ty(other_projection_ty, var);
-                var
-            }
-
-            _ => bug!("should never be invoked with eager normalization"),
-        }
-    }
-
     /// Relate a type inference variable with a value type. This works
     /// by creating a "generalization" G of the value where all the
     /// lifetimes are replaced with fresh inference values. This
@@ -335,12 +299,6 @@ where
                 return Ok(value_ty);
             }
 
-            ty::Alias(ty::Projection, projection_ty)
-                if D::normalization() == NormalizationStrategy::Lazy =>
-            {
-                return Ok(self.relate_projection_ty(projection_ty, self.infcx.tcx.mk_ty_var(vid)));
-            }
-
             _ => (),
         }
 
@@ -627,18 +585,6 @@ where
                 self.relate_opaques(a, b)
             }
 
-            (&ty::Alias(ty::Projection, projection_ty), _)
-                if D::normalization() == NormalizationStrategy::Lazy =>
-            {
-                Ok(self.relate_projection_ty(projection_ty, b))
-            }
-
-            (_, &ty::Alias(ty::Projection, projection_ty))
-                if D::normalization() == NormalizationStrategy::Lazy =>
-            {
-                Ok(self.relate_projection_ty(projection_ty, a))
-            }
-
             _ => {
                 debug!(?a, ?b, ?self.ambient_variance);
 
@@ -663,13 +609,13 @@ where
         debug!(?v_b);
 
         if self.ambient_covariance() {
-            // Covariance: a <= b. Hence, `b: a`.
-            self.push_outlives(v_b, v_a, self.ambient_variance_info);
+            // Covariant: &'a u8 <: &'b u8. Hence, `'a: 'b`.
+            self.push_outlives(v_a, v_b, self.ambient_variance_info);
         }
 
         if self.ambient_contravariance() {
-            // Contravariant: b <= a. Hence, `a: b`.
-            self.push_outlives(v_a, v_b, self.ambient_variance_info);
+            // Contravariant: &'b u8 <: &'a u8. Hence, `'b: 'a`.
+            self.push_outlives(v_b, v_a, self.ambient_variance_info);
         }
 
         Ok(a)
@@ -813,16 +759,23 @@ where
     }
 }
 
-impl<'tcx, D> ConstEquateRelation<'tcx> for TypeRelating<'_, 'tcx, D>
+impl<'tcx, D> ObligationEmittingRelation<'tcx> for TypeRelating<'_, 'tcx, D>
 where
     D: TypeRelatingDelegate<'tcx>,
 {
-    fn const_equate_obligation(&mut self, _a: ty::Const<'tcx>, _b: ty::Const<'tcx>) {
-        // We don't have to worry about the equality of consts during borrow checking
-        // as consts always have a static lifetime.
-        // FIXME(oli-obk): is this really true? We can at least have HKL and with
-        // inline consts we may have further lifetimes that may be unsound to treat as
-        // 'static.
+    fn register_predicates(&mut self, obligations: impl IntoIterator<Item: ty::ToPredicate<'tcx>>) {
+        self.delegate.register_obligations(
+            obligations
+                .into_iter()
+                .map(|to_pred| {
+                    Obligation::new(self.tcx(), ObligationCause::dummy(), self.param_env(), to_pred)
+                })
+                .collect(),
+        );
+    }
+
+    fn register_obligations(&mut self, obligations: PredicateObligations<'tcx>) {
+        self.delegate.register_obligations(obligations);
     }
 }
 
@@ -840,8 +793,8 @@ struct ScopeInstantiator<'me, 'tcx> {
     bound_region_scope: &'me mut BoundRegionScope<'tcx>,
 }
 
-impl<'me, 'tcx> TypeVisitor<'tcx> for ScopeInstantiator<'me, 'tcx> {
-    fn visit_binder<T: TypeVisitable<'tcx>>(
+impl<'me, 'tcx> TypeVisitor<TyCtxt<'tcx>> for ScopeInstantiator<'me, 'tcx> {
+    fn visit_binder<T: TypeVisitable<TyCtxt<'tcx>>>(
         &mut self,
         t: &ty::Binder<'tcx, T>,
     ) -> ControlFlow<Self::BreakTy> {
diff --git a/compiler/rustc_infer/src/infer/opaque_types.rs b/compiler/rustc_infer/src/infer/opaque_types.rs
index c54c66eab..d5c824d4c 100644
--- a/compiler/rustc_infer/src/infer/opaque_types.rs
+++ b/compiler/rustc_infer/src/infer/opaque_types.rs
@@ -3,7 +3,7 @@ use crate::infer::{DefiningAnchor, InferCtxt, InferOk};
 use crate::traits;
 use hir::def::DefKind;
 use hir::def_id::{DefId, LocalDefId};
-use hir::{HirId, OpaqueTyOrigin};
+use hir::OpaqueTyOrigin;
 use rustc_data_structures::sync::Lrc;
 use rustc_data_structures::vec_map::VecMap;
 use rustc_hir as hir;
@@ -13,7 +13,7 @@ use rustc_middle::ty::fold::BottomUpFolder;
 use rustc_middle::ty::GenericArgKind;
 use rustc_middle::ty::{
     self, OpaqueHiddenType, OpaqueTypeKey, Ty, TyCtxt, TypeFoldable, TypeSuperVisitable,
-    TypeVisitable, TypeVisitor,
+    TypeVisitable, TypeVisitableExt, TypeVisitor,
 };
 use rustc_span::Span;
 
@@ -45,10 +45,10 @@ pub struct OpaqueTypeDecl<'tcx> {
 impl<'tcx> InferCtxt<'tcx> {
     /// This is a backwards compatibility hack to prevent breaking changes from
     /// lazy TAIT around RPIT handling.
-    pub fn replace_opaque_types_with_inference_vars<T: TypeFoldable<'tcx>>(
+    pub fn replace_opaque_types_with_inference_vars<T: TypeFoldable<TyCtxt<'tcx>>>(
         &self,
         value: T,
-        body_id: HirId,
+        body_id: LocalDefId,
         span: Span,
         param_env: ty::ParamEnv<'tcx>,
     ) -> InferOk<'tcx, T> {
@@ -57,9 +57,7 @@ impl<'tcx> InferCtxt<'tcx> {
         }
         let mut obligations = vec![];
         let replace_opaque_type = |def_id: DefId| {
-            def_id
-                .as_local()
-                .map_or(false, |def_id| self.opaque_type_origin(def_id, span).is_some())
+            def_id.as_local().map_or(false, |def_id| self.opaque_type_origin(def_id).is_some())
         };
         let value = value.fold_with(&mut BottomUpFolder {
             tcx: self.tcx,
@@ -144,9 +142,9 @@ impl<'tcx> InferCtxt<'tcx> {
                         //     let x = || foo(); // returns the Opaque assoc with `foo`
                         // }
                         // ```
-                        self.opaque_type_origin(def_id, cause.span)?
+                        self.opaque_type_origin(def_id)?
                     }
-                    DefiningAnchor::Bubble => self.opaque_ty_origin_unchecked(def_id, cause.span),
+                    DefiningAnchor::Bubble => self.opaque_type_origin_unchecked(def_id),
                     DefiningAnchor::Error => return None,
                 };
                 if let ty::Alias(ty::Opaque, ty::AliasTy { def_id: b_def_id, .. }) = *b.kind() {
@@ -155,9 +153,8 @@ impl<'tcx> InferCtxt<'tcx> {
                     // no one encounters it in practice.
                     // It does occur however in `fn fut() -> impl Future<Output = i32> { async { 42 } }`,
                     // where it is of no concern, so we only check for TAITs.
-                    if let Some(OpaqueTyOrigin::TyAlias) = b_def_id
-                        .as_local()
-                        .and_then(|b_def_id| self.opaque_type_origin(b_def_id, cause.span))
+                    if let Some(OpaqueTyOrigin::TyAlias) =
+                        b_def_id.as_local().and_then(|b_def_id| self.opaque_type_origin(b_def_id))
                     {
                         self.tcx.sess.emit_err(OpaqueHiddenTypeDiag {
                             span: cause.span,
@@ -371,24 +368,18 @@ impl<'tcx> InferCtxt<'tcx> {
         });
     }
 
+    /// Returns the origin of the opaque type `def_id` if we're currently
+    /// in its defining scope.
     #[instrument(skip(self), level = "trace", ret)]
-    pub fn opaque_type_origin(&self, def_id: LocalDefId, span: Span) -> Option<OpaqueTyOrigin> {
+    pub fn opaque_type_origin(&self, def_id: LocalDefId) -> Option<OpaqueTyOrigin> {
         let opaque_hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id);
         let parent_def_id = match self.defining_use_anchor {
             DefiningAnchor::Bubble | DefiningAnchor::Error => return None,
             DefiningAnchor::Bind(bind) => bind,
         };
-        let item_kind = &self.tcx.hir().expect_item(def_id).kind;
-
-        let hir::ItemKind::OpaqueTy(hir::OpaqueTy { origin, .. }) = item_kind else {
-            span_bug!(
-                span,
-                "weird opaque type: {:#?}, {:#?}",
-                def_id,
-                item_kind
-            )
-        };
-        let in_definition_scope = match *origin {
+
+        let origin = self.opaque_type_origin_unchecked(def_id);
+        let in_definition_scope = match origin {
             // Async `impl Trait`
             hir::OpaqueTyOrigin::AsyncFn(parent) => parent == parent_def_id,
             // Anonymous `impl Trait`
@@ -398,16 +389,17 @@ impl<'tcx> InferCtxt<'tcx> {
                 may_define_opaque_type(self.tcx, parent_def_id, opaque_hir_id)
             }
         };
-        trace!(?origin);
-        in_definition_scope.then_some(*origin)
+        in_definition_scope.then_some(origin)
     }
 
+    /// Returns the origin of the opaque type `def_id` even if we are not in its
+    /// defining scope.
     #[instrument(skip(self), level = "trace", ret)]
-    fn opaque_ty_origin_unchecked(&self, def_id: LocalDefId, span: Span) -> OpaqueTyOrigin {
+    fn opaque_type_origin_unchecked(&self, def_id: LocalDefId) -> OpaqueTyOrigin {
         match self.tcx.hir().expect_item(def_id).kind {
             hir::ItemKind::OpaqueTy(hir::OpaqueTy { origin, .. }) => origin,
             ref itemkind => {
-                span_bug!(span, "weird opaque type: {:?}, {:#?}", def_id, itemkind)
+                bug!("weird opaque type: {:?}, {:#?}", def_id, itemkind)
             }
         }
     }
@@ -431,11 +423,11 @@ pub struct ConstrainOpaqueTypeRegionVisitor<'tcx, OP: FnMut(ty::Region<'tcx>)> {
     pub op: OP,
 }
 
-impl<'tcx, OP> TypeVisitor<'tcx> for ConstrainOpaqueTypeRegionVisitor<'tcx, OP>
+impl<'tcx, OP> TypeVisitor<TyCtxt<'tcx>> for ConstrainOpaqueTypeRegionVisitor<'tcx, OP>
 where
     OP: FnMut(ty::Region<'tcx>),
 {
-    fn visit_binder<T: TypeVisitable<'tcx>>(
+    fn visit_binder<T: TypeVisitable<TyCtxt<'tcx>>>(
         &mut self,
         t: &ty::Binder<'tcx, T>,
     ) -> ControlFlow<Self::BreakTy> {
@@ -553,8 +545,11 @@ impl<'tcx> InferCtxt<'tcx> {
             origin,
         );
         if let Some(prev) = prev {
-            obligations =
-                self.at(&cause, param_env).eq_exp(a_is_expected, prev, hidden_ty)?.obligations;
+            obligations = self
+                .at(&cause, param_env)
+                .define_opaque_types(true)
+                .eq_exp(a_is_expected, prev, hidden_ty)?
+                .obligations;
         }
 
         let item_bounds = tcx.bound_explicit_item_bounds(def_id.to_def_id());
diff --git a/compiler/rustc_infer/src/infer/outlives/components.rs b/compiler/rustc_infer/src/infer/outlives/components.rs
index 3d86279b0..ff23087fe 100644
--- a/compiler/rustc_infer/src/infer/outlives/components.rs
+++ b/compiler/rustc_infer/src/infer/outlives/components.rs
@@ -4,7 +4,7 @@
 
 use rustc_data_structures::sso::SsoHashSet;
 use rustc_middle::ty::subst::{GenericArg, GenericArgKind};
-use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitableExt};
 use smallvec::{smallvec, SmallVec};
 
 #[derive(Debug)]
@@ -112,7 +112,7 @@ fn compute_components<'tcx>(
             }
 
             // All regions are bound inside a witness
-            ty::GeneratorWitness(..) => (),
+            ty::GeneratorWitness(..) | ty::GeneratorWitnessMIR(..) => (),
 
             // OutlivesTypeParameterEnv -- the actual checking that `X:'a`
             // is implied by the environment is done in regionck.
diff --git a/compiler/rustc_infer/src/infer/outlives/mod.rs b/compiler/rustc_infer/src/infer/outlives/mod.rs
index 4daa25767..83f3d5a74 100644
--- a/compiler/rustc_infer/src/infer/outlives/mod.rs
+++ b/compiler/rustc_infer/src/infer/outlives/mod.rs
@@ -21,6 +21,8 @@ pub fn explicit_outlives_bounds<'tcx>(
         .filter_map(move |kind| match kind {
             ty::PredicateKind::Clause(ty::Clause::Projection(..))
             | ty::PredicateKind::Clause(ty::Clause::Trait(..))
+            | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
+            | ty::PredicateKind::AliasEq(..)
             | ty::PredicateKind::Coerce(..)
             | ty::PredicateKind::Subtype(..)
             | ty::PredicateKind::WellFormed(..)
diff --git a/compiler/rustc_infer/src/infer/outlives/obligations.rs b/compiler/rustc_infer/src/infer/outlives/obligations.rs
index 0194549a8..bbe7d4c63 100644
--- a/compiler/rustc_infer/src/infer/outlives/obligations.rs
+++ b/compiler/rustc_infer/src/infer/outlives/obligations.rs
@@ -69,7 +69,7 @@ use crate::traits::{ObligationCause, ObligationCauseCode};
 use rustc_data_structures::undo_log::UndoLogs;
 use rustc_middle::mir::ConstraintCategory;
 use rustc_middle::ty::subst::GenericArgKind;
-use rustc_middle::ty::{self, Region, SubstsRef, Ty, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, Region, SubstsRef, Ty, TyCtxt, TypeVisitableExt};
 use smallvec::smallvec;
 
 impl<'tcx> InferCtxt<'tcx> {
diff --git a/compiler/rustc_infer/src/infer/outlives/test_type_match.rs b/compiler/rustc_infer/src/infer/outlives/test_type_match.rs
index 10b474efd..3c6cc2b90 100644
--- a/compiler/rustc_infer/src/infer/outlives/test_type_match.rs
+++ b/compiler/rustc_infer/src/infer/outlives/test_type_match.rs
@@ -1,7 +1,7 @@
 use std::collections::hash_map::Entry;
 
 use rustc_data_structures::fx::FxHashMap;
-use rustc_middle::ty::TypeVisitable;
+use rustc_middle::ty::TypeVisitableExt;
 use rustc_middle::ty::{
     self,
     error::TypeError,
@@ -186,7 +186,8 @@ impl<'tcx> TypeRelation<'tcx> for Match<'tcx> {
 
     #[instrument(skip(self), level = "debug")]
     fn tys(&mut self, pattern: Ty<'tcx>, value: Ty<'tcx>) -> RelateResult<'tcx, Ty<'tcx>> {
-        if let ty::Error(_) = pattern.kind() {
+        // FIXME(non_lifetime_binders): What to do here?
+        if matches!(pattern.kind(), ty::Error(_) | ty::Bound(..)) {
             // Unlike normal `TypeRelation` rules, `ty::Error` does not equal any type.
             self.no_match()
         } else if pattern == value {
diff --git a/compiler/rustc_infer/src/infer/outlives/verify.rs b/compiler/rustc_infer/src/infer/outlives/verify.rs
index 94de9bc2d..bae246418 100644
--- a/compiler/rustc_infer/src/infer/outlives/verify.rs
+++ b/compiler/rustc_infer/src/infer/outlives/verify.rs
@@ -207,6 +207,7 @@ impl<'cx, 'tcx> VerifyBoundCx<'cx, 'tcx> {
     ///
     /// In some cases, such as when `erased_ty` represents a `ty::Param`, however,
     /// the result is precise.
+    #[instrument(level = "debug", skip(self))]
     fn declared_generic_bounds_from_env_for_erased_ty(
         &self,
         erased_ty: Ty<'tcx>,
diff --git a/compiler/rustc_infer/src/infer/projection.rs b/compiler/rustc_infer/src/infer/projection.rs
index 4667d99ff..f79504770 100644
--- a/compiler/rustc_infer/src/infer/projection.rs
+++ b/compiler/rustc_infer/src/infer/projection.rs
@@ -21,16 +21,28 @@ impl<'tcx> InferCtxt<'tcx> {
         recursion_depth: usize,
         obligations: &mut Vec<PredicateObligation<'tcx>>,
     ) -> Ty<'tcx> {
-        let def_id = projection_ty.def_id;
-        let ty_var = self.next_ty_var(TypeVariableOrigin {
-            kind: TypeVariableOriginKind::NormalizeProjectionType,
-            span: self.tcx.def_span(def_id),
-        });
-        let projection =
-            ty::Binder::dummy(ty::ProjectionPredicate { projection_ty, term: ty_var.into() });
-        let obligation =
-            Obligation::with_depth(self.tcx, cause, recursion_depth, param_env, projection);
-        obligations.push(obligation);
-        ty_var
+        if self.tcx.trait_solver_next() {
+            // FIXME(-Ztrait-solver=next): Instead of branching here,
+            // completely change the normalization routine with the new solver.
+            //
+            // The new solver correctly handles projection equality so this hack
+            // is not necessary. if re-enabled it should emit `PredicateKind::AliasEq`
+            // not `PredicateKind::Clause(Clause::Projection(..))` as in the new solver
+            // `Projection` is used as `normalizes-to` which will fail for `<T as Trait>::Assoc eq ?0`.
+            return projection_ty.to_ty(self.tcx);
+        } else {
+            let def_id = projection_ty.def_id;
+            let ty_var = self.next_ty_var(TypeVariableOrigin {
+                kind: TypeVariableOriginKind::NormalizeProjectionType,
+                span: self.tcx.def_span(def_id),
+            });
+            let projection = ty::Binder::dummy(ty::PredicateKind::Clause(ty::Clause::Projection(
+                ty::ProjectionPredicate { projection_ty, term: ty_var.into() },
+            )));
+            let obligation =
+                Obligation::with_depth(self.tcx, cause, recursion_depth, param_env, projection);
+            obligations.push(obligation);
+            ty_var
+        }
     }
 }
diff --git a/compiler/rustc_infer/src/infer/region_constraints/leak_check.rs b/compiler/rustc_infer/src/infer/region_constraints/leak_check.rs
index c46edc33f..e413b2bb5 100644
--- a/compiler/rustc_infer/src/infer/region_constraints/leak_check.rs
+++ b/compiler/rustc_infer/src/infer/region_constraints/leak_check.rs
@@ -280,7 +280,7 @@ impl<'me, 'tcx> LeakCheck<'me, 'tcx> {
         placeholder1: ty::PlaceholderRegion,
         placeholder2: ty::PlaceholderRegion,
     ) -> TypeError<'tcx> {
-        self.error(placeholder1, self.tcx.mk_region(ty::RePlaceholder(placeholder2)))
+        self.error(placeholder1, self.tcx.mk_re_placeholder(placeholder2))
     }
 
     fn error(
@@ -413,19 +413,19 @@ impl<'tcx> MiniGraph<'tcx> {
         for undo_entry in undo_log {
             match undo_entry {
                 &AddConstraint(Constraint::VarSubVar(a, b)) => {
-                    each_edge(tcx.mk_region(ReVar(a)), tcx.mk_region(ReVar(b)));
+                    each_edge(tcx.mk_re_var(a), tcx.mk_re_var(b));
                 }
                 &AddConstraint(Constraint::RegSubVar(a, b)) => {
-                    each_edge(a, tcx.mk_region(ReVar(b)));
+                    each_edge(a, tcx.mk_re_var(b));
                 }
                 &AddConstraint(Constraint::VarSubReg(a, b)) => {
-                    each_edge(tcx.mk_region(ReVar(a)), b);
+                    each_edge(tcx.mk_re_var(a), b);
                 }
                 &AddConstraint(Constraint::RegSubReg(a, b)) => {
                     each_edge(a, b);
                 }
                 &AddGiven(a, b) => {
-                    each_edge(a, tcx.mk_region(ReVar(b)));
+                    each_edge(a, tcx.mk_re_var(b));
                 }
                 &AddVerify(i) => span_bug!(
                     verifys[i].origin.span(),
diff --git a/compiler/rustc_infer/src/infer/region_constraints/mod.rs b/compiler/rustc_infer/src/infer/region_constraints/mod.rs
index 0428481b7..872f61747 100644
--- a/compiler/rustc_infer/src/infer/region_constraints/mod.rs
+++ b/compiler/rustc_infer/src/infer/region_constraints/mod.rs
@@ -249,7 +249,7 @@ pub enum VerifyBound<'tcx> {
 /// in that case we can show `'b: 'c`. But if `'?x` winds up being something
 /// else, the bound isn't relevant.
 ///
-/// In the [`VerifyBound`], this struct is enclosed in `Binder to account
+/// In the [`VerifyBound`], this struct is enclosed in `Binder` to account
 /// for cases like
 ///
 /// ```rust
@@ -651,7 +651,7 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> {
                 let unified_region = self.unification_table().probe_value(rid);
                 unified_region.0.unwrap_or_else(|| {
                     let root = self.unification_table().find(rid).vid;
-                    tcx.reuse_or_mk_region(region, ty::ReVar(root))
+                    tcx.mk_re_var(root)
                 })
             }
             _ => region,
@@ -675,7 +675,7 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> {
     ) -> Region<'tcx> {
         let vars = TwoRegions { a, b };
         if let Some(&c) = self.combine_map(t).get(&vars) {
-            return tcx.mk_region(ReVar(c));
+            return tcx.mk_re_var(c);
         }
         let a_universe = self.universe(a);
         let b_universe = self.universe(b);
@@ -683,7 +683,7 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> {
         let c = self.new_region_var(c_universe, MiscVariable(origin.span()));
         self.combine_map(t).insert(vars, c);
         self.undo_log.push(AddCombination(t, vars));
-        let new_r = tcx.mk_region(ReVar(c));
+        let new_r = tcx.mk_re_var(c);
         for old_r in [a, b] {
             match t {
                 Glb => self.make_subregion(origin.clone(), new_r, old_r),
@@ -696,9 +696,11 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> {
 
     pub fn universe(&self, region: Region<'tcx>) -> ty::UniverseIndex {
         match *region {
-            ty::ReStatic | ty::ReErased | ty::ReFree(..) | ty::ReEarlyBound(..) => {
-                ty::UniverseIndex::ROOT
-            }
+            ty::ReStatic
+            | ty::ReErased
+            | ty::ReFree(..)
+            | ty::ReEarlyBound(..)
+            | ty::ReError(_) => ty::UniverseIndex::ROOT,
             ty::RePlaceholder(placeholder) => placeholder.universe,
             ty::ReVar(vid) => self.var_universe(vid),
             ty::ReLateBound(..) => bug!("universe(): encountered bound region {:?}", region),
diff --git a/compiler/rustc_infer/src/infer/resolve.rs b/compiler/rustc_infer/src/infer/resolve.rs
index 65b90aa3d..5bb358329 100644
--- a/compiler/rustc_infer/src/infer/resolve.rs
+++ b/compiler/rustc_infer/src/infer/resolve.rs
@@ -2,8 +2,8 @@ use super::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
 use super::{FixupError, FixupResult, InferCtxt, Span};
 use rustc_middle::infer::unify_key::{ConstVariableOrigin, ConstVariableOriginKind};
 use rustc_middle::ty::fold::{FallibleTypeFolder, TypeFolder, TypeSuperFoldable};
-use rustc_middle::ty::visit::{TypeSuperVisitable, TypeVisitor};
-use rustc_middle::ty::{self, Const, InferConst, Ty, TyCtxt, TypeFoldable, TypeVisitable};
+use rustc_middle::ty::visit::{TypeSuperVisitable, TypeVisitableExt, TypeVisitor};
+use rustc_middle::ty::{self, Const, InferConst, Ty, TyCtxt, TypeFoldable};
 
 use std::ops::ControlFlow;
 
@@ -16,26 +16,29 @@ use std::ops::ControlFlow;
 /// useful for printing messages etc but also required at various
 /// points for correctness.
 pub struct OpportunisticVarResolver<'a, 'tcx> {
-    infcx: &'a InferCtxt<'tcx>,
+    // The shallow resolver is used to resolve inference variables at every
+    // level of the type.
+    shallow_resolver: crate::infer::ShallowResolver<'a, 'tcx>,
 }
 
 impl<'a, 'tcx> OpportunisticVarResolver<'a, 'tcx> {
     #[inline]
     pub fn new(infcx: &'a InferCtxt<'tcx>) -> Self {
-        OpportunisticVarResolver { infcx }
+        OpportunisticVarResolver { shallow_resolver: crate::infer::ShallowResolver { infcx } }
     }
 }
 
-impl<'a, 'tcx> TypeFolder<'tcx> for OpportunisticVarResolver<'a, 'tcx> {
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
-        self.infcx.tcx
+impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for OpportunisticVarResolver<'a, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
+        TypeFolder::interner(&self.shallow_resolver)
     }
 
+    #[inline]
     fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
         if !t.has_non_region_infer() {
             t // micro-optimize -- if there is nothing in this type that this fold affects...
         } else {
-            let t = self.infcx.shallow_resolve(t);
+            let t = self.shallow_resolver.fold_ty(t);
             t.super_fold_with(self)
         }
     }
@@ -44,7 +47,7 @@ impl<'a, 'tcx> TypeFolder<'tcx> for OpportunisticVarResolver<'a, 'tcx> {
         if !ct.has_non_region_infer() {
             ct // micro-optimize -- if there is nothing in this const that this fold affects...
         } else {
-            let ct = self.infcx.shallow_resolve(ct);
+            let ct = self.shallow_resolver.fold_const(ct);
             ct.super_fold_with(self)
         }
     }
@@ -67,8 +70,8 @@ impl<'a, 'tcx> OpportunisticRegionResolver<'a, 'tcx> {
     }
 }
 
-impl<'a, 'tcx> TypeFolder<'tcx> for OpportunisticRegionResolver<'a, 'tcx> {
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for OpportunisticRegionResolver<'a, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.infcx.tcx
     }
 
@@ -89,7 +92,7 @@ impl<'a, 'tcx> TypeFolder<'tcx> for OpportunisticRegionResolver<'a, 'tcx> {
                     .borrow_mut()
                     .unwrap_region_constraints()
                     .opportunistic_resolve_var(rid);
-                TypeFolder::tcx(self).reuse_or_mk_region(r, ty::ReVar(resolved))
+                TypeFolder::interner(self).mk_re_var(resolved)
             }
             _ => r,
         }
@@ -121,7 +124,7 @@ impl<'a, 'tcx> UnresolvedTypeOrConstFinder<'a, 'tcx> {
     }
 }
 
-impl<'a, 'tcx> TypeVisitor<'tcx> for UnresolvedTypeOrConstFinder<'a, 'tcx> {
+impl<'a, 'tcx> TypeVisitor<TyCtxt<'tcx>> for UnresolvedTypeOrConstFinder<'a, 'tcx> {
     type BreakTy = (ty::Term<'tcx>, Option<Span>);
     fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
         let t = self.infcx.shallow_resolve(t);
@@ -194,7 +197,7 @@ impl<'a, 'tcx> TypeVisitor<'tcx> for UnresolvedTypeOrConstFinder<'a, 'tcx> {
 /// then an `Err` result is returned.
 pub fn fully_resolve<'tcx, T>(infcx: &InferCtxt<'tcx>, value: T) -> FixupResult<'tcx, T>
 where
-    T: TypeFoldable<'tcx>,
+    T: TypeFoldable<TyCtxt<'tcx>>,
 {
     value.try_fold_with(&mut FullTypeResolver { infcx })
 }
@@ -205,10 +208,10 @@ struct FullTypeResolver<'a, 'tcx> {
     infcx: &'a InferCtxt<'tcx>,
 }
 
-impl<'a, 'tcx> FallibleTypeFolder<'tcx> for FullTypeResolver<'a, 'tcx> {
+impl<'a, 'tcx> FallibleTypeFolder<TyCtxt<'tcx>> for FullTypeResolver<'a, 'tcx> {
     type Error = FixupError<'tcx>;
 
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.infcx.tcx
     }
 
diff --git a/compiler/rustc_infer/src/infer/sub.rs b/compiler/rustc_infer/src/infer/sub.rs
index bd38b52ba..3e8c2052d 100644
--- a/compiler/rustc_infer/src/infer/sub.rs
+++ b/compiler/rustc_infer/src/infer/sub.rs
@@ -1,10 +1,9 @@
 use super::combine::{CombineFields, RelationDir};
-use super::SubregionOrigin;
+use super::{ObligationEmittingRelation, SubregionOrigin};
 
-use crate::infer::combine::ConstEquateRelation;
-use crate::traits::Obligation;
+use crate::traits::{Obligation, PredicateObligations};
 use rustc_middle::ty::relate::{Cause, Relate, RelateResult, TypeRelation};
-use rustc_middle::ty::visit::TypeVisitable;
+use rustc_middle::ty::visit::TypeVisitableExt;
 use rustc_middle::ty::TyVar;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use std::mem;
@@ -127,7 +126,7 @@ impl<'tcx> TypeRelation<'tcx> for Sub<'_, '_, 'tcx> {
 
             (&ty::Error(e), _) | (_, &ty::Error(e)) => {
                 infcx.set_tainted_by_errors(e);
-                Ok(self.tcx().ty_error_with_guaranteed(e))
+                Ok(self.tcx().ty_error(e))
             }
 
             (
@@ -161,7 +160,7 @@ impl<'tcx> TypeRelation<'tcx> for Sub<'_, '_, 'tcx> {
                 let a_types = infcx.tcx.anonymize_bound_vars(a_types);
                 let b_types = infcx.tcx.anonymize_bound_vars(b_types);
                 if a_types.bound_vars() == b_types.bound_vars() {
-                    let (a_types, b_types) = infcx.replace_bound_vars_with_placeholders(
+                    let (a_types, b_types) = infcx.instantiate_binder_with_placeholders(
                         a_types.map_bound(|a_types| (a_types, b_types.skip_binder())),
                     );
                     for (a, b) in std::iter::zip(a_types, b_types) {
@@ -191,12 +190,13 @@ impl<'tcx> TypeRelation<'tcx> for Sub<'_, '_, 'tcx> {
         // from the "cause" field, we could perhaps give more tailored
         // error messages.
         let origin = SubregionOrigin::Subtype(Box::new(self.fields.trace.clone()));
+        // Subtype(&'a u8, &'b u8) => Outlives('a: 'b) => SubRegion('b, 'a)
         self.fields
             .infcx
             .inner
             .borrow_mut()
             .unwrap_region_constraints()
-            .make_subregion(origin, a, b);
+            .make_subregion(origin, b, a);
 
         Ok(a)
     }
@@ -227,8 +227,12 @@ impl<'tcx> TypeRelation<'tcx> for Sub<'_, '_, 'tcx> {
     }
 }
 
-impl<'tcx> ConstEquateRelation<'tcx> for Sub<'_, '_, 'tcx> {
-    fn const_equate_obligation(&mut self, a: ty::Const<'tcx>, b: ty::Const<'tcx>) {
-        self.fields.add_const_equate_obligation(self.a_is_expected, a, b);
+impl<'tcx> ObligationEmittingRelation<'tcx> for Sub<'_, '_, 'tcx> {
+    fn register_predicates(&mut self, obligations: impl IntoIterator<Item: ty::ToPredicate<'tcx>>) {
+        self.fields.register_predicates(obligations);
+    }
+
+    fn register_obligations(&mut self, obligations: PredicateObligations<'tcx>) {
+        self.fields.register_obligations(obligations);
     }
 }
diff --git a/compiler/rustc_infer/src/lib.rs b/compiler/rustc_infer/src/lib.rs
index 4c119a443..bdc313c21 100644
--- a/compiler/rustc_infer/src/lib.rs
+++ b/compiler/rustc_infer/src/lib.rs
@@ -13,6 +13,7 @@
 //! This API is completely unstable and subject to change.
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
+#![feature(associated_type_bounds)]
 #![feature(box_patterns)]
 #![feature(control_flow_enum)]
 #![feature(extend_one)]
@@ -33,6 +34,11 @@ extern crate tracing;
 #[macro_use]
 extern crate rustc_middle;
 
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
+use rustc_macros::fluent_messages;
+
 mod errors;
 pub mod infer;
 pub mod traits;
+
+fluent_messages! { "../locales/en-US.ftl" }
diff --git a/compiler/rustc_infer/src/traits/engine.rs b/compiler/rustc_infer/src/traits/engine.rs
index d3519f4b3..f75344f20 100644
--- a/compiler/rustc_infer/src/traits/engine.rs
+++ b/compiler/rustc_infer/src/traits/engine.rs
@@ -1,6 +1,5 @@
 use crate::infer::InferCtxt;
 use crate::traits::Obligation;
-use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::def_id::DefId;
 use rustc_middle::ty::{self, ToPredicate, Ty};
 
@@ -37,13 +36,19 @@ pub trait TraitEngine<'tcx>: 'tcx {
         obligation: PredicateObligation<'tcx>,
     );
 
-    fn select_all_or_error(&mut self, infcx: &InferCtxt<'tcx>) -> Vec<FulfillmentError<'tcx>>;
-
     fn select_where_possible(&mut self, infcx: &InferCtxt<'tcx>) -> Vec<FulfillmentError<'tcx>>;
 
+    fn collect_remaining_errors(&mut self) -> Vec<FulfillmentError<'tcx>>;
+
     fn pending_obligations(&self) -> Vec<PredicateObligation<'tcx>>;
 
-    fn relationships(&mut self) -> &mut FxHashMap<ty::TyVid, ty::FoundRelationships>;
+    /// Among all pending obligations, collect those are stalled on a inference variable which has
+    /// changed since the last call to `select_where_possible`. Those obligations are marked as
+    /// successful and returned.
+    fn drain_unstalled_obligations(
+        &mut self,
+        infcx: &InferCtxt<'tcx>,
+    ) -> Vec<PredicateObligation<'tcx>>;
 }
 
 pub trait TraitEngineExt<'tcx> {
@@ -52,6 +57,8 @@ pub trait TraitEngineExt<'tcx> {
         infcx: &InferCtxt<'tcx>,
         obligations: impl IntoIterator<Item = PredicateObligation<'tcx>>,
     );
+
+    fn select_all_or_error(&mut self, infcx: &InferCtxt<'tcx>) -> Vec<FulfillmentError<'tcx>>;
 }
 
 impl<'tcx, T: ?Sized + TraitEngine<'tcx>> TraitEngineExt<'tcx> for T {
@@ -64,4 +71,13 @@ impl<'tcx, T: ?Sized + TraitEngine<'tcx>> TraitEngineExt<'tcx> for T {
             self.register_predicate_obligation(infcx, obligation);
         }
     }
+
+    fn select_all_or_error(&mut self, infcx: &InferCtxt<'tcx>) -> Vec<FulfillmentError<'tcx>> {
+        let errors = self.select_where_possible(infcx);
+        if !errors.is_empty() {
+            return errors;
+        }
+
+        self.collect_remaining_errors()
+    }
 }
diff --git a/compiler/rustc_infer/src/traits/mod.rs b/compiler/rustc_infer/src/traits/mod.rs
index 026713b6a..3a8289966 100644
--- a/compiler/rustc_infer/src/traits/mod.rs
+++ b/compiler/rustc_infer/src/traits/mod.rs
@@ -8,6 +8,7 @@ mod project;
 mod structural_impls;
 pub mod util;
 
+use hir::def_id::LocalDefId;
 use rustc_hir as hir;
 use rustc_middle::ty::error::{ExpectedFound, TypeError};
 use rustc_middle::ty::{self, Const, ToPredicate, Ty, TyCtxt};
@@ -146,7 +147,7 @@ impl<'tcx, O> Obligation<'tcx, O> {
     pub fn misc(
         tcx: TyCtxt<'tcx>,
         span: Span,
-        body_id: hir::HirId,
+        body_id: LocalDefId,
         param_env: ty::ParamEnv<'tcx>,
         trait_ref: impl ToPredicate<'tcx, O>,
     ) -> Obligation<'tcx, O> {
diff --git a/compiler/rustc_infer/src/traits/structural_impls.rs b/compiler/rustc_infer/src/traits/structural_impls.rs
index 1c6ab6a08..3a5273b03 100644
--- a/compiler/rustc_infer/src/traits/structural_impls.rs
+++ b/compiler/rustc_infer/src/traits/structural_impls.rs
@@ -1,8 +1,8 @@
 use crate::traits;
 use crate::traits::project::Normalized;
-use rustc_middle::ty;
 use rustc_middle::ty::fold::{FallibleTypeFolder, TypeFoldable};
 use rustc_middle::ty::visit::{TypeVisitable, TypeVisitor};
+use rustc_middle::ty::{self, TyCtxt};
 
 use std::fmt;
 use std::ops::ControlFlow;
@@ -61,8 +61,13 @@ impl<'tcx> fmt::Debug for traits::MismatchedProjectionTypes<'tcx> {
 ///////////////////////////////////////////////////////////////////////////
 // TypeFoldable implementations.
 
-impl<'tcx, O: TypeFoldable<'tcx>> TypeFoldable<'tcx> for traits::Obligation<'tcx, O> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
+impl<'tcx, O: TypeFoldable<TyCtxt<'tcx>>> TypeFoldable<TyCtxt<'tcx>>
+    for traits::Obligation<'tcx, O>
+{
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self, F::Error> {
         Ok(traits::Obligation {
             cause: self.cause,
             recursion_depth: self.recursion_depth,
@@ -72,8 +77,10 @@ impl<'tcx, O: TypeFoldable<'tcx>> TypeFoldable<'tcx> for traits::Obligation<'tcx
     }
 }
 
-impl<'tcx, O: TypeVisitable<'tcx>> TypeVisitable<'tcx> for traits::Obligation<'tcx, O> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+impl<'tcx, O: TypeVisitable<TyCtxt<'tcx>>> TypeVisitable<TyCtxt<'tcx>>
+    for traits::Obligation<'tcx, O>
+{
+    fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
         self.predicate.visit_with(visitor)?;
         self.param_env.visit_with(visitor)
     }
diff --git a/compiler/rustc_infer/src/traits/util.rs b/compiler/rustc_infer/src/traits/util.rs
index cd5bde2a7..c07ff5165 100644
--- a/compiler/rustc_infer/src/traits/util.rs
+++ b/compiler/rustc_infer/src/traits/util.rs
@@ -116,11 +116,11 @@ pub fn elaborate_predicates_with_span<'tcx>(
 
 pub fn elaborate_obligations<'tcx>(
     tcx: TyCtxt<'tcx>,
-    mut obligations: Vec<PredicateObligation<'tcx>>,
+    obligations: Vec<PredicateObligation<'tcx>>,
 ) -> Elaborator<'tcx> {
-    let mut visited = PredicateSet::new(tcx);
-    obligations.retain(|obligation| visited.insert(obligation.predicate));
-    Elaborator { stack: obligations, visited }
+    let mut elaborator = Elaborator { stack: Vec::new(), visited: PredicateSet::new(tcx) };
+    elaborator.extend_deduped(obligations);
+    elaborator
 }
 
 fn predicate_obligation<'tcx>(
@@ -132,6 +132,15 @@ fn predicate_obligation<'tcx>(
 }
 
 impl<'tcx> Elaborator<'tcx> {
+    fn extend_deduped(&mut self, obligations: impl IntoIterator<Item = PredicateObligation<'tcx>>) {
+        // Only keep those bounds that we haven't already seen.
+        // This is necessary to prevent infinite recursion in some
+        // cases. One common case is when people define
+        // `trait Sized: Sized { }` rather than `trait Sized { }`.
+        // let visited = &mut self.visited;
+        self.stack.extend(obligations.into_iter().filter(|o| self.visited.insert(o.predicate)));
+    }
+
     pub fn filter_to_traits(self) -> FilterToTraits<Self> {
         FilterToTraits::new(self)
     }
@@ -145,40 +154,34 @@ impl<'tcx> Elaborator<'tcx> {
                 // Get predicates declared on the trait.
                 let predicates = tcx.super_predicates_of(data.def_id());
 
-                let obligations = predicates.predicates.iter().map(|&(mut pred, span)| {
-                    // when parent predicate is non-const, elaborate it to non-const predicates.
-                    if data.constness == ty::BoundConstness::NotConst {
-                        pred = pred.without_const(tcx);
-                    }
-
-                    let cause = obligation.cause.clone().derived_cause(
-                        bound_predicate.rebind(data),
-                        |derived| {
-                            traits::ImplDerivedObligation(Box::new(
-                                traits::ImplDerivedObligationCause {
-                                    derived,
-                                    impl_def_id: data.def_id(),
-                                    span,
-                                },
-                            ))
-                        },
-                    );
-                    predicate_obligation(
-                        pred.subst_supertrait(tcx, &bound_predicate.rebind(data.trait_ref)),
-                        obligation.param_env,
-                        cause,
-                    )
-                });
+                let obligations =
+                    predicates.predicates.iter().enumerate().map(|(index, &(mut pred, span))| {
+                        // when parent predicate is non-const, elaborate it to non-const predicates.
+                        if data.constness == ty::BoundConstness::NotConst {
+                            pred = pred.without_const(tcx);
+                        }
+
+                        let cause = obligation.cause.clone().derived_cause(
+                            bound_predicate.rebind(data),
+                            |derived| {
+                                traits::ImplDerivedObligation(Box::new(
+                                    traits::ImplDerivedObligationCause {
+                                        derived,
+                                        impl_or_alias_def_id: data.def_id(),
+                                        impl_def_predicate_index: Some(index),
+                                        span,
+                                    },
+                                ))
+                            },
+                        );
+                        predicate_obligation(
+                            pred.subst_supertrait(tcx, &bound_predicate.rebind(data.trait_ref)),
+                            obligation.param_env,
+                            cause,
+                        )
+                    });
                 debug!(?data, ?obligations, "super_predicates");
-
-                // Only keep those bounds that we haven't already seen.
-                // This is necessary to prevent infinite recursion in some
-                // cases. One common case is when people define
-                // `trait Sized: Sized { }` rather than `trait Sized { }`.
-                let visited = &mut self.visited;
-                let obligations = obligations.filter(|o| visited.insert(o.predicate));
-
-                self.stack.extend(obligations);
+                self.extend_deduped(obligations);
             }
             ty::PredicateKind::WellFormed(..) => {
                 // Currently, we do not elaborate WF predicates,
@@ -235,10 +238,9 @@ impl<'tcx> Elaborator<'tcx> {
                     return;
                 }
 
-                let visited = &mut self.visited;
                 let mut components = smallvec![];
                 push_outlives_components(tcx, ty_max, &mut components);
-                self.stack.extend(
+                self.extend_deduped(
                     components
                         .into_iter()
                         .filter_map(|component| match component {
@@ -278,7 +280,6 @@ impl<'tcx> Elaborator<'tcx> {
                         .map(|predicate_kind| {
                             bound_predicate.rebind(predicate_kind).to_predicate(tcx)
                         })
-                        .filter(|&predicate| visited.insert(predicate))
                         .map(|predicate| {
                             predicate_obligation(
                                 predicate,
@@ -292,6 +293,12 @@ impl<'tcx> Elaborator<'tcx> {
                 // Nothing to elaborate
             }
             ty::PredicateKind::Ambiguous => {}
+            ty::PredicateKind::AliasEq(..) => {
+                // No
+            }
+            ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..)) => {
+                // Nothing to elaborate
+            }
         }
     }
 }
diff --git a/compiler/rustc_interface/Cargo.toml b/compiler/rustc_interface/Cargo.toml
index f817c5bc1..ac6e8fca6 100644
--- a/compiler/rustc_interface/Cargo.toml
+++ b/compiler/rustc_interface/Cargo.toml
@@ -20,11 +20,11 @@ rustc_macros = { path = "../rustc_macros" }
 rustc_parse = { path = "../rustc_parse" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
-rustc_serialize = { path = "../rustc_serialize" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_ast_lowering = { path = "../rustc_ast_lowering" }
 rustc_ast_passes = { path = "../rustc_ast_passes" }
 rustc_incremental = { path = "../rustc_incremental" }
+rustc_index = { path = "../rustc_index" }
 rustc_traits = { path = "../rustc_traits" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }
@@ -49,9 +49,6 @@ rustc_target = { path = "../rustc_target" }
 rustc_trait_selection = { path = "../rustc_trait_selection" }
 rustc_ty_utils = { path = "../rustc_ty_utils" }
 
-[dev-dependencies]
-rustc_target = { path = "../rustc_target" }
-
 [features]
 llvm = ['rustc_codegen_llvm']
 rustc_use_parallel_compiler = ['rayon', 'rustc-rayon-core', 'rustc_query_impl/rustc_use_parallel_compiler', 'rustc_errors/rustc_use_parallel_compiler']
diff --git a/compiler/rustc_error_messages/locales/en-US/interface.ftl b/compiler/rustc_interface/locales/en-US.ftl
index 688b04472..37994899a 100644
--- a/compiler/rustc_error_messages/locales/en-US/interface.ftl
+++ b/compiler/rustc_interface/locales/en-US.ftl
@@ -11,10 +11,6 @@ interface_mixed_bin_crate =
 interface_mixed_proc_macro_crate =
     cannot mix `proc-macro` crate type with others
 
-interface_proc_macro_doc_without_arg =
-    Trying to document proc macro crate without passing '--crate-type proc-macro to rustdoc
-    .warn = The generated documentation may be incorrect
-
 interface_error_writing_dependencies =
     error writing dependencies to `{$path}`: {$error}
 
@@ -37,10 +33,20 @@ interface_rustc_error_fatal =
     fatal error triggered by #[rustc_error]
 
 interface_rustc_error_unexpected_annotation =
-    unexpected annotation used with `#[rustc_error(...)]!
+    unexpected annotation used with `#[rustc_error(...)]`!
 
 interface_failed_writing_file =
     failed to write file {$path}: {$error}"
 
 interface_proc_macro_crate_panic_abort =
     building proc macro crate with `panic=abort` may crash the compiler should the proc-macro panic
+
+interface_unsupported_crate_type_for_target =
+    dropping unsupported crate type `{$crate_type}` for target `{$target_triple}`
+
+interface_multiple_output_types_adaption =
+    due to multiple output types requested, the explicitly specified output file name will be adapted for each output type
+
+interface_ignoring_extra_filename = ignoring -C extra-filename flag due to -o flag
+
+interface_ignoring_out_dir = ignoring --out-dir flag due to -o flag
diff --git a/compiler/rustc_interface/src/callbacks.rs b/compiler/rustc_interface/src/callbacks.rs
index ee0552d77..bc6d7c209 100644
--- a/compiler/rustc_interface/src/callbacks.rs
+++ b/compiler/rustc_interface/src/callbacks.rs
@@ -38,7 +38,7 @@ fn track_diagnostic(diagnostic: &mut Diagnostic, f: &mut dyn FnMut(&mut Diagnost
 
             // Diagnostics are tracked, we can ignore the dependency.
             let icx = tls::ImplicitCtxt { task_deps: TaskDepsRef::Ignore, ..icx.clone() };
-            return tls::enter_context(&icx, move |_| (*f)(diagnostic));
+            return tls::enter_context(&icx, move || (*f)(diagnostic));
         }
 
         // In any other case, invoke diagnostics anyway.
diff --git a/compiler/rustc_interface/src/errors.rs b/compiler/rustc_interface/src/errors.rs
index 15d7e977b..0eedee250 100644
--- a/compiler/rustc_interface/src/errors.rs
+++ b/compiler/rustc_interface/src/errors.rs
@@ -1,5 +1,7 @@
 use rustc_macros::Diagnostic;
+use rustc_session::config::CrateType;
 use rustc_span::{Span, Symbol};
+use rustc_target::spec::TargetTriple;
 
 use std::io;
 use std::path::Path;
@@ -30,10 +32,6 @@ pub struct MixedBinCrate;
 pub struct MixedProcMacroCrate;
 
 #[derive(Diagnostic)]
-#[diag(interface_proc_macro_doc_without_arg)]
-pub struct ProcMacroDocWithoutArg;
-
-#[derive(Diagnostic)]
 #[diag(interface_error_writing_dependencies)]
 pub struct ErrorWritingDependencies<'a> {
     pub path: &'a Path,
@@ -91,3 +89,22 @@ pub struct FailedWritingFile<'a> {
 #[derive(Diagnostic)]
 #[diag(interface_proc_macro_crate_panic_abort)]
 pub struct ProcMacroCratePanicAbort;
+
+#[derive(Diagnostic)]
+#[diag(interface_unsupported_crate_type_for_target)]
+pub struct UnsupportedCrateTypeForTarget<'a> {
+    pub crate_type: CrateType,
+    pub target_triple: &'a TargetTriple,
+}
+
+#[derive(Diagnostic)]
+#[diag(interface_multiple_output_types_adaption)]
+pub struct MultipleOutputTypesAdaption;
+
+#[derive(Diagnostic)]
+#[diag(interface_ignoring_extra_filename)]
+pub struct IgnoringExtraFilename;
+
+#[derive(Diagnostic)]
+#[diag(interface_ignoring_out_dir)]
+pub struct IgnoringOutDir;
diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index 7a5e45ada..5e38ca034 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -1,4 +1,3 @@
-pub use crate::passes::BoxedResolver;
 use crate::util;
 
 use rustc_ast::token;
@@ -223,6 +222,7 @@ pub struct Config {
     pub output_dir: Option<PathBuf>,
     pub output_file: Option<PathBuf>,
     pub file_loader: Option<Box<dyn FileLoader + Send + Sync>>,
+    pub locale_resources: &'static [&'static str],
 
     pub lint_caps: FxHashMap<lint::LintId, lint::Level>,
 
@@ -268,6 +268,7 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
                 config.opts,
                 config.crate_cfg,
                 config.crate_check_cfg,
+                config.locale_resources,
                 config.file_loader,
                 CompilerIO {
                     input: config.input,
diff --git a/compiler/rustc_interface/src/lib.rs b/compiler/rustc_interface/src/lib.rs
index 82bc4770b..1abbe8d4f 100644
--- a/compiler/rustc_interface/src/lib.rs
+++ b/compiler/rustc_interface/src/lib.rs
@@ -12,6 +12,9 @@
 #[macro_use]
 extern crate tracing;
 
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
+use rustc_macros::fluent_messages;
+
 mod callbacks;
 mod errors;
 pub mod interface;
@@ -27,3 +30,5 @@ pub use queries::Queries;
 
 #[cfg(test)]
 mod tests;
+
+fluent_messages! { "../locales/en-US.ftl" }
diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs
index 379a76528..81c1d665e 100644
--- a/compiler/rustc_interface/src/passes.rs
+++ b/compiler/rustc_interface/src/passes.rs
@@ -1,9 +1,4 @@
-use crate::errors::{
-    CantEmitMIR, EmojiIdentifier, ErrorWritingDependencies, FerrisIdentifier,
-    GeneratedFileConflictsWithDirectory, InputFileWouldBeOverWritten, MixedBinCrate,
-    MixedProcMacroCrate, OutDirError, ProcMacroCratePanicAbort, ProcMacroDocWithoutArg,
-    TempsDirError,
-};
+use crate::errors;
 use crate::interface::{Compiler, Result};
 use crate::proc_macro_decls;
 use crate::util;
@@ -13,11 +8,12 @@ use rustc_ast::{self as ast, visit};
 use rustc_borrowck as mir_borrowck;
 use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_data_structures::parallel;
+use rustc_data_structures::steal::Steal;
 use rustc_data_structures::sync::{Lrc, OnceCell, WorkerLocal};
-use rustc_errors::{ErrorGuaranteed, PResult};
+use rustc_errors::PResult;
 use rustc_expand::base::{ExtCtxt, LintStoreExpand, ResolverExpand};
-use rustc_hir::def_id::StableCrateId;
-use rustc_lint::{BufferedEarlyLint, EarlyCheckNode, LintStore};
+use rustc_hir::def_id::{StableCrateId, LOCAL_CRATE};
+use rustc_lint::{unerased_lint_store, BufferedEarlyLint, EarlyCheckNode, LintStore};
 use rustc_metadata::creader::CStore;
 use rustc_middle::arena::Arena;
 use rustc_middle::dep_graph::DepGraph;
@@ -28,9 +24,9 @@ use rustc_parse::{parse_crate_from_file, parse_crate_from_source_str, validate_a
 use rustc_passes::{self, hir_stats, layout_test};
 use rustc_plugin_impl as plugin;
 use rustc_query_impl::{OnDiskCache, Queries as TcxQueries};
-use rustc_resolve::{Resolver, ResolverArenas};
+use rustc_resolve::Resolver;
 use rustc_session::config::{CrateType, Input, OutputFilenames, OutputType};
-use rustc_session::cstore::{MetadataLoader, MetadataLoaderDyn, Untracked};
+use rustc_session::cstore::{CrateStoreDyn, MetadataLoader, Untracked};
 use rustc_session::output::filename_for_input;
 use rustc_session::search_paths::PathKind;
 use rustc_session::{Limit, Session};
@@ -40,14 +36,10 @@ use rustc_target::spec::PanicStrategy;
 use rustc_trait_selection::traits;
 
 use std::any::Any;
-use std::cell::RefCell;
 use std::ffi::OsString;
 use std::io::{self, BufWriter, Write};
-use std::marker::PhantomPinned;
 use std::path::{Path, PathBuf};
-use std::pin::Pin;
-use std::rc::Rc;
-use std::sync::LazyLock;
+use std::sync::{Arc, LazyLock};
 use std::{env, fs, iter};
 
 pub fn parse<'a>(sess: &'a Session) -> PResult<'a, ast::Crate> {
@@ -80,93 +72,6 @@ fn count_nodes(krate: &ast::Crate) -> usize {
     counter.count
 }
 
-pub use boxed_resolver::BoxedResolver;
-mod boxed_resolver {
-    use super::*;
-
-    pub struct BoxedResolver(Pin<Box<BoxedResolverInner>>);
-
-    struct BoxedResolverInner {
-        session: Lrc<Session>,
-        resolver_arenas: Option<ResolverArenas<'static>>,
-        resolver: Option<Resolver<'static>>,
-        _pin: PhantomPinned,
-    }
-
-    // Note: Drop order is important to prevent dangling references. Resolver must be dropped first,
-    // then resolver_arenas and session.
-    impl Drop for BoxedResolverInner {
-        fn drop(&mut self) {
-            self.resolver.take();
-            self.resolver_arenas.take();
-        }
-    }
-
-    impl BoxedResolver {
-        pub(super) fn new(
-            session: Lrc<Session>,
-            make_resolver: impl for<'a> FnOnce(&'a Session, &'a ResolverArenas<'a>) -> Resolver<'a>,
-        ) -> BoxedResolver {
-            let mut boxed_resolver = Box::new(BoxedResolverInner {
-                session,
-                resolver_arenas: Some(Resolver::arenas()),
-                resolver: None,
-                _pin: PhantomPinned,
-            });
-            // SAFETY: `make_resolver` takes a resolver arena with an arbitrary lifetime and
-            // returns a resolver with the same lifetime as the arena. We ensure that the arena
-            // outlives the resolver in the drop impl and elsewhere so these transmutes are sound.
-            unsafe {
-                let resolver = make_resolver(
-                    std::mem::transmute::<&Session, &Session>(&boxed_resolver.session),
-                    std::mem::transmute::<&ResolverArenas<'_>, &ResolverArenas<'_>>(
-                        boxed_resolver.resolver_arenas.as_ref().unwrap(),
-                    ),
-                );
-                boxed_resolver.resolver = Some(resolver);
-                BoxedResolver(Pin::new_unchecked(boxed_resolver))
-            }
-        }
-
-        pub fn access<F: for<'a> FnOnce(&mut Resolver<'a>) -> R, R>(&mut self, f: F) -> R {
-            // SAFETY: The resolver doesn't need to be pinned.
-            let mut resolver = unsafe {
-                self.0.as_mut().map_unchecked_mut(|boxed_resolver| &mut boxed_resolver.resolver)
-            };
-            f((&mut *resolver).as_mut().unwrap())
-        }
-
-        pub fn to_resolver_outputs(resolver: Rc<RefCell<BoxedResolver>>) -> ty::ResolverOutputs {
-            match Rc::try_unwrap(resolver) {
-                Ok(resolver) => {
-                    let mut resolver = resolver.into_inner();
-                    // SAFETY: The resolver doesn't need to be pinned.
-                    let mut resolver = unsafe {
-                        resolver
-                            .0
-                            .as_mut()
-                            .map_unchecked_mut(|boxed_resolver| &mut boxed_resolver.resolver)
-                    };
-                    resolver.take().unwrap().into_outputs()
-                }
-                Err(resolver) => resolver.borrow_mut().access(|resolver| resolver.clone_outputs()),
-            }
-        }
-    }
-}
-
-pub fn create_resolver(
-    sess: Lrc<Session>,
-    metadata_loader: Box<MetadataLoaderDyn>,
-    krate: &ast::Crate,
-    crate_name: Symbol,
-) -> BoxedResolver {
-    trace!("create_resolver");
-    BoxedResolver::new(sess, move |sess, resolver_arenas| {
-        Resolver::new(sess, krate, crate_name, metadata_loader, resolver_arenas)
-    })
-}
-
 pub fn register_plugins<'a>(
     sess: &'a Session,
     metadata_loader: &'a dyn MetadataLoader,
@@ -267,14 +172,12 @@ impl LintStoreExpand for LintStoreExpandImpl<'_> {
 /// syntax expansion, secondary `cfg` expansion, synthesis of a test
 /// harness if one is to be provided, injection of a dependency on the
 /// standard library and prelude, and name resolution.
-pub fn configure_and_expand(
-    sess: &Session,
-    lint_store: &LintStore,
-    mut krate: ast::Crate,
-    crate_name: Symbol,
-    resolver: &mut Resolver<'_>,
-) -> Result<ast::Crate> {
-    trace!("configure_and_expand");
+#[instrument(level = "trace", skip(krate, resolver))]
+fn configure_and_expand(mut krate: ast::Crate, resolver: &mut Resolver<'_, '_>) -> ast::Crate {
+    let tcx = resolver.tcx();
+    let sess = tcx.sess;
+    let lint_store = unerased_lint_store(tcx);
+    let crate_name = tcx.crate_name(LOCAL_CRATE);
     pre_expansion_lint(sess, lint_store, resolver.registered_tools(), &krate, crate_name);
     rustc_builtin_macros::register_builtin_macros(resolver);
 
@@ -345,20 +248,19 @@ pub fn configure_and_expand(
             ecx.check_unused_macros();
         });
 
-        let recursion_limit_hit = ecx.reduced_recursion_limit.is_some();
+        // If we hit a recursion limit, exit early to avoid later passes getting overwhelmed
+        // with a large AST
+        if ecx.reduced_recursion_limit.is_some() {
+            sess.abort_if_errors();
+            unreachable!();
+        }
 
         if cfg!(windows) {
             env::set_var("PATH", &old_path);
         }
 
-        if recursion_limit_hit {
-            // If we hit a recursion limit, exit early to avoid later passes getting overwhelmed
-            // with a large AST
-            Err(ErrorGuaranteed::unchecked_claim_error_was_emitted())
-        } else {
-            Ok(krate)
-        }
-    })?;
+        krate
+    });
 
     sess.time("maybe_building_test_harness", || {
         rustc_builtin_macros::test_harness::inject(sess, resolver, &mut krate)
@@ -374,39 +276,29 @@ pub fn configure_and_expand(
 
     if crate_types.len() > 1 {
         if is_executable_crate {
-            sess.emit_err(MixedBinCrate);
+            sess.emit_err(errors::MixedBinCrate);
         }
         if is_proc_macro_crate {
-            sess.emit_err(MixedProcMacroCrate);
+            sess.emit_err(errors::MixedProcMacroCrate);
         }
     }
 
     if is_proc_macro_crate && sess.panic_strategy() == PanicStrategy::Abort {
-        sess.emit_warning(ProcMacroCratePanicAbort);
+        sess.emit_warning(errors::ProcMacroCratePanicAbort);
     }
 
-    // For backwards compatibility, we don't try to run proc macro injection
-    // if rustdoc is run on a proc macro crate without '--crate-type proc-macro' being
-    // specified. This should only affect users who manually invoke 'rustdoc', as
-    // 'cargo doc' will automatically pass the proper '--crate-type' flags.
-    // However, we do emit a warning, to let such users know that they should
-    // start passing '--crate-type proc-macro'
-    if has_proc_macro_decls && sess.opts.actually_rustdoc && !is_proc_macro_crate {
-        sess.emit_warning(ProcMacroDocWithoutArg);
-    } else {
-        krate = sess.time("maybe_create_a_macro_crate", || {
-            let is_test_crate = sess.opts.test;
-            rustc_builtin_macros::proc_macro_harness::inject(
-                sess,
-                resolver,
-                krate,
-                is_proc_macro_crate,
-                has_proc_macro_decls,
-                is_test_crate,
-                sess.diagnostic(),
-            )
-        });
-    }
+    krate = sess.time("maybe_create_a_macro_crate", || {
+        let is_test_crate = sess.opts.test;
+        rustc_builtin_macros::proc_macro_harness::inject(
+            sess,
+            resolver,
+            krate,
+            is_proc_macro_crate,
+            has_proc_macro_decls,
+            is_test_crate,
+            sess.diagnostic(),
+        )
+    });
 
     // Done with macro expansion!
 
@@ -441,9 +333,9 @@ pub fn configure_and_expand(
             spans.sort();
             if ident == sym::ferris {
                 let first_span = spans[0];
-                sess.emit_err(FerrisIdentifier { spans, first_span });
+                sess.emit_err(errors::FerrisIdentifier { spans, first_span });
             } else {
-                sess.emit_err(EmojiIdentifier { spans, ident });
+                sess.emit_err(errors::EmojiIdentifier { spans, ident });
             }
         }
     });
@@ -461,7 +353,7 @@ pub fn configure_and_expand(
         )
     });
 
-    Ok(krate)
+    krate
 }
 
 // Returns all the paths that correspond to generated files.
@@ -548,7 +440,7 @@ fn escape_dep_env(symbol: Symbol) -> String {
 
 fn write_out_deps(
     sess: &Session,
-    boxed_resolver: &RefCell<BoxedResolver>,
+    cstore: &CrateStoreDyn,
     outputs: &OutputFilenames,
     out_filenames: &[PathBuf],
 ) {
@@ -600,20 +492,19 @@ fn write_out_deps(
                 }
             }
 
-            boxed_resolver.borrow_mut().access(|resolver| {
-                for cnum in resolver.cstore().crates_untracked() {
-                    let source = resolver.cstore().crate_source_untracked(cnum);
-                    if let Some((path, _)) = &source.dylib {
-                        files.push(escape_dep_filename(&path.display().to_string()));
-                    }
-                    if let Some((path, _)) = &source.rlib {
-                        files.push(escape_dep_filename(&path.display().to_string()));
-                    }
-                    if let Some((path, _)) = &source.rmeta {
-                        files.push(escape_dep_filename(&path.display().to_string()));
-                    }
+            let cstore = cstore.as_any().downcast_ref::<CStore>().unwrap();
+            for cnum in cstore.crates_untracked() {
+                let source = cstore.crate_source_untracked(cnum);
+                if let Some((path, _)) = &source.dylib {
+                    files.push(escape_dep_filename(&path.display().to_string()));
+                }
+                if let Some((path, _)) = &source.rlib {
+                    files.push(escape_dep_filename(&path.display().to_string()));
                 }
-            });
+                if let Some((path, _)) = &source.rmeta {
+                    files.push(escape_dep_filename(&path.display().to_string()));
+                }
+            }
         }
 
         let mut file = BufWriter::new(fs::File::create(&deps_filename)?);
@@ -656,18 +547,38 @@ fn write_out_deps(
             }
         }
         Err(error) => {
-            sess.emit_fatal(ErrorWritingDependencies { path: &deps_filename, error });
+            sess.emit_fatal(errors::ErrorWritingDependencies { path: &deps_filename, error });
         }
     }
 }
 
-pub fn prepare_outputs(
-    sess: &Session,
-    krate: &ast::Crate,
-    boxed_resolver: &RefCell<BoxedResolver>,
-    crate_name: Symbol,
-) -> Result<OutputFilenames> {
+fn resolver_for_lowering<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    (): (),
+) -> &'tcx Steal<(ty::ResolverAstLowering, Lrc<ast::Crate>)> {
+    let arenas = Resolver::arenas();
+    let krate = tcx.crate_for_resolver(()).steal();
+    let mut resolver = Resolver::new(tcx, &krate, &arenas);
+    let krate = configure_and_expand(krate, &mut resolver);
+
+    // Make sure we don't mutate the cstore from here on.
+    tcx.untracked().cstore.leak();
+
+    let ty::ResolverOutputs {
+        global_ctxt: untracked_resolutions,
+        ast_lowering: untracked_resolver_for_lowering,
+    } = resolver.into_outputs();
+
+    let feed = tcx.feed_unit_query();
+    feed.resolutions(tcx.arena.alloc(untracked_resolutions));
+    tcx.arena.alloc(Steal::new((untracked_resolver_for_lowering, Lrc::new(krate))))
+}
+
+fn output_filenames(tcx: TyCtxt<'_>, (): ()) -> Arc<OutputFilenames> {
+    let sess = tcx.sess;
     let _timer = sess.timer("prepare_outputs");
+    let (_, krate) = &*tcx.resolver_for_lowering(()).borrow();
+    let crate_name = tcx.crate_name(LOCAL_CRATE);
 
     // FIXME: rustdoc passes &[] instead of &krate.attrs here
     let outputs = util::build_output_filenames(&krate.attrs, sess);
@@ -679,25 +590,24 @@ pub fn prepare_outputs(
     if let Some(ref input_path) = sess.io.input.opt_path() {
         if sess.opts.will_create_output_file() {
             if output_contains_path(&output_paths, input_path) {
-                let reported = sess.emit_err(InputFileWouldBeOverWritten { path: input_path });
-                return Err(reported);
+                sess.emit_fatal(errors::InputFileWouldBeOverWritten { path: input_path });
             }
             if let Some(ref dir_path) = output_conflicts_with_dir(&output_paths) {
-                let reported =
-                    sess.emit_err(GeneratedFileConflictsWithDirectory { input_path, dir_path });
-                return Err(reported);
+                sess.emit_fatal(errors::GeneratedFileConflictsWithDirectory {
+                    input_path,
+                    dir_path,
+                });
             }
         }
     }
 
     if let Some(ref dir) = sess.io.temps_dir {
         if fs::create_dir_all(dir).is_err() {
-            let reported = sess.emit_err(TempsDirError);
-            return Err(reported);
+            sess.emit_fatal(errors::TempsDirError);
         }
     }
 
-    write_out_deps(sess, boxed_resolver, &outputs, &output_paths);
+    write_out_deps(sess, &*tcx.cstore_untracked(), &outputs, &output_paths);
 
     let only_dep_info = sess.opts.output_types.contains_key(&OutputType::DepInfo)
         && sess.opts.output_types.len() == 1;
@@ -705,19 +615,20 @@ pub fn prepare_outputs(
     if !only_dep_info {
         if let Some(ref dir) = sess.io.output_dir {
             if fs::create_dir_all(dir).is_err() {
-                let reported = sess.emit_err(OutDirError);
-                return Err(reported);
+                sess.emit_fatal(errors::OutDirError);
             }
         }
     }
 
-    Ok(outputs)
+    outputs.into()
 }
 
 pub static DEFAULT_QUERY_PROVIDERS: LazyLock<Providers> = LazyLock::new(|| {
     let providers = &mut Providers::default();
     providers.analysis = analysis;
     providers.hir_crate = rustc_ast_lowering::lower_to_hir;
+    providers.output_filenames = output_filenames;
+    providers.resolver_for_lowering = resolver_for_lowering;
     proc_macro_decls::provide(providers);
     rustc_const_eval::provide(providers);
     rustc_middle::hir::provide(providers);
@@ -747,30 +658,16 @@ pub static DEFAULT_EXTERN_QUERY_PROVIDERS: LazyLock<ExternProviders> = LazyLock:
     extern_providers
 });
 
-pub struct QueryContext<'tcx> {
-    gcx: &'tcx GlobalCtxt<'tcx>,
-}
-
-impl<'tcx> QueryContext<'tcx> {
-    pub fn enter<F, R>(&mut self, f: F) -> R
-    where
-        F: FnOnce(TyCtxt<'tcx>) -> R,
-    {
-        let icx = ty::tls::ImplicitCtxt::new(self.gcx);
-        ty::tls::enter_context(&icx, |_| f(icx.tcx))
-    }
-}
-
 pub fn create_global_ctxt<'tcx>(
     compiler: &'tcx Compiler,
     lint_store: Lrc<LintStore>,
     dep_graph: DepGraph,
     untracked: Untracked,
     queries: &'tcx OnceCell<TcxQueries<'tcx>>,
-    global_ctxt: &'tcx OnceCell<GlobalCtxt<'tcx>>,
+    gcx_cell: &'tcx OnceCell<GlobalCtxt<'tcx>>,
     arena: &'tcx WorkerLocal<Arena<'tcx>>,
     hir_arena: &'tcx WorkerLocal<rustc_hir::Arena<'tcx>>,
-) -> QueryContext<'tcx> {
+) -> &'tcx GlobalCtxt<'tcx> {
     // We're constructing the HIR here; we don't care what we will
     // read, since we haven't even constructed the *input* to
     // incr. comp. yet.
@@ -794,8 +691,8 @@ pub fn create_global_ctxt<'tcx>(
         TcxQueries::new(local_providers, extern_providers, query_result_on_disk_cache)
     });
 
-    let gcx = sess.time("setup_global_ctxt", || {
-        global_ctxt.get_or_init(move || {
+    sess.time("setup_global_ctxt", || {
+        gcx_cell.get_or_init(move || {
             TyCtxt::create_global_ctxt(
                 sess,
                 lint_store,
@@ -808,9 +705,7 @@ pub fn create_global_ctxt<'tcx>(
                 rustc_query_impl::query_callbacks(arena),
             )
         })
-    });
-
-    QueryContext { gcx }
+    })
 }
 
 /// Runs the resolution, type-checking, region checking and other
@@ -900,6 +795,15 @@ fn analysis(tcx: TyCtxt<'_>, (): ()) -> Result<()> {
         }
     });
 
+    if tcx.sess.opts.unstable_opts.drop_tracking_mir {
+        tcx.hir().par_body_owners(|def_id| {
+            if let rustc_hir::def::DefKind::Generator = tcx.def_kind(def_id) {
+                tcx.ensure().mir_generator_witnesses(def_id);
+                tcx.ensure().check_generator_obligations(def_id);
+            }
+        });
+    }
+
     sess.time("layout_testing", || layout_test::test_layout(tcx));
 
     // Avoid overwhelming user with errors if borrow checking failed.
@@ -975,7 +879,7 @@ pub fn start_codegen<'tcx>(
 
     if tcx.sess.opts.output_types.contains_key(&OutputType::Mir) {
         if let Err(error) = rustc_mir_transform::dump_mir::emit_mir(tcx) {
-            tcx.sess.emit_err(CantEmitMIR { error });
+            tcx.sess.emit_err(errors::CantEmitMIR { error });
             tcx.sess.abort_if_errors();
         }
     }
diff --git a/compiler/rustc_interface/src/queries.rs b/compiler/rustc_interface/src/queries.rs
index d5a49dd75..a96cc95a3 100644
--- a/compiler/rustc_interface/src/queries.rs
+++ b/compiler/rustc_interface/src/queries.rs
@@ -1,27 +1,29 @@
 use crate::errors::{FailedWritingFile, RustcErrorFatal, RustcErrorUnexpectedAnnotation};
 use crate::interface::{Compiler, Result};
-use crate::passes::{self, BoxedResolver, QueryContext};
+use crate::passes;
 
 use rustc_ast as ast;
 use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_codegen_ssa::CodegenResults;
 use rustc_data_structures::steal::Steal;
 use rustc_data_structures::svh::Svh;
-use rustc_data_structures::sync::{Lrc, OnceCell, WorkerLocal};
-use rustc_hir::def_id::LOCAL_CRATE;
+use rustc_data_structures::sync::{AppendOnlyVec, Lrc, OnceCell, RwLock, WorkerLocal};
+use rustc_hir::def_id::{CRATE_DEF_ID, LOCAL_CRATE};
+use rustc_hir::definitions::Definitions;
 use rustc_incremental::DepGraphFuture;
 use rustc_lint::LintStore;
+use rustc_metadata::creader::CStore;
 use rustc_middle::arena::Arena;
 use rustc_middle::dep_graph::DepGraph;
-use rustc_middle::ty::{self, GlobalCtxt, TyCtxt};
+use rustc_middle::ty::{GlobalCtxt, TyCtxt};
 use rustc_query_impl::Queries as TcxQueries;
 use rustc_session::config::{self, OutputFilenames, OutputType};
+use rustc_session::cstore::Untracked;
 use rustc_session::{output::find_crate_name, Session};
 use rustc_span::symbol::sym;
 use rustc_span::Symbol;
 use std::any::Any;
 use std::cell::{RefCell, RefMut};
-use std::rc::Rc;
 use std::sync::Arc;
 
 /// Represent the result of a query.
@@ -64,8 +66,8 @@ impl<'a, T> std::ops::DerefMut for QueryResult<'a, T> {
     }
 }
 
-impl<'a, 'tcx> QueryResult<'a, QueryContext<'tcx>> {
-    pub fn enter<T>(mut self, f: impl FnOnce(TyCtxt<'tcx>) -> T) -> T {
+impl<'a, 'tcx> QueryResult<'a, &'tcx GlobalCtxt<'tcx>> {
+    pub fn enter<T>(&mut self, f: impl FnOnce(TyCtxt<'tcx>) -> T) -> T {
         (*self.0).get_mut().enter(f)
     }
 }
@@ -78,7 +80,7 @@ impl<T> Default for Query<T> {
 
 pub struct Queries<'tcx> {
     compiler: &'tcx Compiler,
-    gcx: OnceCell<GlobalCtxt<'tcx>>,
+    gcx_cell: OnceCell<GlobalCtxt<'tcx>>,
     queries: OnceCell<TcxQueries<'tcx>>,
 
     arena: WorkerLocal<Arena<'tcx>>,
@@ -88,9 +90,9 @@ pub struct Queries<'tcx> {
     parse: Query<ast::Crate>,
     crate_name: Query<Symbol>,
     register_plugins: Query<(ast::Crate, Lrc<LintStore>)>,
-    expansion: Query<(Lrc<ast::Crate>, Rc<RefCell<BoxedResolver>>, Lrc<LintStore>)>,
     dep_graph: Query<DepGraph>,
-    global_ctxt: Query<QueryContext<'tcx>>,
+    // This just points to what's in `gcx_cell`.
+    gcx: Query<&'tcx GlobalCtxt<'tcx>>,
     ongoing_codegen: Query<Box<dyn Any>>,
 }
 
@@ -98,7 +100,7 @@ impl<'tcx> Queries<'tcx> {
     pub fn new(compiler: &'tcx Compiler) -> Queries<'tcx> {
         Queries {
             compiler,
-            gcx: OnceCell::new(),
+            gcx_cell: OnceCell::new(),
             queries: OnceCell::new(),
             arena: WorkerLocal::new(|_| Arena::default()),
             hir_arena: WorkerLocal::new(|_| rustc_hir::Arena::default()),
@@ -106,9 +108,8 @@ impl<'tcx> Queries<'tcx> {
             parse: Default::default(),
             crate_name: Default::default(),
             register_plugins: Default::default(),
-            expansion: Default::default(),
             dep_graph: Default::default(),
-            global_ctxt: Default::default(),
+            gcx: Default::default(),
             ongoing_codegen: Default::default(),
         }
     }
@@ -168,29 +169,6 @@ impl<'tcx> Queries<'tcx> {
         })
     }
 
-    pub fn expansion(
-        &self,
-    ) -> Result<QueryResult<'_, (Lrc<ast::Crate>, Rc<RefCell<BoxedResolver>>, Lrc<LintStore>)>>
-    {
-        trace!("expansion");
-        self.expansion.compute(|| {
-            let crate_name = *self.crate_name()?.borrow();
-            let (krate, lint_store) = self.register_plugins()?.steal();
-            let _timer = self.session().timer("configure_and_expand");
-            let sess = self.session();
-            let mut resolver = passes::create_resolver(
-                sess.clone(),
-                self.codegen_backend().metadata_loader(),
-                &krate,
-                crate_name,
-            );
-            let krate = resolver.access(|resolver| {
-                passes::configure_and_expand(sess, &lint_store, krate, crate_name, resolver)
-            })?;
-            Ok((Lrc::new(krate), Rc::new(RefCell::new(resolver)), lint_store))
-        })
-    }
-
     fn dep_graph(&self) -> Result<QueryResult<'_, DepGraph>> {
         self.dep_graph.compute(|| {
             let sess = self.session();
@@ -207,40 +185,41 @@ impl<'tcx> Queries<'tcx> {
         })
     }
 
-    pub fn global_ctxt(&'tcx self) -> Result<QueryResult<'_, QueryContext<'tcx>>> {
-        self.global_ctxt.compute(|| {
+    pub fn global_ctxt(&'tcx self) -> Result<QueryResult<'_, &'tcx GlobalCtxt<'tcx>>> {
+        self.gcx.compute(|| {
             let crate_name = *self.crate_name()?.borrow();
-            let (krate, resolver, lint_store) = self.expansion()?.steal();
+            let (krate, lint_store) = self.register_plugins()?.steal();
 
-            let outputs = passes::prepare_outputs(self.session(), &krate, &resolver, crate_name)?;
+            let sess = self.session();
 
-            let ty::ResolverOutputs {
-                untracked,
-                global_ctxt: untracked_resolutions,
-                ast_lowering: untracked_resolver_for_lowering,
-            } = BoxedResolver::to_resolver_outputs(resolver);
+            let cstore = RwLock::new(Box::new(CStore::new(sess)) as _);
+            let definitions = RwLock::new(Definitions::new(sess.local_stable_crate_id()));
+            let source_span = AppendOnlyVec::new();
+            let _id = source_span.push(krate.spans.inner_span);
+            debug_assert_eq!(_id, CRATE_DEF_ID);
+            let untracked = Untracked { cstore, source_span, definitions };
 
-            let mut qcx = passes::create_global_ctxt(
+            let qcx = passes::create_global_ctxt(
                 self.compiler,
                 lint_store,
                 self.dep_graph()?.steal(),
                 untracked,
                 &self.queries,
-                &self.gcx,
+                &self.gcx_cell,
                 &self.arena,
                 &self.hir_arena,
             );
 
             qcx.enter(|tcx| {
+                let feed = tcx.feed_local_crate();
+                feed.crate_name(crate_name);
+
                 let feed = tcx.feed_unit_query();
-                feed.resolver_for_lowering(
-                    tcx.arena.alloc(Steal::new((untracked_resolver_for_lowering, krate))),
+                feed.crate_for_resolver(tcx.arena.alloc(Steal::new(krate)));
+                feed.metadata_loader(
+                    tcx.arena.alloc(Steal::new(self.codegen_backend().metadata_loader())),
                 );
-                feed.resolutions(tcx.arena.alloc(untracked_resolutions));
-                feed.output_filenames(tcx.arena.alloc(std::sync::Arc::new(outputs)));
                 feed.features_query(tcx.sess.features_untracked());
-                let feed = tcx.feed_local_crate();
-                feed.crate_name(crate_name);
             });
             Ok(qcx)
         })
@@ -390,7 +369,7 @@ impl Compiler {
 
         // NOTE: intentionally does not compute the global context if it hasn't been built yet,
         // since that likely means there was a parse error.
-        if let Some(Ok(gcx)) = &mut *queries.global_ctxt.result.borrow_mut() {
+        if let Some(Ok(gcx)) = &mut *queries.gcx.result.borrow_mut() {
             let gcx = gcx.get_mut();
             // We assume that no queries are run past here. If there are new queries
             // after this point, they'll show up as "<unknown>" in self-profiling data.
diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index f94bc4d4c..18d84a702 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -5,6 +5,7 @@ use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::{emitter::HumanReadableErrorType, registry, ColorConfig};
 use rustc_session::config::rustc_optgroups;
 use rustc_session::config::Input;
+use rustc_session::config::InstrumentXRay;
 use rustc_session::config::TraitSolver;
 use rustc_session::config::{build_configuration, build_session_options, to_crate_config};
 use rustc_session::config::{
@@ -49,7 +50,7 @@ fn mk_session(matches: getopts::Matches) -> (Session, CfgSpecs) {
         output_file: None,
         temps_dir,
     };
-    let sess = build_session(sessopts, io, None, registry, Default::default(), None, None);
+    let sess = build_session(sessopts, io, None, registry, vec![], Default::default(), None, None);
     (sess, cfg)
 }
 
@@ -690,7 +691,6 @@ fn test_unstable_options_tracking_hash() {
     untracked!(proc_macro_execution_strategy, ProcMacroExecutionStrategy::CrossThread);
     untracked!(profile_closures, true);
     untracked!(query_dep_graph, true);
-    untracked!(save_analysis, true);
     untracked!(self_profile, SwitchWithOptPath::Enabled(None));
     untracked!(self_profile_events, Some(vec![String::new()]));
     untracked!(span_debug, true);
@@ -755,10 +755,11 @@ fn test_unstable_options_tracking_hash() {
     tracked!(inline_mir_threshold, Some(123));
     tracked!(instrument_coverage, Some(InstrumentCoverage::All));
     tracked!(instrument_mcount, true);
+    tracked!(instrument_xray, Some(InstrumentXRay::default()));
+    tracked!(link_directives, false);
     tracked!(link_only, true);
     tracked!(llvm_plugins, vec![String::from("plugin_name")]);
     tracked!(location_detail, LocationDetail { file: true, line: false, column: false });
-    tracked!(log_backtrace, Some("filter".to_string()));
     tracked!(maximal_hir_to_mir_coverage, true);
     tracked!(merge_functions, Some(MergeFunctions::Disabled));
     tracked!(mir_emit_retag, true);
@@ -776,7 +777,6 @@ fn test_unstable_options_tracking_hash() {
     tracked!(packed_bundled_libs, true);
     tracked!(panic_abort_tests, true);
     tracked!(panic_in_drop, PanicStrategy::Abort);
-    tracked!(pick_stable_methods_before_any_unstable, false);
     tracked!(plt, Some(true));
     tracked!(polonius, true);
     tracked!(precise_enum_drop_elaboration, false);
@@ -802,6 +802,7 @@ fn test_unstable_options_tracking_hash() {
     tracked!(teach, true);
     tracked!(thinlto, Some(true));
     tracked!(thir_unsafeck, true);
+    tracked!(tiny_const_eval_limit, true);
     tracked!(tls_model, Some(TlsModel::GeneralDynamic));
     tracked!(trait_solver, TraitSolver::Chalk);
     tracked!(translate_remapped_path_to_local_path, false);
diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs
index 54363e07b..e5d2fb2ea 100644
--- a/compiler/rustc_interface/src/util.rs
+++ b/compiler/rustc_interface/src/util.rs
@@ -1,3 +1,4 @@
+use crate::errors;
 use info;
 use libloading::Library;
 use rustc_ast as ast;
@@ -13,8 +14,8 @@ use rustc_session::filesearch::sysroot_candidates;
 use rustc_session::lint::{self, BuiltinLintDiagnostics, LintBuffer};
 use rustc_session::parse::CrateConfig;
 use rustc_session::{early_error, filesearch, output, Session};
+use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::edition::Edition;
-use rustc_span::lev_distance::find_best_match_for_name;
 use rustc_span::source_map::FileLoader;
 use rustc_span::symbol::{sym, Symbol};
 use session::CompilerIO;
@@ -58,6 +59,7 @@ pub fn create_session(
     sopts: config::Options,
     cfg: FxHashSet<(String, Option<String>)>,
     check_cfg: CheckCfg,
+    locale_resources: &'static [&'static str],
     file_loader: Option<Box<dyn FileLoader + Send + Sync + 'static>>,
     io: CompilerIO,
     lint_caps: FxHashMap<lint::LintId, lint::Level>,
@@ -88,11 +90,15 @@ pub fn create_session(
         }
     };
 
+    let mut locale_resources = Vec::from(locale_resources);
+    locale_resources.push(codegen_backend.locale_resource());
+
     let mut sess = session::build_session(
         sopts,
         io,
         bundle,
         descriptions,
+        locale_resources,
         lint_caps,
         file_loader,
         target_override,
@@ -472,16 +478,15 @@ pub fn collect_crate_types(session: &Session, attrs: &[ast::Attribute]) -> Vec<C
     }
 
     base.retain(|crate_type| {
-        let res = !output::invalid_output_for_target(session, *crate_type);
-
-        if !res {
-            session.warn(&format!(
-                "dropping unsupported crate type `{}` for target `{}`",
-                *crate_type, session.opts.target_triple
-            ));
+        if output::invalid_output_for_target(session, *crate_type) {
+            session.emit_warning(errors::UnsupportedCrateTypeForTarget {
+                crate_type: *crate_type,
+                target_triple: &session.opts.target_triple,
+            });
+            false
+        } else {
+            true
         }
-
-        res
     });
 
     base
@@ -517,19 +522,16 @@ pub fn build_output_filenames(attrs: &[ast::Attribute], sess: &Session) -> Outpu
             let unnamed_output_types =
                 sess.opts.output_types.values().filter(|a| a.is_none()).count();
             let ofile = if unnamed_output_types > 1 {
-                sess.warn(
-                    "due to multiple output types requested, the explicitly specified \
-                     output file name will be adapted for each output type",
-                );
+                sess.emit_warning(errors::MultipleOutputTypesAdaption);
                 None
             } else {
                 if !sess.opts.cg.extra_filename.is_empty() {
-                    sess.warn("ignoring -C extra-filename flag due to -o flag");
+                    sess.emit_warning(errors::IgnoringExtraFilename);
                 }
                 Some(out_file.clone())
             };
             if sess.io.output_dir != None {
-                sess.warn("ignoring --out-dir flag due to -o flag");
+                sess.emit_warning(errors::IgnoringOutDir);
             }
 
             OutputFilenames::new(
diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs
index 6e815863d..b3f4b5cd5 100644
--- a/compiler/rustc_lexer/src/lib.rs
+++ b/compiler/rustc_lexer/src/lib.rs
@@ -166,15 +166,17 @@ pub enum DocStyle {
     Inner,
 }
 
-// Note that the suffix is *not* considered when deciding the `LiteralKind` in
-// this type. This means that float literals like `1f32` are classified by this
-// type as `Int`. (Compare against `rustc_ast::token::LitKind` and
-// `rustc_ast::ast::LitKind.)
+/// Enum representing the literal types supported by the lexer.
+///
+/// Note that the suffix is *not* considered when deciding the `LiteralKind` in
+/// this type. This means that float literals like `1f32` are classified by this
+/// type as `Int`. (Compare against `rustc_ast::token::LitKind` and
+/// `rustc_ast::ast::LitKind`).
 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
 pub enum LiteralKind {
     /// "12_u8", "0o100", "0b120i99", "1f32".
     Int { base: Base, empty_int: bool },
-    /// "12.34f32", "1e3", but not "1f32`.
+    /// "12.34f32", "1e3", but not "1f32".
     Float { base: Base, empty_exponent: bool },
     /// "'a'", "'\\'", "'''", "';"
     Char { terminated: bool },
diff --git a/compiler/rustc_error_messages/locales/en-US/lint.ftl b/compiler/rustc_lint/locales/en-US.ftl
index d63ff77d8..68e62c978 100644
--- a/compiler/rustc_error_messages/locales/en-US/lint.ftl
+++ b/compiler/rustc_lint/locales/en-US.ftl
@@ -24,6 +24,13 @@ lint_for_loops_over_fallibles =
     .use_while_let = to check pattern in a loop use `while let`
     .use_question_mark = consider unwrapping the `Result` with `?` to iterate over its contents
 
+lint_map_unit_fn = `Iterator::map` call that discard the iterator's values
+    .note = `Iterator::map`, like many of the methods on `Iterator`, gets executed lazily, meaning that its effects won't be visible until it is iterated
+    .function_label = this function returns `()`, which is likely not what you wanted
+    .argument_label = called `Iterator::map` with callable that returns `()`
+    .map_label = after this call to map, the resulting iterator is `impl Iterator<Item = ()>`, which means the only information carried by the iterator is the number of items
+    .suggestion = you might have meant to use `Iterator::for_each`
+
 lint_non_binding_let_on_sync_lock =
     non-binding let on a synchronization lock
 
@@ -100,6 +107,8 @@ lint_cstring_ptr = getting the inner pointer of a temporary `CString`
     .note = pointers do not have a lifetime; when calling `as_ptr` the `CString` will be deallocated at the end of the statement because nothing is referencing it as far as the type system is concerned
     .help = for more information, see https://doc.rust-lang.org/reference/destructors.html
 
+lint_multple_supertrait_upcastable = `{$ident}` is object-safe and has multiple supertraits
+
 lint_identifier_non_ascii_char = identifier contains non-ASCII characters
 
 lint_identifier_uncommon_codepoints = identifier contains uncommon Unicode codepoints
@@ -307,6 +316,7 @@ lint_unused_generator =
     .note = generators are lazy and do nothing unless resumed
 
 lint_unused_def = unused {$pre}`{$def}`{$post} that must be used
+    .suggestion = use `let _ = ...` to ignore the resulting value
 
 lint_path_statement_drop = path statement drops value
     .suggestion = use `drop` to clarify the intent
diff --git a/compiler/rustc_lint/src/array_into_iter.rs b/compiler/rustc_lint/src/array_into_iter.rs
index 3593f141d..bccb0a94e 100644
--- a/compiler/rustc_lint/src/array_into_iter.rs
+++ b/compiler/rustc_lint/src/array_into_iter.rs
@@ -1,5 +1,7 @@
-use crate::lints::{ArrayIntoIterDiag, ArrayIntoIterDiagSub};
-use crate::{LateContext, LateLintPass, LintContext};
+use crate::{
+    lints::{ArrayIntoIterDiag, ArrayIntoIterDiagSub},
+    LateContext, LateLintPass, LintContext,
+};
 use rustc_hir as hir;
 use rustc_middle::ty;
 use rustc_middle::ty::adjustment::{Adjust, Adjustment};
diff --git a/compiler/rustc_lint/src/builtin.rs b/compiler/rustc_lint/src/builtin.rs
index fe188162c..5b2100b5d 100644
--- a/compiler/rustc_lint/src/builtin.rs
+++ b/compiler/rustc_lint/src/builtin.rs
@@ -20,6 +20,7 @@
 //! If you define a new `LateLintPass`, you will also need to add it to the
 //! `late_lint_methods!` invocation in `lib.rs`.
 
+use crate::fluent_generated as fluent;
 use crate::{
     errors::BuiltinEllpisisInclusiveRangePatterns,
     lints::{
@@ -50,15 +51,13 @@ use rustc_ast::{self as ast, *};
 use rustc_ast_pretty::pprust::{self, expr_to_string};
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::stack::ensure_sufficient_stack;
-use rustc_errors::{fluent, Applicability, DecorateLint, MultiSpan};
+use rustc_errors::{Applicability, DecorateLint, MultiSpan};
 use rustc_feature::{deprecated_attributes, AttributeGate, BuiltinAttribute, GateIssue, Stability};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::{DefId, LocalDefId, LocalDefIdSet, CRATE_DEF_ID};
 use rustc_hir::intravisit::FnKind as HirFnKind;
-use rustc_hir::{
-    Body, FnDecl, ForeignItemKind, GenericParamKind, HirId, Node, PatKind, PredicateOrigin,
-};
+use rustc_hir::{Body, FnDecl, ForeignItemKind, GenericParamKind, Node, PatKind, PredicateOrigin};
 use rustc_index::vec::Idx;
 use rustc_middle::lint::in_external_macro;
 use rustc_middle::ty::layout::{LayoutError, LayoutOf};
@@ -185,7 +184,7 @@ impl<'tcx> LateLintPass<'tcx> for BoxPointers {
             | hir::ItemKind::Enum(..)
             | hir::ItemKind::Struct(..)
             | hir::ItemKind::Union(..) => {
-                self.check_heap_type(cx, it.span, cx.tcx.type_of(it.owner_id))
+                self.check_heap_type(cx, it.span, cx.tcx.type_of(it.owner_id).subst_identity())
             }
             _ => (),
         }
@@ -194,7 +193,11 @@ impl<'tcx> LateLintPass<'tcx> for BoxPointers {
         match it.kind {
             hir::ItemKind::Struct(ref struct_def, _) | hir::ItemKind::Union(ref struct_def, _) => {
                 for field in struct_def.fields() {
-                    self.check_heap_type(cx, field.span, cx.tcx.type_of(field.def_id));
+                    self.check_heap_type(
+                        cx,
+                        field.span,
+                        cx.tcx.type_of(field.def_id).subst_identity(),
+                    );
                 }
             }
             _ => (),
@@ -582,26 +585,28 @@ impl<'tcx> LateLintPass<'tcx> for MissingDoc {
     }
 
     fn check_impl_item(&mut self, cx: &LateContext<'_>, impl_item: &hir::ImplItem<'_>) {
-        // If the method is an impl for a trait, don't doc.
-        if method_context(cx, impl_item.hir_id()) == MethodLateContext::TraitImpl {
-            return;
-        }
-
-        // If the method is an impl for an item with docs_hidden, don't doc.
-        if method_context(cx, impl_item.hir_id()) == MethodLateContext::PlainImpl {
-            let parent = cx.tcx.hir().get_parent_item(impl_item.hir_id());
-            let impl_ty = cx.tcx.type_of(parent);
-            let outerdef = match impl_ty.kind() {
-                ty::Adt(def, _) => Some(def.did()),
-                ty::Foreign(def_id) => Some(*def_id),
-                _ => None,
-            };
-            let is_hidden = match outerdef {
-                Some(id) => cx.tcx.is_doc_hidden(id),
-                None => false,
-            };
-            if is_hidden {
-                return;
+        let context = method_context(cx, impl_item.owner_id.def_id);
+
+        match context {
+            // If the method is an impl for a trait, don't doc.
+            MethodLateContext::TraitImpl => return,
+            MethodLateContext::TraitAutoImpl => {}
+            // If the method is an impl for an item with docs_hidden, don't doc.
+            MethodLateContext::PlainImpl => {
+                let parent = cx.tcx.hir().get_parent_item(impl_item.hir_id());
+                let impl_ty = cx.tcx.type_of(parent).subst_identity();
+                let outerdef = match impl_ty.kind() {
+                    ty::Adt(def, _) => Some(def.did()),
+                    ty::Foreign(def_id) => Some(*def_id),
+                    _ => None,
+                };
+                let is_hidden = match outerdef {
+                    Some(id) => cx.tcx.is_doc_hidden(id),
+                    None => false,
+                };
+                if is_hidden {
+                    return;
+                }
             }
         }
 
@@ -672,21 +677,21 @@ impl<'tcx> LateLintPass<'tcx> for MissingCopyImplementations {
                     return;
                 }
                 let def = cx.tcx.adt_def(item.owner_id);
-                (def, cx.tcx.mk_adt(def, cx.tcx.intern_substs(&[])))
+                (def, cx.tcx.mk_adt(def, ty::List::empty()))
             }
             hir::ItemKind::Union(_, ref ast_generics) => {
                 if !ast_generics.params.is_empty() {
                     return;
                 }
                 let def = cx.tcx.adt_def(item.owner_id);
-                (def, cx.tcx.mk_adt(def, cx.tcx.intern_substs(&[])))
+                (def, cx.tcx.mk_adt(def, ty::List::empty()))
             }
             hir::ItemKind::Enum(_, ref ast_generics) => {
                 if !ast_generics.params.is_empty() {
                     return;
                 }
                 let def = cx.tcx.adt_def(item.owner_id);
-                (def, cx.tcx.mk_adt(def, cx.tcx.intern_substs(&[])))
+                (def, cx.tcx.mk_adt(def, ty::List::empty()))
             }
             _ => return,
         };
@@ -698,7 +703,7 @@ impl<'tcx> LateLintPass<'tcx> for MissingCopyImplementations {
         // and recommending Copy might be a bad idea.
         for field in def.all_fields() {
             let did = field.did;
-            if cx.tcx.type_of(did).is_unsafe_ptr() {
+            if cx.tcx.type_of(did).subst_identity().is_unsafe_ptr() {
                 return;
             }
         }
@@ -732,7 +737,7 @@ impl<'tcx> LateLintPass<'tcx> for MissingCopyImplementations {
             cx.tcx,
             param_env,
             ty,
-            traits::ObligationCause::misc(item.span, item.hir_id()),
+            traits::ObligationCause::misc(item.span, item.owner_id.def_id),
         )
         .is_ok()
         {
@@ -798,7 +803,7 @@ impl<'tcx> LateLintPass<'tcx> for MissingDebugImplementations {
         if self.impling_types.is_none() {
             let mut impls = LocalDefIdSet::default();
             cx.tcx.for_each_impl(debug, |d| {
-                if let Some(ty_def) = cx.tcx.type_of(d).ty_adt_def() {
+                if let Some(ty_def) = cx.tcx.type_of(d).subst_identity().ty_adt_def() {
                     if let Some(def_id) = ty_def.did().as_local() {
                         impls.insert(def_id);
                     }
@@ -1283,7 +1288,7 @@ declare_lint! {
 }
 
 declare_lint_pass!(
-    /// Explains corresponding feature flag must be enabled for the `#[track_caller] attribute to
+    /// Explains corresponding feature flag must be enabled for the `#[track_caller]` attribute to
     /// do anything
     UngatedAsyncFnTrackCaller => [UNGATED_ASYNC_FN_TRACK_CALLER]
 );
@@ -1296,19 +1301,18 @@ impl<'tcx> LateLintPass<'tcx> for UngatedAsyncFnTrackCaller {
         _: &'tcx FnDecl<'_>,
         _: &'tcx Body<'_>,
         span: Span,
-        hir_id: HirId,
+        def_id: LocalDefId,
     ) {
         if fn_kind.asyncness() == IsAsync::Async
             && !cx.tcx.features().closure_track_caller
-            && let attrs = cx.tcx.hir().attrs(hir_id)
             // Now, check if the function has the `#[track_caller]` attribute
-            && let Some(attr) = attrs.iter().find(|attr| attr.has_name(sym::track_caller))
-            {
-                cx.emit_spanned_lint(UNGATED_ASYNC_FN_TRACK_CALLER, attr.span, BuiltinUngatedAsyncFnTrackCaller {
-                    label: span,
-                    parse_sess: &cx.tcx.sess.parse_sess,
-                });
-            }
+            && let Some(attr) = cx.tcx.get_attr(def_id.to_def_id(), sym::track_caller)
+        {
+            cx.emit_spanned_lint(UNGATED_ASYNC_FN_TRACK_CALLER, attr.span, BuiltinUngatedAsyncFnTrackCaller {
+                label: span,
+                parse_sess: &cx.tcx.sess.parse_sess,
+            });
+        }
     }
 }
 
@@ -1580,7 +1584,7 @@ declare_lint_pass!(
 
 impl<'tcx> LateLintPass<'tcx> for TrivialConstraints {
     fn check_item(&mut self, cx: &LateContext<'tcx>, item: &'tcx hir::Item<'tcx>) {
-        use rustc_middle::ty::visit::TypeVisitable;
+        use rustc_middle::ty::visit::TypeVisitableExt;
         use rustc_middle::ty::Clause;
         use rustc_middle::ty::PredicateKind::*;
 
@@ -1592,15 +1596,19 @@ impl<'tcx> LateLintPass<'tcx> for TrivialConstraints {
                     Clause(Clause::TypeOutlives(..)) |
                     Clause(Clause::RegionOutlives(..)) => "lifetime",
 
+                    // `ConstArgHasType` is never global as `ct` is always a param
+                    Clause(Clause::ConstArgHasType(..)) |
                     // Ignore projections, as they can only be global
                     // if the trait bound is global
                     Clause(Clause::Projection(..)) |
+                    AliasEq(..) |
                     // Ignore bounds that a user can't type
                     WellFormed(..) |
                     ObjectSafe(..) |
                     ClosureKind(..) |
                     Subtype(..) |
                     Coerce(..) |
+                    // FIXME(generic_const_exprs): `ConstEvaluatable` can be written
                     ConstEvaluatable(..) |
                     ConstEquate(..) |
                     Ambiguous |
@@ -2006,7 +2014,7 @@ impl ExplicitOutlivesRequirements {
         inferred_outlives: &[ty::Region<'tcx>],
         predicate_span: Span,
     ) -> Vec<(usize, Span)> {
-        use rustc_middle::middle::resolve_lifetime::Region;
+        use rustc_middle::middle::resolve_bound_vars::ResolvedArg;
 
         bounds
             .iter()
@@ -2016,8 +2024,8 @@ impl ExplicitOutlivesRequirements {
                     return None;
                 };
 
-                let is_inferred = match tcx.named_region(lifetime.hir_id) {
-                    Some(Region::EarlyBound(def_id)) => inferred_outlives
+                let is_inferred = match tcx.named_bound_var(lifetime.hir_id) {
+                    Some(ResolvedArg::EarlyBound(def_id)) => inferred_outlives
                         .iter()
                         .any(|r| matches!(**r, ty::ReEarlyBound(ebr) if { ebr.def_id == def_id })),
                     _ => false,
@@ -2096,7 +2104,7 @@ impl ExplicitOutlivesRequirements {
 
 impl<'tcx> LateLintPass<'tcx> for ExplicitOutlivesRequirements {
     fn check_item(&mut self, cx: &LateContext<'tcx>, item: &'tcx hir::Item<'_>) {
-        use rustc_middle::middle::resolve_lifetime::Region;
+        use rustc_middle::middle::resolve_bound_vars::ResolvedArg;
 
         let def_id = item.owner_id.def_id;
         if let hir::ItemKind::Struct(_, hir_generics)
@@ -2119,8 +2127,8 @@ impl<'tcx> LateLintPass<'tcx> for ExplicitOutlivesRequirements {
                 let (relevant_lifetimes, bounds, predicate_span, in_where_clause) =
                     match where_predicate {
                         hir::WherePredicate::RegionPredicate(predicate) => {
-                            if let Some(Region::EarlyBound(region_def_id)) =
-                                cx.tcx.named_region(predicate.lifetime.hir_id)
+                            if let Some(ResolvedArg::EarlyBound(region_def_id)) =
+                                cx.tcx.named_bound_var(predicate.lifetime.hir_id)
                             {
                                 (
                                     Self::lifetimes_outliving_lifetime(
@@ -2175,13 +2183,31 @@ impl<'tcx> LateLintPass<'tcx> for ExplicitOutlivesRequirements {
                     dropped_predicate_count += 1;
                 }
 
-                if drop_predicate && !in_where_clause {
-                    lint_spans.push(predicate_span);
-                } else if drop_predicate && i + 1 < num_predicates {
-                    // If all the bounds on a predicate were inferable and there are
-                    // further predicates, we want to eat the trailing comma.
-                    let next_predicate_span = hir_generics.predicates[i + 1].span();
-                    where_lint_spans.push(predicate_span.to(next_predicate_span.shrink_to_lo()));
+                if drop_predicate {
+                    if !in_where_clause {
+                        lint_spans.push(predicate_span);
+                    } else if predicate_span.from_expansion() {
+                        // Don't try to extend the span if it comes from a macro expansion.
+                        where_lint_spans.push(predicate_span);
+                    } else if i + 1 < num_predicates {
+                        // If all the bounds on a predicate were inferable and there are
+                        // further predicates, we want to eat the trailing comma.
+                        let next_predicate_span = hir_generics.predicates[i + 1].span();
+                        if next_predicate_span.from_expansion() {
+                            where_lint_spans.push(predicate_span);
+                        } else {
+                            where_lint_spans
+                                .push(predicate_span.to(next_predicate_span.shrink_to_lo()));
+                        }
+                    } else {
+                        // Eat the optional trailing comma after the last predicate.
+                        let where_span = hir_generics.where_clause_span;
+                        if where_span.from_expansion() {
+                            where_lint_spans.push(predicate_span);
+                        } else {
+                            where_lint_spans.push(predicate_span.to(where_span.shrink_to_hi()));
+                        }
+                    }
                 } else {
                     where_lint_spans.extend(self.consolidate_outlives_bound_spans(
                         predicate_span.shrink_to_lo(),
@@ -2225,6 +2251,11 @@ impl<'tcx> LateLintPass<'tcx> for ExplicitOutlivesRequirements {
                     Applicability::MaybeIncorrect
                 };
 
+                // Due to macros, there might be several predicates with the same span
+                // and we only want to suggest removing them once.
+                lint_spans.sort_unstable();
+                lint_spans.dedup();
+
                 cx.emit_spanned_lint(
                     EXPLICIT_OUTLIVES_REQUIREMENTS,
                     lint_spans.clone(),
@@ -2284,11 +2315,8 @@ impl EarlyLintPass for IncompleteFeatures {
             .for_each(|(&name, &span)| {
                 let note = rustc_feature::find_feature_issue(name, GateIssue::Language)
                     .map(|n| BuiltinIncompleteFeaturesNote { n });
-                let help = if HAS_MIN_FEATURES.contains(&name) {
-                    Some(BuiltinIncompleteFeaturesHelp)
-                } else {
-                    None
-                };
+                let help =
+                    HAS_MIN_FEATURES.contains(&name).then_some(BuiltinIncompleteFeaturesHelp);
                 cx.emit_spanned_lint(
                     INCOMPLETE_FEATURES,
                     span,
@@ -2581,7 +2609,7 @@ impl<'tcx> LateLintPass<'tcx> for InvalidValue {
                     ty.tuple_fields().iter().find_map(|field| ty_find_init_error(cx, field, init))
                 }
                 Array(ty, len) => {
-                    if matches!(len.try_eval_usize(cx.tcx, cx.param_env), Some(v) if v > 0) {
+                    if matches!(len.try_eval_target_usize(cx.tcx, cx.param_env), Some(v) if v > 0) {
                         // Array length known at array non-empty -- recurse.
                         ty_find_init_error(cx, *ty, init)
                     } else {
@@ -2608,7 +2636,13 @@ impl<'tcx> LateLintPass<'tcx> for InvalidValue {
                 cx.emit_spanned_lint(
                     INVALID_VALUE,
                     expr.span,
-                    BuiltinUnpermittedTypeInit { msg, ty: conjured_ty, label: expr.span, sub },
+                    BuiltinUnpermittedTypeInit {
+                        msg,
+                        ty: conjured_ty,
+                        label: expr.span,
+                        sub,
+                        tcx: cx.tcx,
+                    },
                 );
             }
         }
@@ -2661,7 +2695,7 @@ pub struct ClashingExternDeclarations {
     /// the symbol should be reported as a clashing declaration.
     // FIXME: Technically, we could just store a &'tcx str here without issue; however, the
     // `impl_lint_pass` macro doesn't currently support lints parametric over a lifetime.
-    seen_decls: FxHashMap<Symbol, HirId>,
+    seen_decls: FxHashMap<Symbol, hir::OwnerId>,
 }
 
 /// Differentiate between whether the name for an extern decl came from the link_name attribute or
@@ -2687,19 +2721,20 @@ impl ClashingExternDeclarations {
     pub(crate) fn new() -> Self {
         ClashingExternDeclarations { seen_decls: FxHashMap::default() }
     }
+
     /// Insert a new foreign item into the seen set. If a symbol with the same name already exists
     /// for the item, return its HirId without updating the set.
-    fn insert(&mut self, tcx: TyCtxt<'_>, fi: &hir::ForeignItem<'_>) -> Option<HirId> {
+    fn insert(&mut self, tcx: TyCtxt<'_>, fi: &hir::ForeignItem<'_>) -> Option<hir::OwnerId> {
         let did = fi.owner_id.to_def_id();
         let instance = Instance::new(did, ty::List::identity_for_item(tcx, did));
         let name = Symbol::intern(tcx.symbol_name(instance).name);
-        if let Some(&hir_id) = self.seen_decls.get(&name) {
+        if let Some(&existing_id) = self.seen_decls.get(&name) {
             // Avoid updating the map with the new entry when we do find a collision. We want to
             // make sure we're always pointing to the first definition as the previous declaration.
             // This lets us avoid emitting "knock-on" diagnostics.
-            Some(hir_id)
+            Some(existing_id)
         } else {
-            self.seen_decls.insert(name, fi.hir_id())
+            self.seen_decls.insert(name, fi.owner_id)
         }
     }
 
@@ -2830,8 +2865,8 @@ impl ClashingExternDeclarations {
                                     structurally_same_type_impl(
                                         seen_types,
                                         cx,
-                                        tcx.type_of(a_did),
-                                        tcx.type_of(b_did),
+                                        tcx.type_of(a_did).subst_identity(),
+                                        tcx.type_of(b_did).subst_identity(),
                                         ckind,
                                     )
                                 },
@@ -2926,16 +2961,16 @@ impl ClashingExternDeclarations {
 impl_lint_pass!(ClashingExternDeclarations => [CLASHING_EXTERN_DECLARATIONS]);
 
 impl<'tcx> LateLintPass<'tcx> for ClashingExternDeclarations {
+    #[instrument(level = "trace", skip(self, cx))]
     fn check_foreign_item(&mut self, cx: &LateContext<'tcx>, this_fi: &hir::ForeignItem<'_>) {
-        trace!("ClashingExternDeclarations: check_foreign_item: {:?}", this_fi);
         if let ForeignItemKind::Fn(..) = this_fi.kind {
             let tcx = cx.tcx;
-            if let Some(existing_hid) = self.insert(tcx, this_fi) {
-                let existing_decl_ty = tcx.type_of(tcx.hir().local_def_id(existing_hid));
-                let this_decl_ty = tcx.type_of(this_fi.owner_id);
+            if let Some(existing_did) = self.insert(tcx, this_fi) {
+                let existing_decl_ty = tcx.type_of(existing_did).skip_binder();
+                let this_decl_ty = tcx.type_of(this_fi.owner_id).subst_identity();
                 debug!(
                     "ClashingExternDeclarations: Comparing existing {:?}: {:?} to this {:?}: {:?}",
-                    existing_hid, existing_decl_ty, this_fi.owner_id, this_decl_ty
+                    existing_did, existing_decl_ty, this_fi.owner_id, this_decl_ty
                 );
                 // Check that the declarations match.
                 if !Self::structurally_same_type(
@@ -2944,7 +2979,7 @@ impl<'tcx> LateLintPass<'tcx> for ClashingExternDeclarations {
                     this_decl_ty,
                     CItemKind::Declaration,
                 ) {
-                    let orig_fi = tcx.hir().expect_foreign_item(existing_hid.expect_owner());
+                    let orig_fi = tcx.hir().expect_foreign_item(existing_did);
                     let orig = Self::name_of_extern_decl(tcx, orig_fi);
 
                     // We want to ensure that we use spans for both decls that include where the
diff --git a/compiler/rustc_lint/src/context.rs b/compiler/rustc_lint/src/context.rs
index 8046cc21c..f5a711315 100644
--- a/compiler/rustc_lint/src/context.rs
+++ b/compiler/rustc_lint/src/context.rs
@@ -39,7 +39,7 @@ use rustc_middle::ty::{self, print::Printer, subst::GenericArg, RegisteredTools,
 use rustc_session::lint::{BuiltinLintDiagnostics, LintExpectationId};
 use rustc_session::lint::{FutureIncompatibleInfo, Level, Lint, LintBuffer, LintId};
 use rustc_session::Session;
-use rustc_span::lev_distance::find_best_match_for_name;
+use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::symbol::{sym, Ident, Symbol};
 use rustc_span::{BytePos, Span};
 use rustc_target::abi;
@@ -487,7 +487,7 @@ impl LintStore {
         let mut groups: Vec<_> = self
             .lint_groups
             .iter()
-            .filter_map(|(k, LintGroup { depr, .. })| if depr.is_none() { Some(k) } else { None })
+            .filter_map(|(k, LintGroup { depr, .. })| depr.is_none().then_some(k))
             .collect();
         groups.sort();
         let groups = groups.iter().map(|k| Symbol::intern(k));
@@ -837,9 +837,17 @@ pub trait LintContext: Sized {
                             (use_span, "'_".to_owned())
                         };
                         debug!(?deletion_span, ?use_span);
+
+                        // issue 107998 for the case such as a wrong function pointer type
+                        // `deletion_span` is empty and there is no need to report lifetime uses here
+                        let suggestions = if deletion_span.is_empty() {
+                            vec![(use_span, replace_lt)]
+                        } else {
+                            vec![(deletion_span, String::new()), (use_span, replace_lt)]
+                        };
                         db.multipart_suggestion(
                             msg,
-                            vec![(deletion_span, String::new()), (use_span, replace_lt)],
+                            suggestions,
                             Applicability::MachineApplicable,
                         );
                     }
@@ -882,6 +890,26 @@ pub trait LintContext: Sized {
                         );
                     }
                 }
+                BuiltinLintDiagnostics::ByteSliceInPackedStructWithDerive => {
+                    db.help("consider implementing the trait by hand, or remove the `packed` attribute");
+                }
+                BuiltinLintDiagnostics::UnusedExternCrate { removal_span }=> {
+                    db.span_suggestion(
+                        removal_span,
+                        "remove it",
+                        "",
+                        Applicability::MachineApplicable,
+                    );
+                }
+                BuiltinLintDiagnostics::ExternCrateNotIdiomatic { vis_span, ident_span }=> {
+                    let suggestion_span = vis_span.between(ident_span);
+                    db.span_suggestion_verbose(
+                        suggestion_span,
+                        "convert it to a `use`",
+                        if vis_span.is_empty() { "use " } else { " use " },
+                        Applicability::MachineApplicable,
+                    );
+                }
             }
             // Rewrap `db`, and pass control to the user.
             decorate(db)
@@ -1101,11 +1129,9 @@ impl<'tcx> LateContext<'tcx> {
                 .maybe_typeck_results()
                 .filter(|typeck_results| typeck_results.hir_owner == id.owner)
                 .or_else(|| {
-                    if self.tcx.has_typeck_results(id.owner.to_def_id()) {
-                        Some(self.tcx.typeck(id.owner.def_id))
-                    } else {
-                        None
-                    }
+                    self.tcx
+                        .has_typeck_results(id.owner.to_def_id())
+                        .then(|| self.tcx.typeck(id.owner.def_id))
                 })
                 .and_then(|typeck_results| typeck_results.type_dependent_def(id))
                 .map_or(Res::Err, |(kind, def_id)| Res::Def(kind, def_id)),
diff --git a/compiler/rustc_lint/src/deref_into_dyn_supertrait.rs b/compiler/rustc_lint/src/deref_into_dyn_supertrait.rs
index a45d8156c..ccf95992a 100644
--- a/compiler/rustc_lint/src/deref_into_dyn_supertrait.rs
+++ b/compiler/rustc_lint/src/deref_into_dyn_supertrait.rs
@@ -59,7 +59,7 @@ impl<'tcx> LateLintPass<'tcx> for DerefIntoDynSupertrait {
         // `Deref` is being implemented for `t`
         if let hir::ItemKind::Impl(impl_) = item.kind
             && let Some(trait_) = &impl_.of_trait
-            && let t = cx.tcx.type_of(item.owner_id)
+            && let t = cx.tcx.type_of(item.owner_id).subst_identity()
             && let opt_did @ Some(did) = trait_.trait_def_id()
             && opt_did == cx.tcx.lang_items().deref_trait()
             // `t` is `dyn t_principal`
diff --git a/compiler/rustc_lint/src/enum_intrinsics_non_enums.rs b/compiler/rustc_lint/src/enum_intrinsics_non_enums.rs
index 73bd41732..f1ba192f2 100644
--- a/compiler/rustc_lint/src/enum_intrinsics_non_enums.rs
+++ b/compiler/rustc_lint/src/enum_intrinsics_non_enums.rs
@@ -4,7 +4,7 @@ use crate::{
     LateContext, LateLintPass,
 };
 use rustc_hir as hir;
-use rustc_middle::ty::{visit::TypeVisitable, Ty};
+use rustc_middle::ty::{visit::TypeVisitableExt, Ty};
 use rustc_span::{symbol::sym, Span};
 
 declare_lint! {
diff --git a/compiler/rustc_lint/src/errors.rs b/compiler/rustc_lint/src/errors.rs
index f3ae26091..9af5284df 100644
--- a/compiler/rustc_lint/src/errors.rs
+++ b/compiler/rustc_lint/src/errors.rs
@@ -1,6 +1,6 @@
+use crate::fluent_generated as fluent;
 use rustc_errors::{
-    fluent, AddToDiagnostic, Diagnostic, ErrorGuaranteed, Handler, IntoDiagnostic,
-    SubdiagnosticMessage,
+    AddToDiagnostic, Diagnostic, ErrorGuaranteed, Handler, IntoDiagnostic, SubdiagnosticMessage,
 };
 use rustc_macros::{Diagnostic, Subdiagnostic};
 use rustc_session::lint::Level;
@@ -116,7 +116,7 @@ impl IntoDiagnostic<'_> for CheckNameUnknown {
         let mut diag = handler.struct_err(fluent::lint_check_name_unknown);
         diag.code(rustc_errors::error_code!(E0602));
         if let Some(suggestion) = self.suggestion {
-            diag.help(fluent::help);
+            diag.help(fluent::lint_help);
             diag.set_arg("suggestion", suggestion);
         }
         diag.set_arg("lint_name", self.lint_name);
diff --git a/compiler/rustc_lint/src/for_loops_over_fallibles.rs b/compiler/rustc_lint/src/for_loops_over_fallibles.rs
index 5219992ee..a3367ae4a 100644
--- a/compiler/rustc_lint/src/for_loops_over_fallibles.rs
+++ b/compiler/rustc_lint/src/for_loops_over_fallibles.rs
@@ -65,11 +65,8 @@ impl<'tcx> LateLintPass<'tcx> for ForLoopsOverFallibles {
             } else {
                 ForLoopsOverFalliblesLoopSub::UseWhileLet { start_span: expr.span.with_hi(pat.span.lo()), end_span: pat.span.between(arg.span), var }
             } ;
-        let question_mark = if suggest_question_mark(cx, adt, substs, expr.span) {
-            Some(ForLoopsOverFalliblesQuestionMark { suggestion: arg.span.shrink_to_hi() })
-        } else {
-            None
-        };
+        let question_mark = suggest_question_mark(cx, adt, substs, expr.span)
+            .then(|| ForLoopsOverFalliblesQuestionMark { suggestion: arg.span.shrink_to_hi() });
         let suggestion = ForLoopsOverFalliblesSuggestion {
             var,
             start_span: expr.span.with_hi(pat.span.lo()),
@@ -139,9 +136,10 @@ fn suggest_question_mark<'tcx>(
 
     let ty = substs.type_at(0);
     let infcx = cx.tcx.infer_ctxt().build();
+    let body_def_id = cx.tcx.hir().body_owner_def_id(body_id);
     let cause = ObligationCause::new(
         span,
-        body_id.hir_id,
+        body_def_id,
         rustc_infer::traits::ObligationCauseCode::MiscObligation,
     );
     let errors = rustc_trait_selection::traits::fully_solve_bound(
diff --git a/compiler/rustc_lint/src/internal.rs b/compiler/rustc_lint/src/internal.rs
index 6cefaea2b..2fd0ef3cd 100644
--- a/compiler/rustc_lint/src/internal.rs
+++ b/compiler/rustc_lint/src/internal.rs
@@ -216,7 +216,7 @@ fn is_ty_or_ty_ctxt(cx: &LateContext<'_>, path: &Path<'_>) -> Option<String> {
         }
         // Only lint on `&Ty` and `&TyCtxt` if it is used outside of a trait.
         Res::SelfTyAlias { alias_to: did, is_trait_impl: false, .. } => {
-            if let ty::Adt(adt, substs) = cx.tcx.type_of(did).kind() {
+            if let ty::Adt(adt, substs) = cx.tcx.type_of(did).subst_identity().kind() {
                 if let Some(name @ (sym::Ty | sym::TyCtxt)) = cx.tcx.get_diagnostic_name(adt.did())
                 {
                     // NOTE: This path is currently unreachable as `Ty<'tcx>` is
diff --git a/compiler/rustc_lint/src/late.rs b/compiler/rustc_lint/src/late.rs
index b2a265674..b42878a02 100644
--- a/compiler/rustc_lint/src/late.rs
+++ b/compiler/rustc_lint/src/late.rs
@@ -66,13 +66,12 @@ impl<'tcx, T: LateLintPass<'tcx>> LateContextAndPass<'tcx, T> {
         self.context.last_node_with_lint_attrs = prev;
     }
 
-    fn with_param_env<F>(&mut self, id: hir::HirId, f: F)
+    fn with_param_env<F>(&mut self, id: hir::OwnerId, f: F)
     where
         F: FnOnce(&mut Self),
     {
         let old_param_env = self.context.param_env;
-        self.context.param_env =
-            self.context.tcx.param_env(self.context.tcx.hir().local_def_id(id));
+        self.context.param_env = self.context.tcx.param_env(id);
         f(self);
         self.context.param_env = old_param_env;
     }
@@ -132,7 +131,7 @@ impl<'tcx, T: LateLintPass<'tcx>> hir_visit::Visitor<'tcx> for LateContextAndPas
         let old_cached_typeck_results = self.context.cached_typeck_results.take();
         let old_enclosing_body = self.context.enclosing_body.take();
         self.with_lint_attrs(it.hir_id(), |cx| {
-            cx.with_param_env(it.hir_id(), |cx| {
+            cx.with_param_env(it.owner_id, |cx| {
                 lint_callback!(cx, check_item, it);
                 hir_visit::walk_item(cx, it);
                 lint_callback!(cx, check_item_post, it);
@@ -145,7 +144,7 @@ impl<'tcx, T: LateLintPass<'tcx>> hir_visit::Visitor<'tcx> for LateContextAndPas
 
     fn visit_foreign_item(&mut self, it: &'tcx hir::ForeignItem<'tcx>) {
         self.with_lint_attrs(it.hir_id(), |cx| {
-            cx.with_param_env(it.hir_id(), |cx| {
+            cx.with_param_env(it.owner_id, |cx| {
                 lint_callback!(cx, check_foreign_item, it);
                 hir_visit::walk_foreign_item(cx, it);
             });
@@ -180,7 +179,7 @@ impl<'tcx, T: LateLintPass<'tcx>> hir_visit::Visitor<'tcx> for LateContextAndPas
         decl: &'tcx hir::FnDecl<'tcx>,
         body_id: hir::BodyId,
         span: Span,
-        id: hir::HirId,
+        id: LocalDefId,
     ) {
         // Wrap in typeck results here, not just in visit_nested_body,
         // in order for `check_fn` to be able to use them.
@@ -268,7 +267,7 @@ impl<'tcx, T: LateLintPass<'tcx>> hir_visit::Visitor<'tcx> for LateContextAndPas
         let generics = self.context.generics.take();
         self.context.generics = Some(&trait_item.generics);
         self.with_lint_attrs(trait_item.hir_id(), |cx| {
-            cx.with_param_env(trait_item.hir_id(), |cx| {
+            cx.with_param_env(trait_item.owner_id, |cx| {
                 lint_callback!(cx, check_trait_item, trait_item);
                 hir_visit::walk_trait_item(cx, trait_item);
             });
@@ -280,7 +279,7 @@ impl<'tcx, T: LateLintPass<'tcx>> hir_visit::Visitor<'tcx> for LateContextAndPas
         let generics = self.context.generics.take();
         self.context.generics = Some(&impl_item.generics);
         self.with_lint_attrs(impl_item.hir_id(), |cx| {
-            cx.with_param_env(impl_item.hir_id(), |cx| {
+            cx.with_param_env(impl_item.owner_id, |cx| {
                 lint_callback!(cx, check_impl_item, impl_item);
                 hir_visit::walk_impl_item(cx, impl_item);
                 lint_callback!(cx, check_impl_item_post, impl_item);
diff --git a/compiler/rustc_lint/src/levels.rs b/compiler/rustc_lint/src/levels.rs
index cca36913d..bc7488fab 100644
--- a/compiler/rustc_lint/src/levels.rs
+++ b/compiler/rustc_lint/src/levels.rs
@@ -1,13 +1,16 @@
-use crate::context::{CheckLintNameResult, LintStore};
-use crate::late::unerased_lint_store;
-use crate::lints::{
-    DeprecatedLintName, IgnoredUnlessCrateSpecified, OverruledAtributeLint, RenamedOrRemovedLint,
-    RenamedOrRemovedLintSuggestion, UnknownLint, UnknownLintSuggestion,
+use crate::{
+    context::{CheckLintNameResult, LintStore},
+    fluent_generated as fluent,
+    late::unerased_lint_store,
+    lints::{
+        DeprecatedLintName, IgnoredUnlessCrateSpecified, OverruledAtributeLint,
+        RenamedOrRemovedLint, RenamedOrRemovedLintSuggestion, UnknownLint, UnknownLintSuggestion,
+    },
 };
 use rustc_ast as ast;
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::fx::FxHashMap;
-use rustc_errors::{fluent, DecorateLint, DiagnosticBuilder, DiagnosticMessage, MultiSpan};
+use rustc_errors::{DecorateLint, DiagnosticBuilder, DiagnosticMessage, MultiSpan};
 use rustc_hir as hir;
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_hir::HirId;
@@ -983,7 +986,7 @@ impl<'s, P: LintLevelsProvider> LintLevelsBuilder<'s, P> {
                     fluent::lint_unknown_gated_lint,
                     |lint| {
                         lint.set_arg("name", lint_id.lint.name_lower());
-                        lint.note(fluent::note);
+                        lint.note(fluent::lint_note);
                         add_feature_diagnostics(lint, &self.sess.parse_sess, feature);
                         lint
                     },
diff --git a/compiler/rustc_lint/src/lib.rs b/compiler/rustc_lint/src/lib.rs
index d6be4da03..35dc533e5 100644
--- a/compiler/rustc_lint/src/lib.rs
+++ b/compiler/rustc_lint/src/lib.rs
@@ -63,7 +63,9 @@ mod late;
 mod let_underscore;
 mod levels;
 mod lints;
+mod map_unit_fn;
 mod methods;
+mod multiple_supertrait_upcastable;
 mod non_ascii_idents;
 mod non_fmt_panic;
 mod nonstandard_style;
@@ -79,8 +81,10 @@ mod unused;
 pub use array_into_iter::ARRAY_INTO_ITER;
 
 use rustc_ast as ast;
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
 use rustc_hir as hir;
 use rustc_hir::def_id::LocalDefId;
+use rustc_macros::fluent_messages;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::lint::builtin::{
@@ -97,7 +101,9 @@ use for_loops_over_fallibles::*;
 use hidden_unicode_codepoints::*;
 use internal::*;
 use let_underscore::*;
+use map_unit_fn::*;
 use methods::*;
+use multiple_supertrait_upcastable::*;
 use non_ascii_idents::*;
 use non_fmt_panic::NonPanicFmt;
 use nonstandard_style::*;
@@ -120,6 +126,8 @@ pub use rustc_session::lint::Level::{self, *};
 pub use rustc_session::lint::{BufferedEarlyLint, FutureIncompatibleInfo, Lint, LintId};
 pub use rustc_session::lint::{LintArray, LintPass};
 
+fluent_messages! { "../locales/en-US.ftl" }
+
 pub fn provide(providers: &mut Providers) {
     levels::provide(providers);
     expect::provide(providers);
@@ -232,6 +240,8 @@ late_lint_methods!(
             InvalidAtomicOrdering: InvalidAtomicOrdering,
             NamedAsmLabels: NamedAsmLabels,
             OpaqueHiddenInferredBound: OpaqueHiddenInferredBound,
+            MultipleSupertraitUpcastable: MultipleSupertraitUpcastable,
+            MapUnitFn: MapUnitFn,
         ]
     ]
 );
@@ -291,7 +301,8 @@ fn register_builtins(store: &mut LintStore) {
         UNUSED_LABELS,
         UNUSED_PARENS,
         UNUSED_BRACES,
-        REDUNDANT_SEMICOLONS
+        REDUNDANT_SEMICOLONS,
+        MAP_UNIT_FN
     );
 
     add_lint_group!("let_underscore", LET_UNDERSCORE_DROP, LET_UNDERSCORE_LOCK);
@@ -321,7 +332,6 @@ fn register_builtins(store: &mut LintStore) {
     store.register_renamed("exceeding_bitshifts", "arithmetic_overflow");
     store.register_renamed("redundant_semicolon", "redundant_semicolons");
     store.register_renamed("overlapping_patterns", "overlapping_range_endpoints");
-    store.register_renamed("safe_packed_borrows", "unaligned_references");
     store.register_renamed("disjoint_capture_migration", "rust_2021_incompatible_closure_captures");
     store.register_renamed("or_patterns_back_compat", "rust_2021_incompatible_or_patterns");
     store.register_renamed("non_fmt_panic", "non_fmt_panics");
@@ -484,6 +494,16 @@ fn register_builtins(store: &mut LintStore) {
         "converted into hard error, see issue #71800 \
          <https://github.com/rust-lang/rust/issues/71800> for more information",
     );
+    store.register_removed(
+        "safe_packed_borrows",
+        "converted into hard error, see issue #82523 \
+         <https://github.com/rust-lang/rust/issues/82523> for more information",
+    );
+    store.register_removed(
+        "unaligned_references",
+        "converted into hard error, see issue #82523 \
+         <https://github.com/rust-lang/rust/issues/82523> for more information",
+    );
 }
 
 fn register_internals(store: &mut LintStore) {
diff --git a/compiler/rustc_lint/src/lints.rs b/compiler/rustc_lint/src/lints.rs
index 329ece28e..20ab0af58 100644
--- a/compiler/rustc_lint/src/lints.rs
+++ b/compiler/rustc_lint/src/lints.rs
@@ -2,13 +2,16 @@
 #![allow(rustc::diagnostic_outside_of_impl)]
 use std::num::NonZeroU32;
 
+use crate::fluent_generated as fluent;
 use rustc_errors::{
-    fluent, AddToDiagnostic, Applicability, DecorateLint, DiagnosticMessage,
-    DiagnosticStyledString, SuggestionStyle,
+    AddToDiagnostic, Applicability, DecorateLint, DiagnosticMessage, DiagnosticStyledString,
+    SuggestionStyle,
 };
 use rustc_hir::def_id::DefId;
 use rustc_macros::{LintDiagnostic, Subdiagnostic};
-use rustc_middle::ty::{PolyExistentialTraitRef, Predicate, Ty, TyCtxt};
+use rustc_middle::ty::{
+    inhabitedness::InhabitedPredicate, PolyExistentialTraitRef, Predicate, Ty, TyCtxt,
+};
 use rustc_session::parse::ParseSess;
 use rustc_span::{edition::Edition, sym, symbol::Ident, Span, Symbol};
 
@@ -21,7 +24,7 @@ use crate::{
 #[diag(lint_array_into_iter)]
 pub struct ArrayIntoIterDiag<'a> {
     pub target: &'a str,
-    #[suggestion(use_iter_suggestion, code = "iter", applicability = "machine-applicable")]
+    #[suggestion(lint_use_iter_suggestion, code = "iter", applicability = "machine-applicable")]
     pub suggestion: Span,
     #[subdiagnostic]
     pub sub: Option<ArrayIntoIterDiagSub>,
@@ -29,12 +32,15 @@ pub struct ArrayIntoIterDiag<'a> {
 
 #[derive(Subdiagnostic)]
 pub enum ArrayIntoIterDiagSub {
-    #[suggestion(remove_into_iter_suggestion, code = "", applicability = "maybe-incorrect")]
+    #[suggestion(lint_remove_into_iter_suggestion, code = "", applicability = "maybe-incorrect")]
     RemoveIntoIter {
         #[primary_span]
         span: Span,
     },
-    #[multipart_suggestion(use_explicit_into_iter_suggestion, applicability = "maybe-incorrect")]
+    #[multipart_suggestion(
+        lint_use_explicit_into_iter_suggestion,
+        applicability = "maybe-incorrect"
+    )]
     UseExplicitIntoIter {
         #[suggestion_part(code = "IntoIterator::into_iter(")]
         start_span: Span,
@@ -161,13 +167,13 @@ pub struct BuiltinDeprecatedAttrLink<'a> {
 
 #[derive(Subdiagnostic)]
 pub enum BuiltinDeprecatedAttrLinkSuggestion<'a> {
-    #[suggestion(msg_suggestion, code = "", applicability = "machine-applicable")]
+    #[suggestion(lint_msg_suggestion, code = "", applicability = "machine-applicable")]
     Msg {
         #[primary_span]
         suggestion: Span,
         msg: &'a str,
     },
-    #[suggestion(default_suggestion, code = "", applicability = "machine-applicable")]
+    #[suggestion(lint_default_suggestion, code = "", applicability = "machine-applicable")]
     Default {
         #[primary_span]
         suggestion: Span,
@@ -199,9 +205,9 @@ pub struct BuiltinUnusedDocComment<'a> {
 
 #[derive(Subdiagnostic)]
 pub enum BuiltinUnusedDocCommentSub {
-    #[help(plain_help)]
+    #[help(lint_plain_help)]
     PlainHelp,
-    #[help(block_help)]
+    #[help(lint_block_help)]
     BlockHelp,
 }
 
@@ -240,7 +246,7 @@ impl<'a> DecorateLint<'a, ()> for BuiltinUngatedAsyncFnTrackCaller<'_> {
         self,
         diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>,
     ) -> &'b mut rustc_errors::DiagnosticBuilder<'a, ()> {
-        diag.span_label(self.label, fluent::label);
+        diag.span_label(self.label, fluent::lint_label);
         rustc_session::parse::add_feature_diagnostics(
             diag,
             &self.parse_sess,
@@ -335,7 +341,7 @@ impl AddToDiagnostic for BuiltinTypeAliasGenericBoundsSuggestion {
         ) -> rustc_errors::SubdiagnosticMessage,
     {
         diag.multipart_suggestion(
-            fluent::suggestion,
+            fluent::lint_suggestion,
             self.suggestions,
             Applicability::MachineApplicable,
         );
@@ -386,7 +392,7 @@ pub struct BuiltinExplicitOutlives {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(suggestion)]
+#[multipart_suggestion(lint_suggestion)]
 pub struct BuiltinExplicitOutlivesSuggestion {
     #[suggestion_part(code = "")]
     pub spans: Vec<Span>,
@@ -405,11 +411,11 @@ pub struct BuiltinIncompleteFeatures {
 }
 
 #[derive(Subdiagnostic)]
-#[help(help)]
+#[help(lint_help)]
 pub struct BuiltinIncompleteFeaturesHelp;
 
 #[derive(Subdiagnostic)]
-#[note(note)]
+#[note(lint_note)]
 pub struct BuiltinIncompleteFeaturesNote {
     pub n: NonZeroU32,
 }
@@ -419,6 +425,7 @@ pub struct BuiltinUnpermittedTypeInit<'a> {
     pub ty: Ty<'a>,
     pub label: Span,
     pub sub: BuiltinUnpermittedTypeInitSub,
+    pub tcx: TyCtxt<'a>,
 }
 
 impl<'a> DecorateLint<'a, ()> for BuiltinUnpermittedTypeInit<'_> {
@@ -428,7 +435,13 @@ impl<'a> DecorateLint<'a, ()> for BuiltinUnpermittedTypeInit<'_> {
     ) -> &'b mut rustc_errors::DiagnosticBuilder<'a, ()> {
         diag.set_arg("ty", self.ty);
         diag.span_label(self.label, fluent::lint_builtin_unpermitted_type_init_label);
-        diag.span_label(self.label, fluent::lint_builtin_unpermitted_type_init_label_suggestion);
+        if let InhabitedPredicate::True = self.ty.inhabited_predicate(self.tcx) {
+            // Only suggest late `MaybeUninit::assume_init` initialization if the type is inhabited.
+            diag.span_label(
+                self.label,
+                fluent::lint_builtin_unpermitted_type_init_label_suggestion,
+            );
+        }
         self.sub.add_to_diagnostic(diag);
         diag
     }
@@ -473,9 +486,9 @@ pub enum BuiltinClashingExtern<'a> {
     SameName {
         this: Symbol,
         orig: Symbol,
-        #[label(previous_decl_label)]
+        #[label(lint_previous_decl_label)]
         previous_decl_label: Span,
-        #[label(mismatch_label)]
+        #[label(lint_mismatch_label)]
         mismatch_label: Span,
         #[subdiagnostic]
         sub: BuiltinClashingExternSub<'a>,
@@ -484,9 +497,9 @@ pub enum BuiltinClashingExtern<'a> {
     DiffName {
         this: Symbol,
         orig: Symbol,
-        #[label(previous_decl_label)]
+        #[label(lint_previous_decl_label)]
         previous_decl_label: Span,
-        #[label(mismatch_label)]
+        #[label(lint_mismatch_label)]
         mismatch_label: Span,
         #[subdiagnostic]
         sub: BuiltinClashingExternSub<'a>,
@@ -562,7 +575,7 @@ pub struct SupertraitAsDerefTarget<'a> {
 }
 
 #[derive(Subdiagnostic)]
-#[label(label)]
+#[label(lint_label)]
 pub struct SupertraitAsDerefTargetLabel {
     #[primary_span]
     pub label: Span,
@@ -595,7 +608,7 @@ pub struct Expectation {
 }
 
 #[derive(Subdiagnostic)]
-#[note(rationale)]
+#[note(lint_rationale)]
 pub struct ExpectationNote {
     pub rationale: Symbol,
 }
@@ -616,13 +629,13 @@ pub struct ForLoopsOverFalliblesDiag<'a> {
 
 #[derive(Subdiagnostic)]
 pub enum ForLoopsOverFalliblesLoopSub<'a> {
-    #[suggestion(remove_next, code = ".by_ref()", applicability = "maybe-incorrect")]
+    #[suggestion(lint_remove_next, code = ".by_ref()", applicability = "maybe-incorrect")]
     RemoveNext {
         #[primary_span]
         suggestion: Span,
         recv_snip: String,
     },
-    #[multipart_suggestion(use_while_let, applicability = "maybe-incorrect")]
+    #[multipart_suggestion(lint_use_while_let, applicability = "maybe-incorrect")]
     UseWhileLet {
         #[suggestion_part(code = "while let {var}(")]
         start_span: Span,
@@ -633,14 +646,14 @@ pub enum ForLoopsOverFalliblesLoopSub<'a> {
 }
 
 #[derive(Subdiagnostic)]
-#[suggestion(use_question_mark, code = "?", applicability = "maybe-incorrect")]
+#[suggestion(lint_use_question_mark, code = "?", applicability = "maybe-incorrect")]
 pub struct ForLoopsOverFalliblesQuestionMark {
     #[primary_span]
     pub suggestion: Span,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(suggestion, applicability = "maybe-incorrect")]
+#[multipart_suggestion(lint_suggestion, applicability = "maybe-incorrect")]
 pub struct ForLoopsOverFalliblesSuggestion<'a> {
     pub var: &'a str,
     #[suggestion_part(code = "if let {var}(")]
@@ -699,13 +712,13 @@ impl AddToDiagnostic for HiddenUnicodeCodepointsDiagSub {
         match self {
             HiddenUnicodeCodepointsDiagSub::Escape { spans } => {
                 diag.multipart_suggestion_with_style(
-                    fluent::suggestion_remove,
+                    fluent::lint_suggestion_remove,
                     spans.iter().map(|(_, span)| (*span, "".to_string())).collect(),
                     Applicability::MachineApplicable,
                     SuggestionStyle::HideCodeAlways,
                 );
                 diag.multipart_suggestion(
-                    fluent::suggestion_escape,
+                    fluent::lint_suggestion_escape,
                     spans
                         .into_iter()
                         .map(|(c, span)| {
@@ -728,13 +741,29 @@ impl AddToDiagnostic for HiddenUnicodeCodepointsDiagSub {
                         .collect::<Vec<String>>()
                         .join(", "),
                 );
-                diag.note(fluent::suggestion_remove);
-                diag.note(fluent::no_suggestion_note_escape);
+                diag.note(fluent::lint_suggestion_remove);
+                diag.note(fluent::lint_no_suggestion_note_escape);
             }
         }
     }
 }
 
+// map_unit_fn.rs
+#[derive(LintDiagnostic)]
+#[diag(lint_map_unit_fn)]
+#[note]
+pub struct MappingToUnit {
+    #[label(lint_function_label)]
+    pub function_label: Span,
+    #[label(lint_argument_label)]
+    pub argument_label: Span,
+    #[label(lint_map_label)]
+    pub map_label: Span,
+    #[suggestion(style = "verbose", code = "{replace}", applicability = "maybe-incorrect")]
+    pub suggestion: Span,
+    pub replace: String,
+}
+
 // internal.rs
 #[derive(LintDiagnostic)]
 #[diag(lint_default_hash_types)]
@@ -874,7 +903,7 @@ pub struct RenamedOrRemovedLint<'a> {
 }
 
 #[derive(Subdiagnostic)]
-#[suggestion(suggestion, code = "{replace}", applicability = "machine-applicable")]
+#[suggestion(lint_suggestion, code = "{replace}", applicability = "machine-applicable")]
 pub struct RenamedOrRemovedLintSuggestion<'a> {
     #[primary_span]
     pub suggestion: Span,
@@ -890,7 +919,7 @@ pub struct UnknownLint {
 }
 
 #[derive(Subdiagnostic)]
-#[suggestion(suggestion, code = "{replace}", applicability = "maybe-incorrect")]
+#[suggestion(lint_suggestion, code = "{replace}", applicability = "maybe-incorrect")]
 pub struct UnknownLintSuggestion {
     #[primary_span]
     pub suggestion: Span,
@@ -910,12 +939,19 @@ pub struct IgnoredUnlessCrateSpecified<'a> {
 #[note]
 #[help]
 pub struct CStringPtr {
-    #[label(as_ptr_label)]
+    #[label(lint_as_ptr_label)]
     pub as_ptr: Span,
-    #[label(unwrap_label)]
+    #[label(lint_unwrap_label)]
     pub unwrap: Span,
 }
 
+// multiple_supertrait_upcastable.rs
+#[derive(LintDiagnostic)]
+#[diag(lint_multple_supertrait_upcastable)]
+pub struct MultipleSupertraitUpcastable {
+    pub ident: Ident,
+}
+
 // non_ascii_idents.rs
 #[derive(LintDiagnostic)]
 #[diag(lint_identifier_non_ascii_char)]
@@ -936,7 +972,7 @@ pub struct ConfusableIdentifierPair {
 
 #[derive(LintDiagnostic)]
 #[diag(lint_mixed_script_confusables)]
-#[note(includes_note)]
+#[note(lint_includes_note)]
 #[note]
 pub struct MixedScriptConfusables {
     pub set: String,
@@ -956,17 +992,17 @@ impl<'a> DecorateLint<'a, ()> for NonFmtPanicUnused {
         diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>,
     ) -> &'b mut rustc_errors::DiagnosticBuilder<'a, ()> {
         diag.set_arg("count", self.count);
-        diag.note(fluent::note);
+        diag.note(fluent::lint_note);
         if let Some(span) = self.suggestion {
             diag.span_suggestion(
                 span.shrink_to_hi(),
-                fluent::add_args_suggestion,
+                fluent::lint_add_args_suggestion,
                 ", ...",
                 Applicability::HasPlaceholders,
             );
             diag.span_suggestion(
                 span.shrink_to_lo(),
-                fluent::add_fmt_suggestion,
+                fluent::lint_add_fmt_suggestion,
                 "\"{}\", ",
                 Applicability::MachineApplicable,
             );
@@ -1000,12 +1036,12 @@ pub struct NonCamelCaseType<'a> {
 
 #[derive(Subdiagnostic)]
 pub enum NonCamelCaseTypeSub {
-    #[label(label)]
+    #[label(lint_label)]
     Label {
         #[primary_span]
         span: Span,
     },
-    #[suggestion(suggestion, code = "{replace}", applicability = "maybe-incorrect")]
+    #[suggestion(lint_suggestion, code = "{replace}", applicability = "maybe-incorrect")]
     Suggestion {
         #[primary_span]
         span: Span,
@@ -1041,15 +1077,15 @@ impl AddToDiagnostic for NonSnakeCaseDiagSub {
     {
         match self {
             NonSnakeCaseDiagSub::Label { span } => {
-                diag.span_label(span, fluent::label);
+                diag.span_label(span, fluent::lint_label);
             }
             NonSnakeCaseDiagSub::Help => {
-                diag.help(fluent::help);
+                diag.help(fluent::lint_help);
             }
             NonSnakeCaseDiagSub::ConvertSuggestion { span, suggestion } => {
                 diag.span_suggestion(
                     span,
-                    fluent::convert_suggestion,
+                    fluent::lint_convert_suggestion,
                     suggestion,
                     Applicability::MaybeIncorrect,
                 );
@@ -1057,16 +1093,16 @@ impl AddToDiagnostic for NonSnakeCaseDiagSub {
             NonSnakeCaseDiagSub::RenameOrConvertSuggestion { span, suggestion } => {
                 diag.span_suggestion(
                     span,
-                    fluent::rename_or_convert_suggestion,
+                    fluent::lint_rename_or_convert_suggestion,
                     suggestion,
                     Applicability::MaybeIncorrect,
                 );
             }
             NonSnakeCaseDiagSub::SuggestionAndNote { span } => {
-                diag.note(fluent::cannot_convert_note);
+                diag.note(fluent::lint_cannot_convert_note);
                 diag.span_suggestion(
                     span,
-                    fluent::rename_suggestion,
+                    fluent::lint_rename_suggestion,
                     "",
                     Applicability::MaybeIncorrect,
                 );
@@ -1086,12 +1122,12 @@ pub struct NonUpperCaseGlobal<'a> {
 
 #[derive(Subdiagnostic)]
 pub enum NonUpperCaseGlobalSub {
-    #[label(label)]
+    #[label(lint_label)]
     Label {
         #[primary_span]
         span: Span,
     },
-    #[suggestion(suggestion, code = "{replace}", applicability = "maybe-incorrect")]
+    #[suggestion(lint_suggestion, code = "{replace}", applicability = "maybe-incorrect")]
     Suggestion {
         #[primary_span]
         span: Span,
@@ -1209,11 +1245,11 @@ impl AddToDiagnostic for OverflowingBinHexSign {
     {
         match self {
             OverflowingBinHexSign::Positive => {
-                diag.note(fluent::positive_note);
+                diag.note(fluent::lint_positive_note);
             }
             OverflowingBinHexSign::Negative => {
-                diag.note(fluent::negative_note);
-                diag.note(fluent::negative_becomes_note);
+                diag.note(fluent::lint_negative_note);
+                diag.note(fluent::lint_negative_becomes_note);
             }
         }
     }
@@ -1222,7 +1258,7 @@ impl AddToDiagnostic for OverflowingBinHexSign {
 #[derive(Subdiagnostic)]
 pub enum OverflowingBinHexSub<'a> {
     #[suggestion(
-        suggestion,
+        lint_suggestion,
         code = "{sans_suffix}{suggestion_ty}",
         applicability = "machine-applicable"
     )]
@@ -1232,7 +1268,7 @@ pub enum OverflowingBinHexSub<'a> {
         suggestion_ty: &'a str,
         sans_suffix: &'a str,
     },
-    #[help(help)]
+    #[help(lint_help)]
     Help { suggestion_ty: &'a str },
 }
 
@@ -1249,7 +1285,7 @@ pub struct OverflowingInt<'a> {
 }
 
 #[derive(Subdiagnostic)]
-#[help(help)]
+#[help(lint_help)]
 pub struct OverflowingIntHelp<'a> {
     pub suggestion_ty: &'a str,
 }
@@ -1301,13 +1337,13 @@ impl<'a> DecorateLint<'a, ()> for ImproperCTypes<'_> {
     ) -> &'b mut rustc_errors::DiagnosticBuilder<'a, ()> {
         diag.set_arg("ty", self.ty);
         diag.set_arg("desc", self.desc);
-        diag.span_label(self.label, fluent::label);
+        diag.span_label(self.label, fluent::lint_label);
         if let Some(help) = self.help {
             diag.help(help);
         }
         diag.note(self.note);
         if let Some(note) = self.span_note {
-            diag.span_note(note, fluent::note);
+            diag.span_note(note, fluent::lint_note);
         }
         diag
     }
@@ -1394,6 +1430,21 @@ pub struct UnusedDef<'a, 'b> {
     pub cx: &'a LateContext<'b>,
     pub def_id: DefId,
     pub note: Option<Symbol>,
+    pub suggestion: Option<UnusedDefSuggestion>,
+}
+
+#[derive(Subdiagnostic)]
+pub enum UnusedDefSuggestion {
+    #[suggestion(
+        lint_suggestion,
+        style = "verbose",
+        code = "let _ = ",
+        applicability = "machine-applicable"
+    )]
+    Default {
+        #[primary_span]
+        span: Span,
+    },
 }
 
 // Needed because of def_path_str
@@ -1409,6 +1460,9 @@ impl<'a> DecorateLint<'a, ()> for UnusedDef<'_, '_> {
         if let Some(note) = self.note {
             diag.note(note.as_str());
         }
+        if let Some(sugg) = self.suggestion {
+            diag.subdiagnostic(sugg);
+        }
         diag
     }
 
@@ -1426,13 +1480,13 @@ pub struct PathStatementDrop {
 
 #[derive(Subdiagnostic)]
 pub enum PathStatementDropSub {
-    #[suggestion(suggestion, code = "drop({snippet});", applicability = "machine-applicable")]
+    #[suggestion(lint_suggestion, code = "drop({snippet});", applicability = "machine-applicable")]
     Suggestion {
         #[primary_span]
         span: Span,
         snippet: String,
     },
-    #[help(help)]
+    #[help(lint_help)]
     Help {
         #[primary_span]
         span: Span,
@@ -1453,7 +1507,7 @@ pub struct UnusedDelim<'a> {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(lint_suggestion, applicability = "machine-applicable")]
 pub struct UnusedDelimSuggestion {
     #[suggestion_part(code = "{start_replace}")]
     pub start_span: Span,
diff --git a/compiler/rustc_lint/src/map_unit_fn.rs b/compiler/rustc_lint/src/map_unit_fn.rs
new file mode 100644
index 000000000..62e8b4fe9
--- /dev/null
+++ b/compiler/rustc_lint/src/map_unit_fn.rs
@@ -0,0 +1,120 @@
+use crate::lints::MappingToUnit;
+use crate::{LateContext, LateLintPass, LintContext};
+
+use rustc_hir::{Expr, ExprKind, HirId, Stmt, StmtKind};
+use rustc_middle::{
+    query::Key,
+    ty::{self, Ty},
+};
+
+declare_lint! {
+    /// The `map_unit_fn` lint checks for `Iterator::map` receive
+    /// a callable that returns `()`.
+    ///
+    /// ### Example
+    ///
+    /// ```rust
+    /// fn foo(items: &mut Vec<u8>) {
+    ///     items.sort();
+    /// }
+    ///
+    /// fn main() {
+    ///     let mut x: Vec<Vec<u8>> = vec![
+    ///         vec![0, 2, 1],
+    ///         vec![5, 4, 3],
+    ///     ];
+    ///     x.iter_mut().map(foo);
+    /// }
+    /// ```
+    ///
+    /// {{produces}}
+    ///
+    /// ### Explanation
+    ///
+    /// Mapping to `()` is almost always a mistake.
+    pub MAP_UNIT_FN,
+    Warn,
+    "`Iterator::map` call that discard the iterator's values"
+}
+
+declare_lint_pass!(MapUnitFn => [MAP_UNIT_FN]);
+
+impl<'tcx> LateLintPass<'tcx> for MapUnitFn {
+    fn check_stmt(&mut self, cx: &LateContext<'tcx>, stmt: &Stmt<'_>) {
+        if stmt.span.from_expansion() {
+            return;
+        }
+
+        if let StmtKind::Semi(expr) = stmt.kind {
+            if let ExprKind::MethodCall(path, receiver, args, span) = expr.kind {
+                if path.ident.name.as_str() == "map" {
+                    if receiver.span.from_expansion()
+                        || args.iter().any(|e| e.span.from_expansion())
+                        || !is_impl_slice(cx, receiver)
+                        || !is_diagnostic_name(cx, expr.hir_id, "IteratorMap")
+                    {
+                        return;
+                    }
+                    let arg_ty = cx.typeck_results().expr_ty(&args[0]);
+                    if let ty::FnDef(id, _) = arg_ty.kind() {
+                        let fn_ty = cx.tcx.fn_sig(id).skip_binder();
+                        let ret_ty = fn_ty.output().skip_binder();
+                        if is_unit_type(ret_ty) {
+                            cx.emit_spanned_lint(
+                                MAP_UNIT_FN,
+                                span,
+                                MappingToUnit {
+                                    function_label: cx.tcx.span_of_impl(*id).unwrap(),
+                                    argument_label: args[0].span,
+                                    map_label: arg_ty.default_span(cx.tcx),
+                                    suggestion: path.ident.span,
+                                    replace: "for_each".to_string(),
+                                },
+                            )
+                        }
+                    } else if let ty::Closure(id, subs) = arg_ty.kind() {
+                        let cl_ty = subs.as_closure().sig();
+                        let ret_ty = cl_ty.output().skip_binder();
+                        if is_unit_type(ret_ty) {
+                            cx.emit_spanned_lint(
+                                MAP_UNIT_FN,
+                                span,
+                                MappingToUnit {
+                                    function_label: cx.tcx.span_of_impl(*id).unwrap(),
+                                    argument_label: args[0].span,
+                                    map_label: arg_ty.default_span(cx.tcx),
+                                    suggestion: path.ident.span,
+                                    replace: "for_each".to_string(),
+                                },
+                            )
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+fn is_impl_slice(cx: &LateContext<'_>, expr: &Expr<'_>) -> bool {
+    if let Some(method_id) = cx.typeck_results().type_dependent_def_id(expr.hir_id) {
+        if let Some(impl_id) = cx.tcx.impl_of_method(method_id) {
+            return cx.tcx.type_of(impl_id).skip_binder().is_slice();
+        }
+    }
+    false
+}
+
+fn is_unit_type(ty: Ty<'_>) -> bool {
+    ty.is_unit() || ty.is_never()
+}
+
+fn is_diagnostic_name(cx: &LateContext<'_>, id: HirId, name: &str) -> bool {
+    if let Some(def_id) = cx.typeck_results().type_dependent_def_id(id) {
+        if let Some(item) = cx.tcx.get_diagnostic_name(def_id) {
+            if item.as_str() == name {
+                return true;
+            }
+        }
+    }
+    false
+}
diff --git a/compiler/rustc_lint/src/multiple_supertrait_upcastable.rs b/compiler/rustc_lint/src/multiple_supertrait_upcastable.rs
new file mode 100644
index 000000000..c2ed0e19f
--- /dev/null
+++ b/compiler/rustc_lint/src/multiple_supertrait_upcastable.rs
@@ -0,0 +1,60 @@
+use crate::{LateContext, LateLintPass, LintContext};
+
+use rustc_hir as hir;
+use rustc_span::sym;
+
+declare_lint! {
+    /// The `multiple_supertrait_upcastable` lint detects when an object-safe trait has multiple
+    /// supertraits.
+    ///
+    /// ### Example
+    ///
+    /// ```rust
+    /// trait A {}
+    /// trait B {}
+    ///
+    /// #[warn(multiple_supertrait_upcastable)]
+    /// trait C: A + B {}
+    /// ```
+    ///
+    /// {{produces}}
+    ///
+    /// ### Explanation
+    ///
+    /// To support upcasting with multiple supertraits, we need to store multiple vtables and this
+    /// can result in extra space overhead, even if no code actually uses upcasting.
+    /// This lint allows users to identify when such scenarios occur and to decide whether the
+    /// additional overhead is justified.
+    pub MULTIPLE_SUPERTRAIT_UPCASTABLE,
+    Allow,
+    "detect when an object-safe trait has multiple supertraits",
+    @feature_gate = sym::multiple_supertrait_upcastable;
+}
+
+declare_lint_pass!(MultipleSupertraitUpcastable => [MULTIPLE_SUPERTRAIT_UPCASTABLE]);
+
+impl<'tcx> LateLintPass<'tcx> for MultipleSupertraitUpcastable {
+    fn check_item(&mut self, cx: &LateContext<'tcx>, item: &'tcx hir::Item<'tcx>) {
+        let def_id = item.owner_id.to_def_id();
+        // NOTE(nbdd0121): use `object_safety_violations` instead of `check_is_object_safe` because
+        // the latter will report `where_clause_object_safety` lint.
+        if let hir::ItemKind::Trait(_, _, _, _, _) = item.kind
+            && cx.tcx.object_safety_violations(def_id).is_empty()
+        {
+            let direct_super_traits_iter = cx.tcx
+                    .super_predicates_of(def_id)
+                    .predicates
+                    .into_iter()
+                    .filter_map(|(pred, _)| pred.to_opt_poly_trait_pred());
+            if direct_super_traits_iter.count() > 1 {
+                cx.emit_spanned_lint(
+                    MULTIPLE_SUPERTRAIT_UPCASTABLE,
+                    cx.tcx.def_span(def_id),
+                    crate::lints::MultipleSupertraitUpcastable {
+                        ident: item.ident
+                    },
+                );
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_lint/src/non_fmt_panic.rs b/compiler/rustc_lint/src/non_fmt_panic.rs
index 4a02c6cce..5bb1abfd2 100644
--- a/compiler/rustc_lint/src/non_fmt_panic.rs
+++ b/compiler/rustc_lint/src/non_fmt_panic.rs
@@ -1,7 +1,7 @@
 use crate::lints::{NonFmtPanicBraces, NonFmtPanicUnused};
-use crate::{LateContext, LateLintPass, LintContext};
+use crate::{fluent_generated as fluent, LateContext, LateLintPass, LintContext};
 use rustc_ast as ast;
-use rustc_errors::{fluent, Applicability};
+use rustc_errors::Applicability;
 use rustc_hir as hir;
 use rustc_infer::infer::TyCtxtInferExt;
 use rustc_middle::lint::in_external_macro;
@@ -122,18 +122,18 @@ fn check_panic<'tcx>(cx: &LateContext<'tcx>, f: &'tcx hir::Expr<'tcx>, arg: &'tc
     #[allow(rustc::diagnostic_outside_of_impl)]
     cx.struct_span_lint(NON_FMT_PANICS, arg_span, fluent::lint_non_fmt_panic, |lint| {
         lint.set_arg("name", symbol);
-        lint.note(fluent::note);
-        lint.note(fluent::more_info_note);
+        lint.note(fluent::lint_note);
+        lint.note(fluent::lint_more_info_note);
         if !is_arg_inside_call(arg_span, span) {
             // No clue where this argument is coming from.
             return lint;
         }
         if arg_macro.map_or(false, |id| cx.tcx.is_diagnostic_item(sym::format_macro, id)) {
             // A case of `panic!(format!(..))`.
-            lint.note(fluent::supports_fmt_note);
+            lint.note(fluent::lint_supports_fmt_note);
             if let Some((open, close, _)) = find_delimiters(cx, arg_span) {
                 lint.multipart_suggestion(
-                    fluent::supports_fmt_suggestion,
+                    fluent::lint_supports_fmt_suggestion,
                     vec![
                         (arg_span.until(open.shrink_to_hi()), "".into()),
                         (close.until(arg_span.shrink_to_hi()), "".into()),
@@ -146,7 +146,7 @@ fn check_panic<'tcx>(cx: &LateContext<'tcx>, f: &'tcx hir::Expr<'tcx>, arg: &'tc
             // If this is a &str or String, we can confidently give the `"{}", ` suggestion.
             let is_str = matches!(
                 ty.kind(),
-                ty::Ref(_, r, _) if *r.kind() == ty::Str,
+                ty::Ref(_, r, _) if r.is_str(),
             ) || matches!(
                 ty.ty_adt_def(),
                 Some(ty_def) if Some(ty_def.did()) == cx.tcx.lang_items().string(),
@@ -179,7 +179,7 @@ fn check_panic<'tcx>(cx: &LateContext<'tcx>, f: &'tcx hir::Expr<'tcx>, arg: &'tc
             if suggest_display {
                 lint.span_suggestion_verbose(
                     arg_span.shrink_to_lo(),
-                    fluent::display_suggestion,
+                    fluent::lint_display_suggestion,
                     "\"{}\", ",
                     fmt_applicability,
                 );
@@ -187,7 +187,7 @@ fn check_panic<'tcx>(cx: &LateContext<'tcx>, f: &'tcx hir::Expr<'tcx>, arg: &'tc
                 lint.set_arg("ty", ty);
                 lint.span_suggestion_verbose(
                     arg_span.shrink_to_lo(),
-                    fluent::debug_suggestion,
+                    fluent::lint_debug_suggestion,
                     "\"{:?}\", ",
                     fmt_applicability,
                 );
@@ -197,7 +197,7 @@ fn check_panic<'tcx>(cx: &LateContext<'tcx>, f: &'tcx hir::Expr<'tcx>, arg: &'tc
                 if let Some((open, close, del)) = find_delimiters(cx, span) {
                     lint.set_arg("already_suggested", suggest_display || suggest_debug);
                     lint.multipart_suggestion(
-                        fluent::panic_suggestion,
+                        fluent::lint_panic_suggestion,
                         if del == '(' {
                             vec![(span.until(open), "std::panic::panic_any".into())]
                         } else {
diff --git a/compiler/rustc_lint/src/nonstandard_style.rs b/compiler/rustc_lint/src/nonstandard_style.rs
index 74d234fab..71e2e66bd 100644
--- a/compiler/rustc_lint/src/nonstandard_style.rs
+++ b/compiler/rustc_lint/src/nonstandard_style.rs
@@ -10,8 +10,9 @@ use rustc_hir::def::{DefKind, Res};
 use rustc_hir::intravisit::FnKind;
 use rustc_hir::{GenericParamKind, PatKind};
 use rustc_middle::ty;
-use rustc_span::symbol::sym;
-use rustc_span::{symbol::Ident, BytePos, Span};
+use rustc_span::def_id::LocalDefId;
+use rustc_span::symbol::{sym, Ident};
+use rustc_span::{BytePos, Span};
 use rustc_target::spec::abi::Abi;
 
 #[derive(PartialEq)]
@@ -21,9 +22,8 @@ pub enum MethodLateContext {
     PlainImpl,
 }
 
-pub fn method_context(cx: &LateContext<'_>, id: hir::HirId) -> MethodLateContext {
-    let def_id = cx.tcx.hir().local_def_id(id);
-    let item = cx.tcx.associated_item(def_id);
+pub fn method_context(cx: &LateContext<'_>, id: LocalDefId) -> MethodLateContext {
+    let item = cx.tcx.associated_item(id);
     match item.container {
         ty::TraitContainer => MethodLateContext::TraitAutoImpl,
         ty::ImplContainer => match cx.tcx.impl_trait_ref(item.container_id(cx.tcx)) {
@@ -379,13 +379,13 @@ impl<'tcx> LateLintPass<'tcx> for NonSnakeCase {
         _: &hir::FnDecl<'_>,
         _: &hir::Body<'_>,
         _: Span,
-        id: hir::HirId,
+        id: LocalDefId,
     ) {
-        let attrs = cx.tcx.hir().attrs(id);
         match &fk {
             FnKind::Method(ident, sig, ..) => match method_context(cx, id) {
                 MethodLateContext::PlainImpl => {
-                    if sig.header.abi != Abi::Rust && cx.sess().contains_name(attrs, sym::no_mangle)
+                    if sig.header.abi != Abi::Rust
+                        && cx.tcx.has_attr(id.to_def_id(), sym::no_mangle)
                     {
                         return;
                     }
@@ -398,7 +398,7 @@ impl<'tcx> LateLintPass<'tcx> for NonSnakeCase {
             },
             FnKind::ItemFn(ident, _, header) => {
                 // Skip foreign-ABI #[no_mangle] functions (Issue #31924)
-                if header.abi != Abi::Rust && cx.sess().contains_name(attrs, sym::no_mangle) {
+                if header.abi != Abi::Rust && cx.tcx.has_attr(id.to_def_id(), sym::no_mangle) {
                     return;
                 }
                 self.check_snake_case(cx, "function", ident);
diff --git a/compiler/rustc_lint/src/opaque_hidden_inferred_bound.rs b/compiler/rustc_lint/src/opaque_hidden_inferred_bound.rs
index 42442cfb1..883a56cb3 100644
--- a/compiler/rustc_lint/src/opaque_hidden_inferred_bound.rs
+++ b/compiler/rustc_lint/src/opaque_hidden_inferred_bound.rs
@@ -149,7 +149,7 @@ impl<'tcx> LateLintPass<'tcx> for OpaqueHiddenInferredBound {
 struct OpaqueHiddenInferredBoundLint<'tcx> {
     ty: Ty<'tcx>,
     proj_ty: Ty<'tcx>,
-    #[label(specifically)]
+    #[label(lint_specifically)]
     assoc_pred_span: Span,
     #[subdiagnostic]
     add_bound: Option<AddBound<'tcx>>,
diff --git a/compiler/rustc_lint/src/pass_by_value.rs b/compiler/rustc_lint/src/pass_by_value.rs
index 392e13f2f..2bb2a3aab 100644
--- a/compiler/rustc_lint/src/pass_by_value.rs
+++ b/compiler/rustc_lint/src/pass_by_value.rs
@@ -50,7 +50,7 @@ fn path_for_pass_by_value(cx: &LateContext<'_>, ty: &hir::Ty<'_>) -> Option<Stri
                 return Some(format!("{}{}", name, gen_args(cx, path_segment)));
             }
             Res::SelfTyAlias { alias_to: did, is_trait_impl: false, .. } => {
-                if let ty::Adt(adt, substs) = cx.tcx.type_of(did).kind() {
+                if let ty::Adt(adt, substs) = cx.tcx.type_of(did).subst_identity().kind() {
                     if cx.tcx.has_attr(adt.did(), sym::rustc_pass_by_value) {
                         return Some(cx.tcx.def_path_str_with_substs(adt.did(), substs));
                     }
diff --git a/compiler/rustc_lint/src/passes.rs b/compiler/rustc_lint/src/passes.rs
index 0bf01c4e5..16964565b 100644
--- a/compiler/rustc_lint/src/passes.rs
+++ b/compiler/rustc_lint/src/passes.rs
@@ -4,6 +4,7 @@ use rustc_ast as ast;
 use rustc_hir as hir;
 use rustc_session::lint::builtin::HardwiredLints;
 use rustc_session::lint::LintPass;
+use rustc_span::def_id::LocalDefId;
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
 
@@ -36,7 +37,7 @@ macro_rules! late_lint_methods {
                 b: &'tcx hir::FnDecl<'tcx>,
                 c: &'tcx hir::Body<'tcx>,
                 d: Span,
-                e: hir::HirId);
+                e: LocalDefId);
             fn check_trait_item(a: &'tcx hir::TraitItem<'tcx>);
             fn check_impl_item(a: &'tcx hir::ImplItem<'tcx>);
             fn check_impl_item_post(a: &'tcx hir::ImplItem<'tcx>);
diff --git a/compiler/rustc_lint/src/types.rs b/compiler/rustc_lint/src/types.rs
index be47a3e23..85958c417 100644
--- a/compiler/rustc_lint/src/types.rs
+++ b/compiler/rustc_lint/src/types.rs
@@ -1,19 +1,25 @@
-use crate::lints::{
-    AtomicOrderingFence, AtomicOrderingLoad, AtomicOrderingStore, ImproperCTypes,
-    InvalidAtomicOrderingDiag, OnlyCastu8ToChar, OverflowingBinHex, OverflowingBinHexSign,
-    OverflowingBinHexSub, OverflowingInt, OverflowingIntHelp, OverflowingLiteral, OverflowingUInt,
-    RangeEndpointOutOfRange, UnusedComparisons, VariantSizeDifferencesDiag,
+use crate::{
+    fluent_generated as fluent,
+    lints::{
+        AtomicOrderingFence, AtomicOrderingLoad, AtomicOrderingStore, ImproperCTypes,
+        InvalidAtomicOrderingDiag, OnlyCastu8ToChar, OverflowingBinHex, OverflowingBinHexSign,
+        OverflowingBinHexSub, OverflowingInt, OverflowingIntHelp, OverflowingLiteral,
+        OverflowingUInt, RangeEndpointOutOfRange, UnusedComparisons, VariantSizeDifferencesDiag,
+    },
 };
 use crate::{LateContext, LateLintPass, LintContext};
 use rustc_ast as ast;
 use rustc_attr as attr;
 use rustc_data_structures::fx::FxHashSet;
-use rustc_errors::{fluent, DiagnosticMessage};
+use rustc_errors::DiagnosticMessage;
 use rustc_hir as hir;
 use rustc_hir::{is_range_literal, Expr, ExprKind, Node};
 use rustc_middle::ty::layout::{IntegerExt, LayoutOf, SizeSkeleton};
 use rustc_middle::ty::subst::SubstsRef;
-use rustc_middle::ty::{self, AdtKind, DefIdTree, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable};
+use rustc_middle::ty::{
+    self, AdtKind, DefIdTree, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitableExt,
+};
+use rustc_span::def_id::LocalDefId;
 use rustc_span::source_map;
 use rustc_span::symbol::sym;
 use rustc_span::{Span, Symbol};
@@ -650,7 +656,7 @@ pub fn transparent_newtype_field<'a, 'tcx>(
 ) -> Option<&'a ty::FieldDef> {
     let param_env = tcx.param_env(variant.def_id);
     variant.fields.iter().find(|field| {
-        let field_ty = tcx.type_of(field.did);
+        let field_ty = tcx.type_of(field.did).subst_identity();
         let is_zst = tcx.layout_of(param_env.and(field_ty)).map_or(false, |layout| layout.is_zst());
         !is_zst
     })
@@ -1107,6 +1113,7 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
             | ty::Closure(..)
             | ty::Generator(..)
             | ty::GeneratorWitness(..)
+            | ty::GeneratorWitnessMIR(..)
             | ty::Placeholder(..)
             | ty::FnDef(..) => bug!("unexpected type in foreign function: {:?}", ty),
         }
@@ -1142,7 +1149,7 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
 
     fn check_for_opaque_ty(&mut self, sp: Span, ty: Ty<'tcx>) -> bool {
         struct ProhibitOpaqueTypes;
-        impl<'tcx> ty::visit::TypeVisitor<'tcx> for ProhibitOpaqueTypes {
+        impl<'tcx> ty::visit::TypeVisitor<TyCtxt<'tcx>> for ProhibitOpaqueTypes {
             type BreakTy = Ty<'tcx>;
 
             fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
@@ -1223,9 +1230,8 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
         }
     }
 
-    fn check_foreign_fn(&mut self, id: hir::HirId, decl: &hir::FnDecl<'_>) {
-        let def_id = self.cx.tcx.hir().local_def_id(id);
-        let sig = self.cx.tcx.fn_sig(def_id);
+    fn check_foreign_fn(&mut self, def_id: LocalDefId, decl: &hir::FnDecl<'_>) {
+        let sig = self.cx.tcx.fn_sig(def_id).subst_identity();
         let sig = self.cx.tcx.erase_late_bound_regions(sig);
 
         for (input_ty, input_hir) in iter::zip(sig.inputs(), decl.inputs) {
@@ -1238,9 +1244,8 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
         }
     }
 
-    fn check_foreign_static(&mut self, id: hir::HirId, span: Span) {
-        let def_id = self.cx.tcx.hir().local_def_id(id);
-        let ty = self.cx.tcx.type_of(def_id);
+    fn check_foreign_static(&mut self, id: hir::OwnerId, span: Span) {
+        let ty = self.cx.tcx.type_of(id).subst_identity();
         self.check_type_for_ffi_and_report_errors(span, ty, true, false);
     }
 
@@ -1260,10 +1265,10 @@ impl<'tcx> LateLintPass<'tcx> for ImproperCTypesDeclarations {
         if !vis.is_internal_abi(abi) {
             match it.kind {
                 hir::ForeignItemKind::Fn(ref decl, _, _) => {
-                    vis.check_foreign_fn(it.hir_id(), decl);
+                    vis.check_foreign_fn(it.owner_id.def_id, decl);
                 }
                 hir::ForeignItemKind::Static(ref ty, _) => {
-                    vis.check_foreign_static(it.hir_id(), ty.span);
+                    vis.check_foreign_static(it.owner_id, ty.span);
                 }
                 hir::ForeignItemKind::Type => (),
             }
@@ -1279,7 +1284,7 @@ impl<'tcx> LateLintPass<'tcx> for ImproperCTypesDefinitions {
         decl: &'tcx hir::FnDecl<'_>,
         _: &'tcx hir::Body<'_>,
         _: Span,
-        hir_id: hir::HirId,
+        id: LocalDefId,
     ) {
         use hir::intravisit::FnKind;
 
@@ -1291,7 +1296,7 @@ impl<'tcx> LateLintPass<'tcx> for ImproperCTypesDefinitions {
 
         let mut vis = ImproperCTypesVisitor { cx, mode: CItemKind::Definition };
         if !vis.is_internal_abi(abi) {
-            vis.check_foreign_fn(hir_id, decl);
+            vis.check_foreign_fn(id, decl);
         }
     }
 }
@@ -1301,7 +1306,7 @@ declare_lint_pass!(VariantSizeDifferences => [VARIANT_SIZE_DIFFERENCES]);
 impl<'tcx> LateLintPass<'tcx> for VariantSizeDifferences {
     fn check_item(&mut self, cx: &LateContext<'_>, it: &hir::Item<'_>) {
         if let hir::ItemKind::Enum(ref enum_definition, _) = it.kind {
-            let t = cx.tcx.type_of(it.owner_id);
+            let t = cx.tcx.type_of(it.owner_id).subst_identity();
             let ty = cx.tcx.erase_regions(t);
             let Ok(layout) = cx.layout_of(ty) else { return };
             let Variants::Multiple {
@@ -1421,7 +1426,7 @@ impl InvalidAtomicOrdering {
             && recognized_names.contains(&method_path.ident.name)
             && let Some(m_def_id) = cx.typeck_results().type_dependent_def_id(expr.hir_id)
             && let Some(impl_did) = cx.tcx.impl_of_method(m_def_id)
-            && let Some(adt) = cx.tcx.type_of(impl_did).ty_adt_def()
+            && let Some(adt) = cx.tcx.type_of(impl_did).subst_identity().ty_adt_def()
             // skip extension traits, only lint functions from the standard library
             && cx.tcx.trait_id_of_impl(impl_did).is_none()
             && let parent = cx.tcx.parent(adt.did())
diff --git a/compiler/rustc_lint/src/unused.rs b/compiler/rustc_lint/src/unused.rs
index 4c9b3df2d..3a92f5806 100644
--- a/compiler/rustc_lint/src/unused.rs
+++ b/compiler/rustc_lint/src/unused.rs
@@ -1,7 +1,7 @@
 use crate::lints::{
     PathStatementDrop, PathStatementDropSub, PathStatementNoEffect, UnusedAllocationDiag,
-    UnusedAllocationMutDiag, UnusedClosure, UnusedDef, UnusedDelim, UnusedDelimSuggestion,
-    UnusedGenerator, UnusedImportBracesDiag, UnusedOp, UnusedResult,
+    UnusedAllocationMutDiag, UnusedClosure, UnusedDef, UnusedDefSuggestion, UnusedDelim,
+    UnusedDelimSuggestion, UnusedGenerator, UnusedImportBracesDiag, UnusedOp, UnusedResult,
 };
 use crate::Lint;
 use crate::{EarlyContext, EarlyLintPass, LateContext, LateLintPass, LintContext};
@@ -309,7 +309,7 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults {
                         None
                     }
                 }
-                ty::Array(ty, len) => match len.try_eval_usize(cx.tcx, cx.param_env) {
+                ty::Array(ty, len) => match len.try_eval_target_usize(cx.tcx, cx.param_env) {
                     // If the array is empty we don't lint, to avoid false positives
                     Some(0) | None => None,
                     // If the array is definitely non-empty, we can do `#[must_use]` checking.
@@ -418,6 +418,19 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults {
                     );
                 }
                 MustUsePath::Def(span, def_id, reason) => {
+                    let suggestion = if matches!(
+                        cx.tcx.get_diagnostic_name(*def_id),
+                        Some(sym::add)
+                            | Some(sym::sub)
+                            | Some(sym::mul)
+                            | Some(sym::div)
+                            | Some(sym::rem)
+                            | Some(sym::neg),
+                    ) {
+                        Some(UnusedDefSuggestion::Default { span: span.shrink_to_lo() })
+                    } else {
+                        None
+                    };
                     cx.emit_spanned_lint(
                         UNUSED_MUST_USE,
                         *span,
@@ -427,6 +440,7 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults {
                             cx,
                             def_id: *def_id,
                             note: *reason,
+                            suggestion,
                         },
                     );
                 }
@@ -495,6 +509,7 @@ enum UnusedDelimsCtx {
     ArrayLenExpr,
     AnonConst,
     MatchArmExpr,
+    IndexExpr,
 }
 
 impl From<UnusedDelimsCtx> for &'static str {
@@ -514,6 +529,7 @@ impl From<UnusedDelimsCtx> for &'static str {
             UnusedDelimsCtx::LetScrutineeExpr => "`let` scrutinee expression",
             UnusedDelimsCtx::ArrayLenExpr | UnusedDelimsCtx::AnonConst => "const expression",
             UnusedDelimsCtx::MatchArmExpr => "match arm expression",
+            UnusedDelimsCtx::IndexExpr => "index expression",
         }
     }
 }
@@ -661,6 +677,10 @@ trait UnusedDelimLint {
         keep_space: (bool, bool),
     ) {
         let primary_span = if let Some((lo, hi)) = spans {
+            if hi.is_empty() {
+                // do not point at delims that do not exist
+                return;
+            }
             MultiSpan::from(vec![lo, hi])
         } else {
             MultiSpan::from(value_span)
@@ -733,6 +753,8 @@ trait UnusedDelimLint {
                 (value, UnusedDelimsCtx::ReturnValue, false, Some(left), None)
             }
 
+            Index(_, ref value) => (value, UnusedDelimsCtx::IndexExpr, false, None, None),
+
             Assign(_, ref value, _) | AssignOp(.., ref value) => {
                 (value, UnusedDelimsCtx::AssignedValue, false, None, None)
             }
diff --git a/compiler/rustc_lint_defs/src/builtin.rs b/compiler/rustc_lint_defs/src/builtin.rs
index b6481d70b..46ec1a2dc 100644
--- a/compiler/rustc_lint_defs/src/builtin.rs
+++ b/compiler/rustc_lint_defs/src/builtin.rs
@@ -1188,51 +1188,6 @@ declare_lint! {
 }
 
 declare_lint! {
-    /// The `unaligned_references` lint detects unaligned references to fields
-    /// of [packed] structs.
-    ///
-    /// [packed]: https://doc.rust-lang.org/reference/type-layout.html#the-alignment-modifiers
-    ///
-    /// ### Example
-    ///
-    /// ```rust,compile_fail
-    /// #[repr(packed)]
-    /// pub struct Foo {
-    ///     field1: u64,
-    ///     field2: u8,
-    /// }
-    ///
-    /// fn main() {
-    ///     unsafe {
-    ///         let foo = Foo { field1: 0, field2: 0 };
-    ///         let _ = &foo.field1;
-    ///         println!("{}", foo.field1); // An implicit `&` is added here, triggering the lint.
-    ///     }
-    /// }
-    /// ```
-    ///
-    /// {{produces}}
-    ///
-    /// ### Explanation
-    ///
-    /// Creating a reference to an insufficiently aligned packed field is [undefined behavior] and
-    /// should be disallowed. Using an `unsafe` block does not change anything about this. Instead,
-    /// the code should do a copy of the data in the packed field or use raw pointers and unaligned
-    /// accesses. See [issue #82523] for more information.
-    ///
-    /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html
-    /// [issue #82523]: https://github.com/rust-lang/rust/issues/82523
-    pub UNALIGNED_REFERENCES,
-    Deny,
-    "detects unaligned references to fields of packed structs",
-    @future_incompatible = FutureIncompatibleInfo {
-        reference: "issue #82523 <https://github.com/rust-lang/rust/issues/82523>",
-        reason: FutureIncompatibilityReason::FutureReleaseErrorReportNow,
-    };
-    report_in_external_macro
-}
-
-declare_lint! {
     /// The `const_item_mutation` lint detects attempts to mutate a `const`
     /// item.
     ///
@@ -3308,7 +3263,6 @@ declare_lint_pass! {
         PUB_USE_OF_PRIVATE_EXTERN_CRATE,
         INVALID_TYPE_PARAM_DEFAULT,
         RENAMED_AND_REMOVED_LINTS,
-        UNALIGNED_REFERENCES,
         CONST_ITEM_MUTATION,
         PATTERNS_IN_FNS_WITHOUT_BODY,
         MISSING_FRAGMENT_SPECIFIER,
@@ -3381,6 +3335,7 @@ declare_lint_pass! {
         REPR_TRANSPARENT_EXTERNAL_PRIVATE_FIELDS,
         NAMED_ARGUMENTS_USED_POSITIONALLY,
         IMPLIED_BOUNDS_ENTAILMENT,
+        BYTE_SLICE_IN_PACKED_STRUCT_WITH_DERIVE,
     ]
 }
 
@@ -3531,9 +3486,15 @@ declare_lint! {
     ///
     /// ### Explanation
     ///
-    /// Previously, there were very like checks being performed on `#[doc(..)]`
-    /// unlike the other attributes. It'll now catch all the issues that it
-    /// silently ignored previously.
+    /// Previously, incorrect usage of the `#[doc(..)]` attribute was not
+    /// being validated. Usually these should be rejected as a hard error,
+    /// but this lint was introduced to avoid breaking any existing
+    /// crates which included them.
+    ///
+    /// This is a [future-incompatible] lint to transition this to a hard
+    /// error in the future. See [issue #82730] for more details.
+    ///
+    /// [issue #82730]: https://github.com/rust-lang/rust/issues/82730
     pub INVALID_DOC_ATTRIBUTES,
     Warn,
     "detects invalid `#[doc(...)]` attributes",
@@ -4109,3 +4070,66 @@ declare_lint! {
         reason: FutureIncompatibilityReason::FutureReleaseErrorReportNow,
     };
 }
+
+declare_lint! {
+    /// The `byte_slice_in_packed_struct_with_derive` lint detects cases where a byte slice field
+    /// (`[u8]`) or string slice field (`str`) is used in a `packed` struct that derives one or
+    /// more built-in traits.
+    ///
+    /// ### Example
+    ///
+    /// ```rust
+    /// #[repr(packed)]
+    /// #[derive(Hash)]
+    /// struct FlexZeroSlice {
+    ///     width: u8,
+    ///     data: [u8],
+    /// }
+    /// ```
+    ///
+    /// {{produces}}
+    ///
+    /// ### Explanation
+    ///
+    /// This was previously accepted but is being phased out, because fields in packed structs are
+    /// now required to implement `Copy` for `derive` to work. Byte slices and string slices are a
+    /// temporary exception because certain crates depended on them.
+    pub BYTE_SLICE_IN_PACKED_STRUCT_WITH_DERIVE,
+    Warn,
+    "`[u8]` or `str` used in a packed struct with `derive`",
+    @future_incompatible = FutureIncompatibleInfo {
+        reference: "issue #107457 <https://github.com/rust-lang/rust/issues/107457>",
+        reason: FutureIncompatibilityReason::FutureReleaseErrorReportNow,
+    };
+    report_in_external_macro
+}
+
+declare_lint! {
+    /// The `invalid_macro_export_arguments` lint detects cases where `#[macro_export]` is being used with invalid arguments.
+    ///
+    /// ### Example
+    ///
+    /// ```rust,compile_fail
+    /// #![deny(invalid_macro_export_arguments)]
+    ///
+    /// #[macro_export(invalid_parameter)]
+    /// macro_rules! myMacro {
+    ///    () => {
+    ///         // [...]
+    ///    }
+    /// }
+    ///
+    /// #[macro_export(too, many, items)]
+    /// ```
+    ///
+    /// {{produces}}
+    ///
+    /// ### Explanation
+    ///
+    /// The only valid argument is `#[macro_export(local_inner_macros)]` or no argument (`#[macro_export]`).
+    /// You can't have multiple arguments in a `#[macro_export(..)]`, or mention arguments other than `local_inner_macros`.
+    ///
+    pub INVALID_MACRO_EXPORT_ARGUMENTS,
+    Warn,
+    "\"invalid_parameter\" isn't a valid argument for `#[macro_export]`",
+}
diff --git a/compiler/rustc_lint_defs/src/lib.rs b/compiler/rustc_lint_defs/src/lib.rs
index 7054d1e9f..534aff7fb 100644
--- a/compiler/rustc_lint_defs/src/lib.rs
+++ b/compiler/rustc_lint_defs/src/lib.rs
@@ -521,6 +521,14 @@ pub enum BuiltinLintDiagnostics {
         /// Indicates if the named argument is used as a width/precision for formatting
         is_formatting_arg: bool,
     },
+    ByteSliceInPackedStructWithDerive,
+    UnusedExternCrate {
+        removal_span: Span,
+    },
+    ExternCrateNotIdiomatic {
+        vis_span: Span,
+        ident_span: Span,
+    },
 }
 
 /// Lints that are buffered up early on in the `Session` before the
diff --git a/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h b/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h
index 727cfc441..058906283 100644
--- a/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h
+++ b/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h
@@ -4,7 +4,6 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/Triple.h"
 #include "llvm/Analysis/Lint.h"
 #include "llvm/Analysis/Passes.h"
 #include "llvm/IR/IRBuilder.h"
@@ -15,6 +14,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/DynamicLibrary.h"
 #include "llvm/Support/FormattedStream.h"
+#include "llvm/Support/JSON.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/Memory.h"
 #include "llvm/Support/SourceMgr.h"
@@ -44,6 +44,12 @@
 #include "llvm/IR/IRPrintingPasses.h"
 #include "llvm/Linker/Linker.h"
 
+#if LLVM_VERSION_GE(16, 0)
+#include "llvm/TargetParser/Triple.h"
+#else
+#include "llvm/ADT/Triple.h"
+#endif
+
 extern "C" void LLVMRustSetLastError(const char *);
 
 enum class LLVMRustResult { Success, Failure };
diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp
index f728bff0e..4761ce83f 100644
--- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp
@@ -9,11 +9,11 @@
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
-#include "llvm/InitializePasses.h"
 #include "llvm/IR/AutoUpgrade.h"
 #include "llvm/IR/AssemblyAnnotationWriter.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Verifier.h"
+#include "llvm/MC/TargetRegistry.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Object/IRObjectFile.h"
 #include "llvm/Passes/PassBuilder.h"
@@ -21,12 +21,10 @@
 #include "llvm/Passes/StandardInstrumentations.h"
 #include "llvm/Support/CBindingWrapping.h"
 #include "llvm/Support/FileSystem.h"
-#include "llvm/Support/Host.h"
-#if LLVM_VERSION_LT(14, 0)
-#include "llvm/Support/TargetRegistry.h"
-#else
-#include "llvm/MC/TargetRegistry.h"
+#if LLVM_VERSION_GE(17, 0)
+#include "llvm/Support/VirtualFileSystem.h"
 #endif
+#include "llvm/Support/Host.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
 #include "llvm/Transforms/IPO/AlwaysInliner.h"
@@ -59,22 +57,6 @@ typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
 
-extern "C" void LLVMInitializePasses() {
-  PassRegistry &Registry = *PassRegistry::getPassRegistry();
-  initializeCore(Registry);
-  initializeCodeGen(Registry);
-  initializeScalarOpts(Registry);
-  initializeVectorization(Registry);
-  initializeIPO(Registry);
-  initializeAnalysis(Registry);
-  initializeTransformUtils(Registry);
-  initializeInstCombine(Registry);
-#if LLVM_VERSION_LT(16, 0)
-  initializeInstrumentation(Registry);
-#endif
-  initializeTarget(Registry);
-}
-
 extern "C" void LLVMTimeTraceProfilerInitialize() {
   timeTraceProfilerInitialize(
       /* TimeTraceGranularity */ 0,
@@ -264,10 +246,6 @@ enum class LLVMRustPassBuilderOptLevel {
   Oz,
 };
 
-#if LLVM_VERSION_LT(14,0)
-using OptimizationLevel = PassBuilder::OptimizationLevel;
-#endif
-
 static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
   switch (Level) {
   case LLVMRustPassBuilderOptLevel::O0:
@@ -599,6 +577,8 @@ struct LLVMRustSanitizerOptions {
   bool SanitizeThread;
   bool SanitizeHWAddress;
   bool SanitizeHWAddressRecover;
+  bool SanitizeKernelAddress;
+  bool SanitizeKernelAddressRecover;
 };
 
 extern "C" LLVMRustResult
@@ -652,20 +632,39 @@ LLVMRustOptimize(
 #else
   std::optional<PGOOptions> PGOOpt;
 #endif
+#if LLVM_VERSION_GE(17, 0)
+  auto FS = vfs::getRealFileSystem();
+#endif
   if (PGOGenPath) {
     assert(!PGOUsePath && !PGOSampleUsePath);
-    PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr,
-                        PGOOptions::NoCSAction, DebugInfoForProfiling);
+    PGOOpt = PGOOptions(PGOGenPath, "", "",
+#if LLVM_VERSION_GE(17, 0)
+                        FS,
+#endif
+                        PGOOptions::IRInstr, PGOOptions::NoCSAction,
+                        DebugInfoForProfiling);
   } else if (PGOUsePath) {
     assert(!PGOSampleUsePath);
-    PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse,
-                        PGOOptions::NoCSAction, DebugInfoForProfiling);
+    PGOOpt = PGOOptions(PGOUsePath, "", "",
+#if LLVM_VERSION_GE(17, 0)
+                        FS,
+#endif
+                        PGOOptions::IRUse, PGOOptions::NoCSAction,
+                        DebugInfoForProfiling);
   } else if (PGOSampleUsePath) {
-    PGOOpt = PGOOptions(PGOSampleUsePath, "", "", PGOOptions::SampleUse,
-                        PGOOptions::NoCSAction, DebugInfoForProfiling);
+    PGOOpt = PGOOptions(PGOSampleUsePath, "", "",
+#if LLVM_VERSION_GE(17, 0)
+                        FS,
+#endif
+                        PGOOptions::SampleUse, PGOOptions::NoCSAction,
+                        DebugInfoForProfiling);
   } else if (DebugInfoForProfiling) {
-    PGOOpt = PGOOptions("", "", "", PGOOptions::NoAction,
-                        PGOOptions::NoCSAction, DebugInfoForProfiling);
+    PGOOpt = PGOOptions("", "", "",
+#if LLVM_VERSION_GE(17, 0)
+                        FS,
+#endif
+                        PGOOptions::NoAction, PGOOptions::NoCSAction,
+                        DebugInfoForProfiling);
   }
 
   PassBuilder PB(TM, PTO, PGOOpt, &PIC);
@@ -725,28 +724,19 @@ LLVMRustOptimize(
 
   if (SanitizerOptions) {
     if (SanitizerOptions->SanitizeMemory) {
-#if LLVM_VERSION_GE(14, 0)
       MemorySanitizerOptions Options(
           SanitizerOptions->SanitizeMemoryTrackOrigins,
           SanitizerOptions->SanitizeMemoryRecover,
           /*CompileKernel=*/false,
           /*EagerChecks=*/true);
-#else
-      MemorySanitizerOptions Options(
-          SanitizerOptions->SanitizeMemoryTrackOrigins,
-          SanitizerOptions->SanitizeMemoryRecover,
-          /*CompileKernel=*/false);
-#endif
       OptimizerLastEPCallbacks.push_back(
         [Options](ModulePassManager &MPM, OptimizationLevel Level) {
-#if LLVM_VERSION_GE(14, 0) && LLVM_VERSION_LT(16, 0)
+#if LLVM_VERSION_LT(16, 0)
           MPM.addPass(ModuleMemorySanitizerPass(Options));
+          MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options)));
 #else
           MPM.addPass(MemorySanitizerPass(Options));
 #endif
-#if LLVM_VERSION_LT(16, 0)
-          MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options)));
-#endif
         }
       );
     }
@@ -754,26 +744,23 @@ LLVMRustOptimize(
     if (SanitizerOptions->SanitizeThread) {
       OptimizerLastEPCallbacks.push_back(
         [](ModulePassManager &MPM, OptimizationLevel Level) {
-#if LLVM_VERSION_GE(14, 0)
           MPM.addPass(ModuleThreadSanitizerPass());
-#else
-          MPM.addPass(ThreadSanitizerPass());
-#endif
           MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
         }
       );
     }
 
-    if (SanitizerOptions->SanitizeAddress) {
+    if (SanitizerOptions->SanitizeAddress || SanitizerOptions->SanitizeKernelAddress) {
       OptimizerLastEPCallbacks.push_back(
         [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
+          auto CompileKernel = SanitizerOptions->SanitizeKernelAddress;
 #if LLVM_VERSION_LT(15, 0)
           MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
 #endif
-#if LLVM_VERSION_GE(14, 0)
           AddressSanitizerOptions opts = AddressSanitizerOptions{
-            /*CompileKernel=*/false,
-            SanitizerOptions->SanitizeAddressRecover,
+            CompileKernel,
+            SanitizerOptions->SanitizeAddressRecover
+              || SanitizerOptions->SanitizeKernelAddressRecover,
             /*UseAfterScope=*/true,
             AsanDetectStackUseAfterReturnMode::Runtime,
           };
@@ -782,28 +769,16 @@ LLVMRustOptimize(
 #else
           MPM.addPass(AddressSanitizerPass(opts));
 #endif
-#else
-          MPM.addPass(ModuleAddressSanitizerPass(
-              /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
-          MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(
-              /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
-              /*UseAfterScope=*/true)));
-#endif
         }
       );
     }
     if (SanitizerOptions->SanitizeHWAddress) {
       OptimizerLastEPCallbacks.push_back(
         [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
-#if LLVM_VERSION_GE(14, 0)
           HWAddressSanitizerOptions opts(
               /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover,
               /*DisableOptimization=*/false);
           MPM.addPass(HWAddressSanitizerPass(opts));
-#else
-          MPM.addPass(HWAddressSanitizerPass(
-              /*CompileKernel=*/false, SanitizerOptions->SanitizeHWAddressRecover));
-#endif
         }
       );
     }
@@ -1012,23 +987,8 @@ LLVMRustPrintModule(LLVMModuleRef M, const char *Path, DemangleFn Demangle) {
 }
 
 extern "C" void LLVMRustPrintPasses() {
-  LLVMInitializePasses();
-  struct MyListener : PassRegistrationListener {
-    void passEnumerate(const PassInfo *Info) {
-      StringRef PassArg = Info->getPassArgument();
-      StringRef PassName = Info->getPassName();
-      if (!PassArg.empty()) {
-        // These unsigned->signed casts could theoretically overflow, but
-        // realistically never will (and even if, the result is implementation
-        // defined rather plain UB).
-        printf("%15.*s - %.*s\n", (int)PassArg.size(), PassArg.data(),
-               (int)PassName.size(), PassName.data());
-      }
-    }
-  } Listener;
-
-  PassRegistry *PR = PassRegistry::getPassRegistry();
-  PR->enumerateWith(&Listener);
+  PassBuilder PB;
+  PB.printPassNames(outs());
 }
 
 extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols,
@@ -1306,11 +1266,7 @@ extern "C" bool
 LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
   Module &Mod = *unwrap(M);
   const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier());
-#if LLVM_VERSION_GE(14, 0)
   thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true);
-#else
-  thinLTOResolvePrevailingInModule(Mod, DefinedGlobals);
-#endif
   return true;
 }
 
diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
index 87b0e1273..e3493caaa 100644
--- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
@@ -63,11 +63,7 @@ static LLVM_THREAD_LOCAL char *LastError;
 //
 // Notably it exits the process with code 101, unlike LLVM's default of 1.
 static void FatalErrorHandler(void *UserData,
-#if LLVM_VERSION_LT(14, 0)
-                              const std::string& Reason,
-#else
                               const char* Reason,
-#endif
                               bool GenCrashDiag) {
   // Do the same thing that the default error handler does.
   std::cerr << "LLVM ERROR: " << Reason << std::endl;
@@ -249,18 +245,10 @@ static Attribute::AttrKind fromRust(LLVMRustAttribute Kind) {
 template<typename T> static inline void AddAttributes(T *t, unsigned Index,
                                                       LLVMAttributeRef *Attrs, size_t AttrsLen) {
   AttributeList PAL = t->getAttributes();
-  AttributeList PALNew;
-#if LLVM_VERSION_LT(14, 0)
-  AttrBuilder B;
-  for (LLVMAttributeRef Attr : makeArrayRef(Attrs, AttrsLen))
-    B.addAttribute(unwrap(Attr));
-  PALNew = PAL.addAttributes(t->getContext(), Index, B);
-#else
   AttrBuilder B(t->getContext());
   for (LLVMAttributeRef Attr : ArrayRef<LLVMAttributeRef>(Attrs, AttrsLen))
     B.addAttribute(unwrap(Attr));
-  PALNew = PAL.addAttributesAtIndex(t->getContext(), Index, B);
-#endif
+  AttributeList PALNew = PAL.addAttributesAtIndex(t->getContext(), Index, B);
   t->setAttributes(PALNew);
 }
 
@@ -1763,6 +1751,19 @@ LLVMRustModuleCost(LLVMModuleRef M) {
   return std::distance(std::begin(f), std::end(f));
 }
 
+extern "C" void
+LLVMRustModuleInstructionStats(LLVMModuleRef M, RustStringRef Str)
+{
+  RawRustStringOstream OS(Str);
+  llvm::json::OStream JOS(OS);
+  auto Module = unwrap(M);
+
+  JOS.object([&] {
+    JOS.attribute("module", Module->getName());
+    JOS.attribute("total", Module->getInstructionCount());
+  });
+}
+
 // Vector reductions:
 extern "C" LLVMValueRef
 LLVMRustBuildVectorReduceFAdd(LLVMBuilderRef B, LLVMValueRef Acc, LLVMValueRef Src) {
diff --git a/compiler/rustc_log/src/lib.rs b/compiler/rustc_log/src/lib.rs
index fc1cabd2d..22924efa9 100644
--- a/compiler/rustc_log/src/lib.rs
+++ b/compiler/rustc_log/src/lib.rs
@@ -14,7 +14,7 @@
 //!
 //! ```
 //! fn main() {
-//!     rustc_log::init_rustc_env_logger().unwrap();
+//!     rustc_log::init_env_logger("LOG").unwrap();
 //!
 //!     let edition = rustc_span::edition::Edition::Edition2021;
 //!     rustc_span::create_session_globals_then(edition, || {
@@ -23,9 +23,9 @@
 //! }
 //! ```
 //!
-//! Now `RUSTC_LOG=debug cargo run` will run your minimal main.rs and show
+//! Now `LOG=debug cargo run` will run your minimal main.rs and show
 //! rustc's debug logging. In a workflow like this, one might also add
-//! `std::env::set_var("RUSTC_LOG", "debug")` to the top of main so that `cargo
+//! `std::env::set_var("LOG", "debug")` to the top of main so that `cargo
 //! run` by itself is sufficient to get logs.
 //!
 //! The reason rustc_log is a tiny separate crate, as opposed to exposing the
@@ -53,26 +53,7 @@ use tracing_subscriber::fmt::{
 };
 use tracing_subscriber::layer::SubscriberExt;
 
-pub fn init_rustc_env_logger() -> Result<(), Error> {
-    init_rustc_env_logger_with_backtrace_option(&None)
-}
-
-pub fn init_rustc_env_logger_with_backtrace_option(
-    backtrace_target: &Option<String>,
-) -> Result<(), Error> {
-    init_env_logger_with_backtrace_option("RUSTC_LOG", backtrace_target)
-}
-
-/// In contrast to `init_rustc_env_logger` this allows you to choose an env var
-/// other than `RUSTC_LOG`.
 pub fn init_env_logger(env: &str) -> Result<(), Error> {
-    init_env_logger_with_backtrace_option(env, &None)
-}
-
-pub fn init_env_logger_with_backtrace_option(
-    env: &str,
-    backtrace_target: &Option<String>,
-) -> Result<(), Error> {
     let filter = match env::var(env) {
         Ok(env) => EnvFilter::new(env),
         _ => EnvFilter::default().add_directive(Directive::from(LevelFilter::WARN)),
@@ -106,16 +87,16 @@ pub fn init_env_logger_with_backtrace_option(
     let layer = layer.with_thread_ids(true).with_thread_names(true);
 
     let subscriber = tracing_subscriber::Registry::default().with(filter).with(layer);
-    match backtrace_target {
-        Some(str) => {
+    match env::var(format!("{env}_BACKTRACE")) {
+        Ok(str) => {
             let fmt_layer = tracing_subscriber::fmt::layer()
                 .with_writer(io::stderr)
                 .without_time()
-                .event_format(BacktraceFormatter { backtrace_target: str.to_string() });
+                .event_format(BacktraceFormatter { backtrace_target: str });
             let subscriber = subscriber.with(fmt_layer);
             tracing::subscriber::set_global_default(subscriber).unwrap();
         }
-        None => {
+        Err(_) => {
             tracing::subscriber::set_global_default(subscriber).unwrap();
         }
     };
diff --git a/compiler/rustc_macros/src/diagnostics/diagnostic.rs b/compiler/rustc_macros/src/diagnostics/diagnostic.rs
index 9ff944864..12a954258 100644
--- a/compiler/rustc_macros/src/diagnostics/diagnostic.rs
+++ b/compiler/rustc_macros/src/diagnostics/diagnostic.rs
@@ -57,7 +57,7 @@ impl<'a> DiagnosticDerive<'a> {
                 }
                 Some(slug) => {
                     quote! {
-                        let mut #diag = #handler.struct_diagnostic(rustc_errors::fluent::#slug);
+                        let mut #diag = #handler.struct_diagnostic(crate::fluent_generated::#slug);
                     }
                 }
             };
@@ -149,7 +149,7 @@ impl<'a> LintDiagnosticDerive<'a> {
                 }
                 Some(slug) => {
                     quote! {
-                        rustc_errors::fluent::#slug.into()
+                        crate::fluent_generated::#slug.into()
                     }
                 }
             }
diff --git a/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs b/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs
index e405462bb..46068f8c8 100644
--- a/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs
+++ b/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs
@@ -322,11 +322,7 @@ impl<'a> DiagnosticDeriveVariantBuilder<'a> {
                 let generated_code = self
                     .generate_inner_field_code(
                         attr,
-                        FieldInfo {
-                            binding: binding_info,
-                            ty: inner_ty.inner_type().unwrap_or(&field.ty),
-                            span: &field.span(),
-                        },
+                        FieldInfo { binding: binding_info, ty: inner_ty, span: &field.span() },
                         binding,
                     )
                     .unwrap_or_else(|v| v.to_compile_error());
@@ -418,9 +414,9 @@ impl<'a> DiagnosticDeriveVariantBuilder<'a> {
                 Ok(self.add_spanned_subdiagnostic(binding, &fn_ident, slug))
             }
             SubdiagnosticKind::Note | SubdiagnosticKind::Help | SubdiagnosticKind::Warn => {
-                if type_matches_path(info.ty, &["rustc_span", "Span"]) {
+                if type_matches_path(info.ty.inner_type(), &["rustc_span", "Span"]) {
                     Ok(self.add_spanned_subdiagnostic(binding, &fn_ident, slug))
-                } else if type_is_unit(info.ty) {
+                } else if type_is_unit(info.ty.inner_type()) {
                     Ok(self.add_subdiagnostic(&fn_ident, slug))
                 } else {
                     report_type_error(attr, "`Span` or `()`")?
@@ -432,6 +428,15 @@ impl<'a> DiagnosticDeriveVariantBuilder<'a> {
                 code_field,
                 code_init,
             } => {
+                if let FieldInnerTy::Vec(_) = info.ty {
+                    throw_invalid_attr!(attr, &meta, |diag| {
+                        diag
+                        .note("`#[suggestion(...)]` applied to `Vec` field is ambiguous")
+                        .help("to show a suggestion consisting of multiple parts, use a `Subdiagnostic` annotated with `#[multipart_suggestion(...)]`")
+                        .help("to show a variable set of suggestions, use a `Vec` of `Subdiagnostic`s annotated with `#[suggestion(...)]`")
+                    });
+                }
+
                 let (span_field, mut applicability) = self.span_and_applicability_of_ty(info)?;
 
                 if let Some((static_applicability, span)) = static_applicability {
@@ -447,7 +452,7 @@ impl<'a> DiagnosticDeriveVariantBuilder<'a> {
                 Ok(quote! {
                     #diag.span_suggestions_with_style(
                         #span_field,
-                        rustc_errors::fluent::#slug,
+                        crate::fluent_generated::#slug,
                         #code_field,
                         #applicability,
                         #style
@@ -471,7 +476,7 @@ impl<'a> DiagnosticDeriveVariantBuilder<'a> {
         quote! {
             #diag.#fn_name(
                 #field_binding,
-                rustc_errors::fluent::#fluent_attr_identifier
+                crate::fluent_generated::#fluent_attr_identifier
             );
         }
     }
@@ -481,7 +486,7 @@ impl<'a> DiagnosticDeriveVariantBuilder<'a> {
     fn add_subdiagnostic(&self, kind: &Ident, fluent_attr_identifier: Path) -> TokenStream {
         let diag = &self.parent.diag;
         quote! {
-            #diag.#kind(rustc_errors::fluent::#fluent_attr_identifier);
+            #diag.#kind(crate::fluent_generated::#fluent_attr_identifier);
         }
     }
 
@@ -489,7 +494,7 @@ impl<'a> DiagnosticDeriveVariantBuilder<'a> {
         &self,
         info: FieldInfo<'_>,
     ) -> Result<(TokenStream, SpannedOption<TokenStream>), DiagnosticDeriveError> {
-        match &info.ty {
+        match &info.ty.inner_type() {
             // If `ty` is `Span` w/out applicability, then use `Applicability::Unspecified`.
             ty @ Type::Path(..) if type_matches_path(ty, &["rustc_span", "Span"]) => {
                 let binding = &info.binding.binding;
diff --git a/compiler/rustc_macros/src/diagnostics/fluent.rs b/compiler/rustc_macros/src/diagnostics/fluent.rs
index 32338f9df..38c0f4895 100644
--- a/compiler/rustc_macros/src/diagnostics/fluent.rs
+++ b/compiler/rustc_macros/src/diagnostics/fluent.rs
@@ -4,7 +4,10 @@ use annotate_snippets::{
 };
 use fluent_bundle::{FluentBundle, FluentError, FluentResource};
 use fluent_syntax::{
-    ast::{Attribute, Entry, Identifier, Message},
+    ast::{
+        Attribute, Entry, Expression, Identifier, InlineExpression, Message, Pattern,
+        PatternElement,
+    },
     parser::ParserError,
 };
 use proc_macro::{Diagnostic, Level, Span};
@@ -16,52 +19,9 @@ use std::{
     io::Read,
     path::{Path, PathBuf},
 };
-use syn::{
-    parse::{Parse, ParseStream},
-    parse_macro_input,
-    punctuated::Punctuated,
-    token, Ident, LitStr, Result,
-};
+use syn::{parse_macro_input, Ident, LitStr};
 use unic_langid::langid;
 
-struct Resource {
-    krate: Ident,
-    #[allow(dead_code)]
-    fat_arrow_token: token::FatArrow,
-    resource_path: LitStr,
-}
-
-impl Parse for Resource {
-    fn parse(input: ParseStream<'_>) -> Result<Self> {
-        Ok(Resource {
-            krate: input.parse()?,
-            fat_arrow_token: input.parse()?,
-            resource_path: input.parse()?,
-        })
-    }
-}
-
-struct Resources(Punctuated<Resource, token::Comma>);
-
-impl Parse for Resources {
-    fn parse(input: ParseStream<'_>) -> Result<Self> {
-        let mut resources = Punctuated::new();
-        loop {
-            if input.is_empty() || input.peek(token::Brace) {
-                break;
-            }
-            let value = input.parse()?;
-            resources.push_value(value);
-            if !input.peek(token::Comma) {
-                break;
-            }
-            let punct = input.parse()?;
-            resources.push_punct(punct);
-        }
-        Ok(Resources(resources))
-    }
-}
-
 /// Helper function for returning an absolute path for macro-invocation relative file paths.
 ///
 /// If the input is already absolute, then the input is returned. If the input is not absolute,
@@ -81,224 +41,285 @@ fn invocation_relative_path_to_absolute(span: Span, path: &str) -> PathBuf {
     }
 }
 
+/// Tokens to be returned when the macro cannot proceed.
+fn failed(crate_name: &Ident) -> proc_macro::TokenStream {
+    quote! {
+        pub static DEFAULT_LOCALE_RESOURCE: &'static str = "";
+
+        #[allow(non_upper_case_globals)]
+        #[doc(hidden)]
+        pub(crate) mod fluent_generated {
+            pub mod #crate_name {
+            }
+
+            pub mod _subdiag {
+                pub const help: crate::SubdiagnosticMessage =
+                    crate::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed("help"));
+                pub const note: crate::SubdiagnosticMessage =
+                    crate::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed("note"));
+                pub const warn: crate::SubdiagnosticMessage =
+                    crate::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed("warn"));
+                pub const label: crate::SubdiagnosticMessage =
+                    crate::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed("label"));
+                pub const suggestion: crate::SubdiagnosticMessage =
+                    crate::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed("suggestion"));
+            }
+        }
+    }
+    .into()
+}
+
 /// See [rustc_macros::fluent_messages].
 pub(crate) fn fluent_messages(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
-    let resources = parse_macro_input!(input as Resources);
+    let crate_name = std::env::var("CARGO_PKG_NAME")
+        // If `CARGO_PKG_NAME` is missing, then we're probably running in a test, so use
+        // `no_crate`.
+        .unwrap_or_else(|_| "no_crate".to_string())
+        .replace("rustc_", "");
 
     // Cannot iterate over individual messages in a bundle, so do that using the
     // `FluentResource` instead. Construct a bundle anyway to find out if there are conflicting
     // messages in the resources.
     let mut bundle = FluentBundle::new(vec![langid!("en-US")]);
 
-    // Map of Fluent identifiers to the `Span` of the resource that defined them, used for better
-    // diagnostics.
-    let mut previous_defns = HashMap::new();
-
     // Set of Fluent attribute names already output, to avoid duplicate type errors - any given
     // constant created for a given attribute is the same.
     let mut previous_attrs = HashSet::new();
 
-    let mut includes = TokenStream::new();
-    let mut generated = TokenStream::new();
+    let resource_str = parse_macro_input!(input as LitStr);
+    let resource_span = resource_str.span().unwrap();
+    let relative_ftl_path = resource_str.value();
+    let absolute_ftl_path = invocation_relative_path_to_absolute(resource_span, &relative_ftl_path);
 
-    for res in resources.0 {
-        let krate_span = res.krate.span().unwrap();
-        let path_span = res.resource_path.span().unwrap();
+    let crate_name = Ident::new(&crate_name, resource_str.span());
 
-        let relative_ftl_path = res.resource_path.value();
-        let absolute_ftl_path =
-            invocation_relative_path_to_absolute(krate_span, &relative_ftl_path);
-        // As this macro also outputs an `include_str!` for this file, the macro will always be
-        // re-executed when the file changes.
-        let mut resource_file = match File::open(absolute_ftl_path) {
-            Ok(resource_file) => resource_file,
-            Err(e) => {
-                Diagnostic::spanned(path_span, Level::Error, "could not open Fluent resource")
-                    .note(e.to_string())
-                    .emit();
-                continue;
-            }
-        };
-        let mut resource_contents = String::new();
-        if let Err(e) = resource_file.read_to_string(&mut resource_contents) {
-            Diagnostic::spanned(path_span, Level::Error, "could not read Fluent resource")
+    // As this macro also outputs an `include_str!` for this file, the macro will always be
+    // re-executed when the file changes.
+    let mut resource_file = match File::open(absolute_ftl_path) {
+        Ok(resource_file) => resource_file,
+        Err(e) => {
+            Diagnostic::spanned(resource_span, Level::Error, "could not open Fluent resource")
                 .note(e.to_string())
                 .emit();
-            continue;
+            return failed(&crate_name);
         }
-        let resource = match FluentResource::try_new(resource_contents) {
-            Ok(resource) => resource,
-            Err((this, errs)) => {
-                Diagnostic::spanned(path_span, Level::Error, "could not parse Fluent resource")
-                    .help("see additional errors emitted")
-                    .emit();
-                for ParserError { pos, slice: _, kind } in errs {
-                    let mut err = kind.to_string();
-                    // Entirely unnecessary string modification so that the error message starts
-                    // with a lowercase as rustc errors do.
-                    err.replace_range(
-                        0..1,
-                        &err.chars().next().unwrap().to_lowercase().to_string(),
-                    );
+    };
+    let mut resource_contents = String::new();
+    if let Err(e) = resource_file.read_to_string(&mut resource_contents) {
+        Diagnostic::spanned(resource_span, Level::Error, "could not read Fluent resource")
+            .note(e.to_string())
+            .emit();
+        return failed(&crate_name);
+    }
+
+    let resource = match FluentResource::try_new(resource_contents) {
+        Ok(resource) => resource,
+        Err((this, errs)) => {
+            Diagnostic::spanned(resource_span, Level::Error, "could not parse Fluent resource")
+                .help("see additional errors emitted")
+                .emit();
+            for ParserError { pos, slice: _, kind } in errs {
+                let mut err = kind.to_string();
+                // Entirely unnecessary string modification so that the error message starts
+                // with a lowercase as rustc errors do.
+                err.replace_range(0..1, &err.chars().next().unwrap().to_lowercase().to_string());
 
-                    let line_starts: Vec<usize> = std::iter::once(0)
-                        .chain(
-                            this.source()
-                                .char_indices()
-                                .filter_map(|(i, c)| Some(i + 1).filter(|_| c == '\n')),
-                        )
-                        .collect();
-                    let line_start = line_starts
-                        .iter()
-                        .enumerate()
-                        .map(|(line, idx)| (line + 1, idx))
-                        .filter(|(_, idx)| **idx <= pos.start)
-                        .last()
-                        .unwrap()
-                        .0;
+                let line_starts: Vec<usize> = std::iter::once(0)
+                    .chain(
+                        this.source()
+                            .char_indices()
+                            .filter_map(|(i, c)| Some(i + 1).filter(|_| c == '\n')),
+                    )
+                    .collect();
+                let line_start = line_starts
+                    .iter()
+                    .enumerate()
+                    .map(|(line, idx)| (line + 1, idx))
+                    .filter(|(_, idx)| **idx <= pos.start)
+                    .last()
+                    .unwrap()
+                    .0;
 
-                    let snippet = Snippet {
-                        title: Some(Annotation {
-                            label: Some(&err),
-                            id: None,
+                let snippet = Snippet {
+                    title: Some(Annotation {
+                        label: Some(&err),
+                        id: None,
+                        annotation_type: AnnotationType::Error,
+                    }),
+                    footer: vec![],
+                    slices: vec![Slice {
+                        source: this.source(),
+                        line_start,
+                        origin: Some(&relative_ftl_path),
+                        fold: true,
+                        annotations: vec![SourceAnnotation {
+                            label: "",
                             annotation_type: AnnotationType::Error,
-                        }),
-                        footer: vec![],
-                        slices: vec![Slice {
-                            source: this.source(),
-                            line_start,
-                            origin: Some(&relative_ftl_path),
-                            fold: true,
-                            annotations: vec![SourceAnnotation {
-                                label: "",
-                                annotation_type: AnnotationType::Error,
-                                range: (pos.start, pos.end - 1),
-                            }],
+                            range: (pos.start, pos.end - 1),
                         }],
-                        opt: Default::default(),
-                    };
-                    let dl = DisplayList::from(snippet);
-                    eprintln!("{dl}\n");
+                    }],
+                    opt: Default::default(),
+                };
+                let dl = DisplayList::from(snippet);
+                eprintln!("{dl}\n");
+            }
+
+            return failed(&crate_name);
+        }
+    };
+
+    let mut constants = TokenStream::new();
+    let mut previous_defns = HashMap::new();
+    let mut message_refs = Vec::new();
+    for entry in resource.entries() {
+        if let Entry::Message(Message { id: Identifier { name }, attributes, value, .. }) = entry {
+            let _ = previous_defns.entry(name.to_string()).or_insert(resource_span);
+            if name.contains('-') {
+                Diagnostic::spanned(
+                    resource_span,
+                    Level::Error,
+                    format!("name `{name}` contains a '-' character"),
+                )
+                .help("replace any '-'s with '_'s")
+                .emit();
+            }
+
+            if let Some(Pattern { elements }) = value {
+                for elt in elements {
+                    if let PatternElement::Placeable {
+                        expression:
+                            Expression::Inline(InlineExpression::MessageReference { id, .. }),
+                    } = elt
+                    {
+                        message_refs.push((id.name, *name));
+                    }
                 }
+            }
+
+            // `typeck_foo_bar` => `foo_bar` (in `typeck.ftl`)
+            // `const_eval_baz` => `baz` (in `const_eval.ftl`)
+            // `const-eval-hyphen-having` => `hyphen_having` (in `const_eval.ftl`)
+            // The last case we error about above, but we want to fall back gracefully
+            // so that only the error is being emitted and not also one about the macro
+            // failing.
+            let crate_prefix = format!("{crate_name}_");
+
+            let snake_name = name.replace('-', "_");
+            if !snake_name.starts_with(&crate_prefix) {
+                Diagnostic::spanned(
+                    resource_span,
+                    Level::Error,
+                    format!("name `{name}` does not start with the crate name"),
+                )
+                .help(format!(
+                    "prepend `{crate_prefix}` to the slug name: `{crate_prefix}{snake_name}`"
+                ))
+                .emit();
+            };
+            let snake_name = Ident::new(&snake_name, resource_str.span());
+
+            if !previous_attrs.insert(snake_name.clone()) {
                 continue;
             }
-        };
 
-        let mut constants = TokenStream::new();
-        for entry in resource.entries() {
-            let span = res.krate.span();
-            if let Entry::Message(Message { id: Identifier { name }, attributes, .. }) = entry {
-                let _ = previous_defns.entry(name.to_string()).or_insert(path_span);
+            let msg = format!("Constant referring to Fluent message `{name}` from `{crate_name}`");
+            constants.extend(quote! {
+                #[doc = #msg]
+                pub const #snake_name: crate::DiagnosticMessage =
+                    crate::DiagnosticMessage::FluentIdentifier(
+                        std::borrow::Cow::Borrowed(#name),
+                        None
+                    );
+            });
 
-                if name.contains('-') {
-                    Diagnostic::spanned(
-                        path_span,
-                        Level::Error,
-                        format!("name `{name}` contains a '-' character"),
-                    )
-                    .help("replace any '-'s with '_'s")
-                    .emit();
+            for Attribute { id: Identifier { name: attr_name }, .. } in attributes {
+                let snake_name = Ident::new(
+                    &format!("{}{}", &crate_prefix, &attr_name.replace('-', "_")),
+                    resource_str.span(),
+                );
+                if !previous_attrs.insert(snake_name.clone()) {
+                    continue;
                 }
 
-                // Require that the message name starts with the crate name
-                // `hir_typeck_foo_bar` (in `hir_typeck.ftl`)
-                // `const_eval_baz` (in `const_eval.ftl`)
-                // `const-eval-hyphen-having` => `hyphen_having` (in `const_eval.ftl`)
-                // The last case we error about above, but we want to fall back gracefully
-                // so that only the error is being emitted and not also one about the macro
-                // failing.
-                let crate_prefix = format!("{}_", res.krate);
-
-                let snake_name = name.replace('-', "_");
-                if !snake_name.starts_with(&crate_prefix) {
+                if attr_name.contains('-') {
                     Diagnostic::spanned(
-                        path_span,
+                        resource_span,
                         Level::Error,
-                        format!("name `{name}` does not start with the crate name"),
+                        format!("attribute `{attr_name}` contains a '-' character"),
                     )
-                    .help(format!(
-                        "prepend `{crate_prefix}` to the slug name: `{crate_prefix}{snake_name}`"
-                    ))
+                    .help("replace any '-'s with '_'s")
                     .emit();
-                };
-
-                let snake_name = Ident::new(&snake_name, span);
+                }
 
+                let msg = format!(
+                    "Constant referring to Fluent message `{name}.{attr_name}` from `{crate_name}`"
+                );
                 constants.extend(quote! {
-                    pub const #snake_name: crate::DiagnosticMessage =
-                        crate::DiagnosticMessage::FluentIdentifier(
-                            std::borrow::Cow::Borrowed(#name),
-                            None
+                    #[doc = #msg]
+                    pub const #snake_name: crate::SubdiagnosticMessage =
+                        crate::SubdiagnosticMessage::FluentAttr(
+                            std::borrow::Cow::Borrowed(#attr_name)
                         );
                 });
-
-                for Attribute { id: Identifier { name: attr_name }, .. } in attributes {
-                    let snake_name = Ident::new(&attr_name.replace('-', "_"), span);
-                    if !previous_attrs.insert(snake_name.clone()) {
-                        continue;
-                    }
-
-                    if attr_name.contains('-') {
-                        Diagnostic::spanned(
-                            path_span,
-                            Level::Error,
-                            format!("attribute `{attr_name}` contains a '-' character"),
-                        )
-                        .help("replace any '-'s with '_'s")
-                        .emit();
-                    }
-
-                    constants.extend(quote! {
-                        pub const #snake_name: crate::SubdiagnosticMessage =
-                            crate::SubdiagnosticMessage::FluentAttr(
-                                std::borrow::Cow::Borrowed(#attr_name)
-                            );
-                    });
-                }
             }
         }
+    }
 
-        if let Err(errs) = bundle.add_resource(resource) {
-            for e in errs {
-                match e {
-                    FluentError::Overriding { kind, id } => {
-                        Diagnostic::spanned(
-                            path_span,
-                            Level::Error,
-                            format!("overrides existing {kind}: `{id}`"),
-                        )
-                        .span_help(previous_defns[&id], "previously defined in this resource")
-                        .emit();
-                    }
-                    FluentError::ResolverError(_) | FluentError::ParserError(_) => unreachable!(),
+    for (mref, name) in message_refs.into_iter() {
+        if !previous_defns.contains_key(mref) {
+            Diagnostic::spanned(
+                resource_span,
+                Level::Error,
+                format!("referenced message `{mref}` does not exist (in message `{name}`)"),
+            )
+            .help(&format!("you may have meant to use a variable reference (`{{${mref}}}`)"))
+            .emit();
+        }
+    }
+
+    if let Err(errs) = bundle.add_resource(resource) {
+        for e in errs {
+            match e {
+                FluentError::Overriding { kind, id } => {
+                    Diagnostic::spanned(
+                        resource_span,
+                        Level::Error,
+                        format!("overrides existing {kind}: `{id}`"),
+                    )
+                    .emit();
                 }
+                FluentError::ResolverError(_) | FluentError::ParserError(_) => unreachable!(),
             }
         }
-
-        includes.extend(quote! { include_str!(#relative_ftl_path), });
-
-        generated.extend(constants);
     }
 
     quote! {
+        /// Raw content of Fluent resource for this crate, generated by `fluent_messages` macro,
+        /// imported by `rustc_driver` to include all crates' resources in one bundle.
+        pub static DEFAULT_LOCALE_RESOURCE: &'static str = include_str!(#relative_ftl_path);
+
         #[allow(non_upper_case_globals)]
         #[doc(hidden)]
-        pub mod fluent_generated {
-            pub static DEFAULT_LOCALE_RESOURCES: &'static [&'static str] = &[
-                #includes
-            ];
-
-            #generated
+        /// Auto-generated constants for type-checked references to Fluent messages.
+        pub(crate) mod fluent_generated {
+            #constants
 
+            /// Constants expected to exist by the diagnostic derive macros to use as default Fluent
+            /// identifiers for different subdiagnostic kinds.
             pub mod _subdiag {
+                /// Default for `#[help]`
                 pub const help: crate::SubdiagnosticMessage =
                     crate::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed("help"));
+                /// Default for `#[note]`
                 pub const note: crate::SubdiagnosticMessage =
                     crate::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed("note"));
+                /// Default for `#[warn]`
                 pub const warn: crate::SubdiagnosticMessage =
                     crate::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed("warn"));
+                /// Default for `#[label]`
                 pub const label: crate::SubdiagnosticMessage =
                     crate::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed("label"));
+                /// Default for `#[suggestion]`
                 pub const suggestion: crate::SubdiagnosticMessage =
                     crate::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed("suggestion"));
             }
diff --git a/compiler/rustc_macros/src/diagnostics/subdiagnostic.rs b/compiler/rustc_macros/src/diagnostics/subdiagnostic.rs
index baffd3cec..90660fc1f 100644
--- a/compiler/rustc_macros/src/diagnostics/subdiagnostic.rs
+++ b/compiler/rustc_macros/src/diagnostics/subdiagnostic.rs
@@ -247,11 +247,7 @@ impl<'parent, 'a> SubdiagnosticDeriveVariantBuilder<'parent, 'a> {
                     return quote! {};
                 }
 
-                let info = FieldInfo {
-                    binding,
-                    ty: inner_ty.inner_type().unwrap_or(&ast.ty),
-                    span: &ast.span(),
-                };
+                let info = FieldInfo { binding, ty: inner_ty, span: &ast.span() };
 
                 let generated = self
                     .generate_field_code_inner(kind_stats, attr, info, inner_ty.will_iterate())
@@ -312,6 +308,21 @@ impl<'parent, 'a> SubdiagnosticDeriveVariantBuilder<'parent, 'a> {
                     let binding = info.binding.binding.clone();
                     // FIXME(#100717): support `Option<Span>` on `primary_span` like in the
                     // diagnostic derive
+                    if !matches!(info.ty, FieldInnerTy::Plain(_)) {
+                        throw_invalid_attr!(attr, &Meta::Path(path), |diag| {
+                            let diag = diag.note("there must be exactly one primary span");
+
+                            if kind_stats.has_normal_suggestion {
+                                diag.help(
+                                    "to create a suggestion with multiple spans, \
+                                     use `#[multipart_suggestion]` instead",
+                                )
+                            } else {
+                                diag
+                            }
+                        });
+                    }
+
                     self.span_field.set_once(binding, span);
                 }
 
@@ -501,7 +512,9 @@ impl<'parent, 'a> SubdiagnosticDeriveVariantBuilder<'parent, 'a> {
         let mut calls = TokenStream::new();
         for (kind, slug) in kind_slugs {
             let message = format_ident!("__message");
-            calls.extend(quote! { let #message = #f(#diag, rustc_errors::fluent::#slug.into()); });
+            calls.extend(
+                quote! { let #message = #f(#diag, crate::fluent_generated::#slug.into()); },
+            );
 
             let name = format_ident!("{}{}", if span_field.is_some() { "span_" } else { "" }, kind);
             let call = match kind {
diff --git a/compiler/rustc_macros/src/diagnostics/utils.rs b/compiler/rustc_macros/src/diagnostics/utils.rs
index 6f52a3de1..27b8f676f 100644
--- a/compiler/rustc_macros/src/diagnostics/utils.rs
+++ b/compiler/rustc_macros/src/diagnostics/utils.rs
@@ -80,7 +80,7 @@ fn report_error_if_not_applied_to_ty(
     path: &[&str],
     ty_name: &str,
 ) -> Result<(), DiagnosticDeriveError> {
-    if !type_matches_path(info.ty, path) {
+    if !type_matches_path(info.ty.inner_type(), path) {
         report_type_error(attr, ty_name)?;
     }
 
@@ -105,8 +105,8 @@ pub(crate) fn report_error_if_not_applied_to_span(
     attr: &Attribute,
     info: &FieldInfo<'_>,
 ) -> Result<(), DiagnosticDeriveError> {
-    if !type_matches_path(info.ty, &["rustc_span", "Span"])
-        && !type_matches_path(info.ty, &["rustc_errors", "MultiSpan"])
+    if !type_matches_path(info.ty.inner_type(), &["rustc_span", "Span"])
+        && !type_matches_path(info.ty.inner_type(), &["rustc_errors", "MultiSpan"])
     {
         report_type_error(attr, "`Span` or `MultiSpan`")?;
     }
@@ -115,44 +115,50 @@ pub(crate) fn report_error_if_not_applied_to_span(
 }
 
 /// Inner type of a field and type of wrapper.
+#[derive(Copy, Clone)]
 pub(crate) enum FieldInnerTy<'ty> {
     /// Field is wrapped in a `Option<$inner>`.
     Option(&'ty Type),
     /// Field is wrapped in a `Vec<$inner>`.
     Vec(&'ty Type),
     /// Field isn't wrapped in an outer type.
-    None,
+    Plain(&'ty Type),
 }
 
 impl<'ty> FieldInnerTy<'ty> {
     /// Returns inner type for a field, if there is one.
     ///
-    /// - If `ty` is an `Option`, returns `FieldInnerTy::Option { inner: (inner type) }`.
-    /// - If `ty` is a `Vec`, returns `FieldInnerTy::Vec { inner: (inner type) }`.
-    /// - Otherwise returns `None`.
+    /// - If `ty` is an `Option<Inner>`, returns `FieldInnerTy::Option(Inner)`.
+    /// - If `ty` is a `Vec<Inner>`, returns `FieldInnerTy::Vec(Inner)`.
+    /// - Otherwise returns `FieldInnerTy::Plain(ty)`.
     pub(crate) fn from_type(ty: &'ty Type) -> Self {
-        let variant: &dyn Fn(&'ty Type) -> FieldInnerTy<'ty> =
-            if type_matches_path(ty, &["std", "option", "Option"]) {
-                &FieldInnerTy::Option
-            } else if type_matches_path(ty, &["std", "vec", "Vec"]) {
-                &FieldInnerTy::Vec
-            } else {
-                return FieldInnerTy::None;
+        fn single_generic_type(ty: &Type) -> &Type {
+            let Type::Path(ty_path) = ty else {
+                panic!("expected path type");
             };
 
-        if let Type::Path(ty_path) = ty {
             let path = &ty_path.path;
             let ty = path.segments.iter().last().unwrap();
-            if let syn::PathArguments::AngleBracketed(bracketed) = &ty.arguments {
-                if bracketed.args.len() == 1 {
-                    if let syn::GenericArgument::Type(ty) = &bracketed.args[0] {
-                        return variant(ty);
-                    }
-                }
-            }
+            let syn::PathArguments::AngleBracketed(bracketed) = &ty.arguments else {
+                panic!("expected bracketed generic arguments");
+            };
+
+            assert_eq!(bracketed.args.len(), 1);
+
+            let syn::GenericArgument::Type(ty) = &bracketed.args[0] else {
+                panic!("expected generic parameter to be a type generic");
+            };
+
+            ty
         }
 
-        unreachable!();
+        if type_matches_path(ty, &["std", "option", "Option"]) {
+            FieldInnerTy::Option(single_generic_type(ty))
+        } else if type_matches_path(ty, &["std", "vec", "Vec"]) {
+            FieldInnerTy::Vec(single_generic_type(ty))
+        } else {
+            FieldInnerTy::Plain(ty)
+        }
     }
 
     /// Returns `true` if `FieldInnerTy::with` will result in iteration for this inner type (i.e.
@@ -160,15 +166,16 @@ impl<'ty> FieldInnerTy<'ty> {
     pub(crate) fn will_iterate(&self) -> bool {
         match self {
             FieldInnerTy::Vec(..) => true,
-            FieldInnerTy::Option(..) | FieldInnerTy::None => false,
+            FieldInnerTy::Option(..) | FieldInnerTy::Plain(_) => false,
         }
     }
 
-    /// Returns `Option` containing inner type if there is one.
-    pub(crate) fn inner_type(&self) -> Option<&'ty Type> {
+    /// Returns the inner type.
+    pub(crate) fn inner_type(&self) -> &'ty Type {
         match self {
-            FieldInnerTy::Option(inner) | FieldInnerTy::Vec(inner) => Some(inner),
-            FieldInnerTy::None => None,
+            FieldInnerTy::Option(inner) | FieldInnerTy::Vec(inner) | FieldInnerTy::Plain(inner) => {
+                inner
+            }
         }
     }
 
@@ -185,7 +192,7 @@ impl<'ty> FieldInnerTy<'ty> {
                     #inner
                 }
             },
-            FieldInnerTy::None => quote! { #inner },
+            FieldInnerTy::Plain(..) => quote! { #inner },
         }
     }
 }
@@ -194,7 +201,7 @@ impl<'ty> FieldInnerTy<'ty> {
 /// `generate_*` methods from walking the attributes themselves.
 pub(crate) struct FieldInfo<'a> {
     pub(crate) binding: &'a BindingInfo<'a>,
-    pub(crate) ty: &'a Type,
+    pub(crate) ty: FieldInnerTy<'a>,
     pub(crate) span: &'a proc_macro2::Span,
 }
 
diff --git a/compiler/rustc_macros/src/lib.rs b/compiler/rustc_macros/src/lib.rs
index bb3722fe1..737500cc2 100644
--- a/compiler/rustc_macros/src/lib.rs
+++ b/compiler/rustc_macros/src/lib.rs
@@ -61,9 +61,7 @@ pub fn newtype_index(input: TokenStream) -> TokenStream {
 /// For example, given the following invocation of the macro..
 ///
 /// ```ignore (rust)
-/// fluent_messages! {
-///     typeck => "./typeck.ftl",
-/// }
+/// fluent_messages! { "./typeck.ftl" }
 /// ```
 /// ..where `typeck.ftl` has the following contents..
 ///
@@ -77,9 +75,7 @@ pub fn newtype_index(input: TokenStream) -> TokenStream {
 /// will generate the following code:
 ///
 /// ```ignore (rust)
-/// pub static DEFAULT_LOCALE_RESOURCES: &'static [&'static str] = &[
-///     include_str!("./typeck.ftl"),
-/// ];
+/// pub static DEFAULT_LOCALE_RESOURCE: &'static [&'static str] = include_str!("./typeck.ftl");
 ///
 /// mod fluent_generated {
 ///     mod typeck {
@@ -124,8 +120,27 @@ decl_derive!([TyDecodable] => serialize::type_decodable_derive);
 decl_derive!([TyEncodable] => serialize::type_encodable_derive);
 decl_derive!([MetadataDecodable] => serialize::meta_decodable_derive);
 decl_derive!([MetadataEncodable] => serialize::meta_encodable_derive);
-decl_derive!([TypeFoldable, attributes(type_foldable)] => type_foldable::type_foldable_derive);
-decl_derive!([TypeVisitable, attributes(type_visitable)] => type_visitable::type_visitable_derive);
+decl_derive!(
+    [TypeFoldable, attributes(type_foldable)] =>
+    /// Derives `TypeFoldable` for the annotated `struct` or `enum` (`union` is not supported).
+    ///
+    /// The fold will produce a value of the same struct or enum variant as the input, with
+    /// each field respectively folded using the `TypeFoldable` implementation for its type.
+    /// However, if a field of a struct or an enum variant is annotated with
+    /// `#[type_foldable(identity)]` then that field will retain its incumbent value (and its
+    /// type is not required to implement `TypeFoldable`).
+    type_foldable::type_foldable_derive
+);
+decl_derive!(
+    [TypeVisitable, attributes(type_visitable)] =>
+    /// Derives `TypeVisitable` for the annotated `struct` or `enum` (`union` is not supported).
+    ///
+    /// Each field of the struct or enum variant will be visited in definition order, using the
+    /// `TypeVisitable` implementation for its type. However, if a field of a struct or an enum
+    /// variant is annotated with `#[type_visitable(ignore)]` then that field will not be
+    /// visited (and its type is not required to implement `TypeVisitable`).
+    type_visitable::type_visitable_derive
+);
 decl_derive!([Lift, attributes(lift)] => lift::lift_derive);
 decl_derive!(
     [Diagnostic, attributes(
diff --git a/compiler/rustc_macros/src/serialize.rs b/compiler/rustc_macros/src/serialize.rs
index 82e6972d0..8d017d149 100644
--- a/compiler/rustc_macros/src/serialize.rs
+++ b/compiler/rustc_macros/src/serialize.rs
@@ -42,6 +42,12 @@ fn decodable_body(
     }
     let ty_name = s.ast().ident.to_string();
     let decode_body = match s.variants() {
+        [] => {
+            let message = format!("`{}` has no variants to decode", ty_name);
+            quote! {
+                panic!(#message)
+            }
+        }
         [vi] => vi.construct(|field, _index| decode_field(field)),
         variants => {
             let match_inner: TokenStream = variants
@@ -139,6 +145,11 @@ fn encodable_body(
     });
 
     let encode_body = match s.variants() {
+        [] => {
+            quote! {
+                match *self {}
+            }
+        }
         [_] => {
             let encode_inner = s.each_variant(|vi| {
                 vi.bindings()
@@ -160,6 +171,23 @@ fn encodable_body(
             }
         }
         _ => {
+            let disc = {
+                let mut variant_idx = 0usize;
+                let encode_inner = s.each_variant(|_| {
+                    let result = quote! {
+                        #variant_idx
+                    };
+                    variant_idx += 1;
+                    result
+                });
+                quote! {
+                    let disc = match *self {
+                        #encode_inner
+                    };
+                    ::rustc_serialize::Encoder::emit_usize(__encoder, disc);
+                }
+            };
+
             let mut variant_idx = 0usize;
             let encode_inner = s.each_variant(|vi| {
                 let encode_fields: TokenStream = vi
@@ -176,26 +204,11 @@ fn encodable_body(
                         result
                     })
                     .collect();
-
-                let result = if !vi.bindings().is_empty() {
-                    quote! {
-                        ::rustc_serialize::Encoder::emit_enum_variant(
-                            __encoder,
-                            #variant_idx,
-                            |__encoder| { #encode_fields }
-                        )
-                    }
-                } else {
-                    quote! {
-                        ::rustc_serialize::Encoder::emit_fieldless_enum_variant::<#variant_idx>(
-                            __encoder,
-                        )
-                    }
-                };
                 variant_idx += 1;
-                result
+                encode_fields
             });
             quote! {
+                #disc
                 match *self {
                     #encode_inner
                 }
diff --git a/compiler/rustc_macros/src/type_foldable.rs b/compiler/rustc_macros/src/type_foldable.rs
index 23e619221..388e254cd 100644
--- a/compiler/rustc_macros/src/type_foldable.rs
+++ b/compiler/rustc_macros/src/type_foldable.rs
@@ -1,5 +1,5 @@
-use quote::quote;
-use syn::parse_quote;
+use quote::{quote, ToTokens};
+use syn::{parse_quote, Attribute, Meta, NestedMeta};
 
 pub fn type_foldable_derive(mut s: synstructure::Structure<'_>) -> proc_macro2::TokenStream {
     if let syn::Data::Union(_) = s.ast().data {
@@ -16,16 +16,37 @@ pub fn type_foldable_derive(mut s: synstructure::Structure<'_>) -> proc_macro2::
         let bindings = vi.bindings();
         vi.construct(|_, index| {
             let bind = &bindings[index];
-            quote! {
-                ::rustc_middle::ty::fold::TypeFoldable::try_fold_with(#bind, __folder)?
+
+            // retain value of fields with #[type_foldable(identity)]
+            let fixed = bind
+                .ast()
+                .attrs
+                .iter()
+                .map(Attribute::parse_meta)
+                .filter_map(Result::ok)
+                .flat_map(|attr| match attr {
+                    Meta::List(list) if list.path.is_ident("type_foldable") => list.nested,
+                    _ => Default::default(),
+                })
+                .any(|nested| match nested {
+                    NestedMeta::Meta(Meta::Path(path)) => path.is_ident("identity"),
+                    _ => false,
+                });
+
+            if fixed {
+                bind.to_token_stream()
+            } else {
+                quote! {
+                    ::rustc_middle::ty::fold::TypeFoldable::try_fold_with(#bind, __folder)?
+                }
             }
         })
     });
 
     s.bound_impl(
-        quote!(::rustc_middle::ty::fold::TypeFoldable<'tcx>),
+        quote!(::rustc_middle::ty::fold::TypeFoldable<::rustc_middle::ty::TyCtxt<'tcx>>),
         quote! {
-            fn try_fold_with<__F: ::rustc_middle::ty::fold::FallibleTypeFolder<'tcx>>(
+            fn try_fold_with<__F: ::rustc_middle::ty::fold::FallibleTypeFolder<::rustc_middle::ty::TyCtxt<'tcx>>>(
                 self,
                 __folder: &mut __F
             ) -> Result<Self, __F::Error> {
diff --git a/compiler/rustc_macros/src/type_visitable.rs b/compiler/rustc_macros/src/type_visitable.rs
index 1f95661ce..f6f4c4779 100644
--- a/compiler/rustc_macros/src/type_visitable.rs
+++ b/compiler/rustc_macros/src/type_visitable.rs
@@ -1,11 +1,28 @@
 use quote::quote;
-use syn::parse_quote;
+use syn::{parse_quote, Attribute, Meta, NestedMeta};
 
 pub fn type_visitable_derive(mut s: synstructure::Structure<'_>) -> proc_macro2::TokenStream {
     if let syn::Data::Union(_) = s.ast().data {
         panic!("cannot derive on union")
     }
 
+    // ignore fields with #[type_visitable(ignore)]
+    s.filter(|bi| {
+        !bi.ast()
+            .attrs
+            .iter()
+            .map(Attribute::parse_meta)
+            .filter_map(Result::ok)
+            .flat_map(|attr| match attr {
+                Meta::List(list) if list.path.is_ident("type_visitable") => list.nested,
+                _ => Default::default(),
+            })
+            .any(|nested| match nested {
+                NestedMeta::Meta(Meta::Path(path)) => path.is_ident("ignore"),
+                _ => false,
+            })
+    });
+
     if !s.ast().generics.lifetimes().any(|lt| lt.lifetime.ident == "tcx") {
         s.add_impl_generic(parse_quote! { 'tcx });
     }
@@ -19,9 +36,9 @@ pub fn type_visitable_derive(mut s: synstructure::Structure<'_>) -> proc_macro2:
     s.bind_with(|_| synstructure::BindStyle::Move);
 
     s.bound_impl(
-        quote!(::rustc_middle::ty::visit::TypeVisitable<'tcx>),
+        quote!(::rustc_middle::ty::visit::TypeVisitable<::rustc_middle::ty::TyCtxt<'tcx>>),
         quote! {
-            fn visit_with<__V: ::rustc_middle::ty::visit::TypeVisitor<'tcx>>(
+            fn visit_with<__V: ::rustc_middle::ty::visit::TypeVisitor<::rustc_middle::ty::TyCtxt<'tcx>>>(
                 &self,
                 __visitor: &mut __V
             ) -> ::std::ops::ControlFlow<__V::BreakTy> {
diff --git a/compiler/rustc_metadata/Cargo.toml b/compiler/rustc_metadata/Cargo.toml
index 6d85103c9..bee5c8541 100644
--- a/compiler/rustc_metadata/Cargo.toml
+++ b/compiler/rustc_metadata/Cargo.toml
@@ -6,6 +6,7 @@ edition = "2021"
 [lib]
 
 [dependencies]
+bitflags = "1.2.1"
 libloading = "0.7.1"
 odht = { version = "0.3.1", features = ["nightly"] }
 snap = "1"
diff --git a/compiler/rustc_error_messages/locales/en-US/metadata.ftl b/compiler/rustc_metadata/locales/en-US.ftl
index 79b8b4172..79b8b4172 100644
--- a/compiler/rustc_error_messages/locales/en-US/metadata.ftl
+++ b/compiler/rustc_metadata/locales/en-US.ftl
diff --git a/compiler/rustc_metadata/src/creader.rs b/compiler/rustc_metadata/src/creader.rs
index 653f2b39d..b05626311 100644
--- a/compiler/rustc_metadata/src/creader.rs
+++ b/compiler/rustc_metadata/src/creader.rs
@@ -1,10 +1,6 @@
 //! Validates all used crates and extern libraries and loads their metadata
 
-use crate::errors::{
-    ConflictingAllocErrorHandler, ConflictingGlobalAlloc, CrateNotPanicRuntime,
-    GlobalAllocRequired, NoMultipleAllocErrorHandler, NoMultipleGlobalAlloc, NoPanicStrategy,
-    NoTransitiveNeedsDep, NotProfilerRuntime, ProfilerBuiltinsNeedsCore,
-};
+use crate::errors;
 use crate::locator::{CrateError, CrateLocator, CratePaths};
 use crate::rmeta::{CrateDep, CrateMetadata, CrateNumMap, CrateRoot, MetadataBlob};
 
@@ -12,15 +8,15 @@ use rustc_ast::expand::allocator::AllocatorKind;
 use rustc_ast::{self as ast, *};
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::svh::Svh;
-use rustc_data_structures::sync::{Lrc, ReadGuard};
+use rustc_data_structures::sync::MappedReadGuard;
 use rustc_expand::base::SyntaxExtension;
 use rustc_hir::def_id::{CrateNum, LocalDefId, StableCrateId, LOCAL_CRATE};
 use rustc_hir::definitions::Definitions;
 use rustc_index::vec::IndexVec;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::config::{self, CrateType, ExternLocation};
+use rustc_session::cstore::ExternCrateSource;
 use rustc_session::cstore::{CrateDepKind, CrateSource, ExternCrate};
-use rustc_session::cstore::{ExternCrateSource, MetadataLoaderDyn};
 use rustc_session::lint;
 use rustc_session::output::validate_crate_name;
 use rustc_session::search_paths::PathKind;
@@ -33,11 +29,11 @@ use rustc_target::spec::{PanicStrategy, TargetTriple};
 use proc_macro::bridge::client::ProcMacro;
 use std::ops::Fn;
 use std::path::Path;
-use std::{cmp, env};
+use std::time::Duration;
+use std::{cmp, env, iter};
 
-#[derive(Clone)]
 pub struct CStore {
-    metas: IndexVec<CrateNum, Option<Lrc<CrateMetadata>>>,
+    metas: IndexVec<CrateNum, Option<Box<CrateMetadata>>>,
     injected_panic_runtime: Option<CrateNum>,
     /// This crate needs an allocator and either provides it itself, or finds it in a dependency.
     /// If the above is true, then this field denotes the kind of the found allocator.
@@ -64,17 +60,22 @@ impl std::fmt::Debug for CStore {
     }
 }
 
-pub struct CrateLoader<'a> {
+pub struct CrateLoader<'a, 'tcx: 'a> {
     // Immutable configuration.
-    sess: &'a Session,
-    metadata_loader: &'a MetadataLoaderDyn,
-    definitions: ReadGuard<'a, Definitions>,
-    local_crate_name: Symbol,
+    tcx: TyCtxt<'tcx>,
     // Mutable output.
     cstore: &'a mut CStore,
     used_extern_options: &'a mut FxHashSet<Symbol>,
 }
 
+impl<'a, 'tcx> std::ops::Deref for CrateLoader<'a, 'tcx> {
+    type Target = TyCtxt<'tcx>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.tcx
+    }
+}
+
 pub enum LoadedMacro {
     MacroDef(ast::Item, Edition),
     ProcMacro(SyntaxExtension),
@@ -131,11 +132,10 @@ impl<'a> std::fmt::Debug for CrateDump<'a> {
 }
 
 impl CStore {
-    pub fn from_tcx(tcx: TyCtxt<'_>) -> &CStore {
-        tcx.cstore_untracked()
-            .as_any()
-            .downcast_ref::<CStore>()
-            .expect("`tcx.cstore` is not a `CStore`")
+    pub fn from_tcx(tcx: TyCtxt<'_>) -> MappedReadGuard<'_, CStore> {
+        MappedReadGuard::map(tcx.cstore_untracked(), |c| {
+            c.as_any().downcast_ref::<CStore>().expect("`tcx.cstore` is not a `CStore`")
+        })
     }
 
     fn alloc_new_crate_num(&mut self) -> CrateNum {
@@ -156,7 +156,7 @@ impl CStore {
 
     fn set_crate_data(&mut self, cnum: CrateNum, data: CrateMetadata) {
         assert!(self.metas[cnum].is_none(), "Overwriting crate metadata entry");
-        self.metas[cnum] = Some(Lrc::new(data));
+        self.metas[cnum] = Some(Box::new(data));
     }
 
     pub(crate) fn iter_crate_data(&self) -> impl Iterator<Item = (CrateNum, &CrateMetadata)> {
@@ -248,7 +248,7 @@ impl CStore {
             // order to make array indices in `metas` match with the
             // corresponding `CrateNum`. This first entry will always remain
             // `None`.
-            metas: IndexVec::from_elem_n(None, 1),
+            metas: IndexVec::from_iter(iter::once(None)),
             injected_panic_runtime: None,
             allocator_kind: None,
             alloc_error_handler_kind: None,
@@ -260,23 +260,13 @@ impl CStore {
     }
 }
 
-impl<'a> CrateLoader<'a> {
+impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
     pub fn new(
-        sess: &'a Session,
-        metadata_loader: &'a MetadataLoaderDyn,
-        local_crate_name: Symbol,
+        tcx: TyCtxt<'tcx>,
         cstore: &'a mut CStore,
-        definitions: ReadGuard<'a, Definitions>,
         used_extern_options: &'a mut FxHashSet<Symbol>,
     ) -> Self {
-        CrateLoader {
-            sess,
-            metadata_loader,
-            local_crate_name,
-            cstore,
-            used_extern_options,
-            definitions,
-        }
+        CrateLoader { tcx, cstore, used_extern_options }
     }
     pub fn cstore(&self) -> &CStore {
         &self.cstore
@@ -359,7 +349,12 @@ impl<'a> CrateLoader<'a> {
         for (_, other) in self.cstore.iter_crate_data() {
             // Same stable crate id but different SVH
             if other.stable_crate_id() == root.stable_crate_id() && other.hash() != root.hash() {
-                return Err(CrateError::SymbolConflictsOthers(root.name()));
+                bug!(
+                    "Previously returned E0523 here. \
+                     See https://github.com/rust-lang/rust/pull/100599 for additional discussion.\
+                     root.name() = {}.",
+                    root.name()
+                );
             }
         }
 
@@ -562,9 +557,10 @@ impl<'a> CrateLoader<'a> {
             (LoadResult::Previous(cnum), None)
         } else {
             info!("falling back to a load");
+            let metadata_loader = self.tcx.metadata_loader(()).borrow();
             let mut locator = CrateLocator::new(
                 self.sess,
-                &*self.metadata_loader,
+                &**metadata_loader,
                 name,
                 hash,
                 extra_filename,
@@ -689,8 +685,7 @@ impl<'a> CrateLoader<'a> {
     ) -> Result<&'static [ProcMacro], CrateError> {
         // Make sure the path contains a / or the linker will search for it.
         let path = env::current_dir().unwrap().join(path);
-        let lib = unsafe { libloading::Library::new(path) }
-            .map_err(|err| CrateError::DlOpen(err.to_string()))?;
+        let lib = load_dylib(&path, 5).map_err(|err| CrateError::DlOpen(err))?;
 
         let sym_name = self.sess.generate_proc_macro_decls_symbol(stable_crate_id);
         let sym = unsafe { lib.get::<*const &[ProcMacro]>(sym_name.as_bytes()) }
@@ -768,10 +763,11 @@ impl<'a> CrateLoader<'a> {
         // Sanity check the loaded crate to ensure it is indeed a panic runtime
         // and the panic strategy is indeed what we thought it was.
         if !data.is_panic_runtime() {
-            self.sess.emit_err(CrateNotPanicRuntime { crate_name: name });
+            self.sess.emit_err(errors::CrateNotPanicRuntime { crate_name: name });
         }
         if data.required_panic_strategy() != Some(desired_strategy) {
-            self.sess.emit_err(NoPanicStrategy { crate_name: name, strategy: desired_strategy });
+            self.sess
+                .emit_err(errors::NoPanicStrategy { crate_name: name, strategy: desired_strategy });
         }
 
         self.cstore.injected_panic_runtime = Some(cnum);
@@ -791,7 +787,7 @@ impl<'a> CrateLoader<'a> {
 
         let name = Symbol::intern(&self.sess.opts.unstable_opts.profiler_runtime);
         if name == sym::profiler_builtins && self.sess.contains_name(&krate.attrs, sym::no_core) {
-            self.sess.emit_err(ProfilerBuiltinsNeedsCore);
+            self.sess.emit_err(errors::ProfilerBuiltinsNeedsCore);
         }
 
         let Some(cnum) = self.resolve_crate(name, DUMMY_SP, CrateDepKind::Implicit) else { return; };
@@ -799,21 +795,22 @@ impl<'a> CrateLoader<'a> {
 
         // Sanity check the loaded crate to ensure it is indeed a profiler runtime
         if !data.is_profiler_runtime() {
-            self.sess.emit_err(NotProfilerRuntime { crate_name: name });
+            self.sess.emit_err(errors::NotProfilerRuntime { crate_name: name });
         }
     }
 
     fn inject_allocator_crate(&mut self, krate: &ast::Crate) {
         self.cstore.has_global_allocator = match &*global_allocator_spans(&self.sess, krate) {
             [span1, span2, ..] => {
-                self.sess.emit_err(NoMultipleGlobalAlloc { span2: *span2, span1: *span1 });
+                self.sess.emit_err(errors::NoMultipleGlobalAlloc { span2: *span2, span1: *span1 });
                 true
             }
             spans => !spans.is_empty(),
         };
         self.cstore.has_alloc_error_handler = match &*alloc_error_handler_spans(&self.sess, krate) {
             [span1, span2, ..] => {
-                self.sess.emit_err(NoMultipleAllocErrorHandler { span2: *span2, span1: *span1 });
+                self.sess
+                    .emit_err(errors::NoMultipleAllocErrorHandler { span2: *span2, span1: *span1 });
                 true
             }
             spans => !spans.is_empty(),
@@ -849,7 +846,7 @@ impl<'a> CrateLoader<'a> {
             if data.has_global_allocator() {
                 match global_allocator {
                     Some(other_crate) => {
-                        self.sess.emit_err(ConflictingGlobalAlloc {
+                        self.sess.emit_err(errors::ConflictingGlobalAlloc {
                             crate_name: data.name(),
                             other_crate_name: other_crate,
                         });
@@ -864,7 +861,7 @@ impl<'a> CrateLoader<'a> {
             if data.has_alloc_error_handler() {
                 match alloc_error_handler {
                     Some(other_crate) => {
-                        self.sess.emit_err(ConflictingAllocErrorHandler {
+                        self.sess.emit_err(errors::ConflictingAllocErrorHandler {
                             crate_name: data.name(),
                             other_crate_name: other_crate,
                         });
@@ -884,7 +881,7 @@ impl<'a> CrateLoader<'a> {
             if !self.sess.contains_name(&krate.attrs, sym::default_lib_allocator)
                 && !self.cstore.iter_crate_data().any(|(_, data)| data.has_default_lib_allocator())
             {
-                self.sess.emit_err(GlobalAllocRequired);
+                self.sess.emit_err(errors::GlobalAllocRequired);
             }
             self.cstore.allocator_kind = Some(AllocatorKind::Default);
         }
@@ -917,7 +914,7 @@ impl<'a> CrateLoader<'a> {
         for dep in self.cstore.crate_dependencies_in_reverse_postorder(krate) {
             let data = self.cstore.get_crate_data(dep);
             if needs_dep(&data) {
-                self.sess.emit_err(NoTransitiveNeedsDep {
+                self.sess.emit_err(errors::NoTransitiveNeedsDep {
                     crate_name: self.cstore.get_crate_data(krate).name(),
                     needs_crate_name: what,
                     deps_crate_name: data.name(),
@@ -968,7 +965,7 @@ impl<'a> CrateLoader<'a> {
                     &format!(
                         "external crate `{}` unused in `{}`: remove the dependency or add `use {} as _;`",
                         name,
-                        self.local_crate_name,
+                        self.tcx.crate_name(LOCAL_CRATE),
                         name),
                 );
         }
@@ -988,6 +985,7 @@ impl<'a> CrateLoader<'a> {
         &mut self,
         item: &ast::Item,
         def_id: LocalDefId,
+        definitions: &Definitions,
     ) -> Option<CrateNum> {
         match item.kind {
             ast::ItemKind::ExternCrate(orig_name) => {
@@ -1010,7 +1008,7 @@ impl<'a> CrateLoader<'a> {
 
                 let cnum = self.resolve_crate(name, item.span, dep_kind)?;
 
-                let path_len = self.definitions.def_path(def_id).data.len();
+                let path_len = definitions.def_path(def_id).data.len();
                 self.update_extern_crate(
                     cnum,
                     ExternCrate {
@@ -1093,3 +1091,41 @@ fn alloc_error_handler_spans(sess: &Session, krate: &ast::Crate) -> Vec<Span> {
     visit::walk_crate(&mut f, krate);
     f.spans
 }
+
+// On Windows the compiler would sometimes intermittently fail to open the
+// proc-macro DLL with `Error::LoadLibraryExW`. It is suspected that something in the
+// system still holds a lock on the file, so we retry a few times before calling it
+// an error.
+fn load_dylib(path: &Path, max_attempts: usize) -> Result<libloading::Library, String> {
+    assert!(max_attempts > 0);
+
+    let mut last_error = None;
+
+    for attempt in 0..max_attempts {
+        match unsafe { libloading::Library::new(&path) } {
+            Ok(lib) => {
+                if attempt > 0 {
+                    debug!(
+                        "Loaded proc-macro `{}` after {} attempts.",
+                        path.display(),
+                        attempt + 1
+                    );
+                }
+                return Ok(lib);
+            }
+            Err(err) => {
+                // Only try to recover from this specific error.
+                if !matches!(err, libloading::Error::LoadLibraryExW { .. }) {
+                    return Err(err.to_string());
+                }
+
+                last_error = Some(err);
+                std::thread::sleep(Duration::from_millis(100));
+                debug!("Failed to load proc-macro `{}`. Retrying.", path.display());
+            }
+        }
+    }
+
+    debug!("Failed to load proc-macro `{}` even after {} attempts.", path.display(), max_attempts);
+    Err(format!("{} (retried {} times)", last_error.unwrap(), max_attempts))
+}
diff --git a/compiler/rustc_metadata/src/dependency_format.rs b/compiler/rustc_metadata/src/dependency_format.rs
index cee4ba56a..39ef4276f 100644
--- a/compiler/rustc_metadata/src/dependency_format.rs
+++ b/compiler/rustc_metadata/src/dependency_format.rs
@@ -113,37 +113,37 @@ fn calculate_type(tcx: TyCtxt<'_>, ty: CrateType) -> DependencyList {
         CrateType::Staticlib => Linkage::Static,
     };
 
-    if preferred_linkage == Linkage::NotLinked {
+    match preferred_linkage {
         // If the crate is not linked, there are no link-time dependencies.
-        return Vec::new();
-    }
-
-    if preferred_linkage == Linkage::Static {
-        // Attempt static linkage first. For dylibs and executables, we may be
-        // able to retry below with dynamic linkage.
-        if let Some(v) = attempt_static(tcx) {
-            return v;
-        }
+        Linkage::NotLinked => return Vec::new(),
+        Linkage::Static => {
+            // Attempt static linkage first. For dylibs and executables, we may be
+            // able to retry below with dynamic linkage.
+            if let Some(v) = attempt_static(tcx) {
+                return v;
+            }
 
-        // Staticlibs and static executables must have all static dependencies.
-        // If any are not found, generate some nice pretty errors.
-        if ty == CrateType::Staticlib
-            || (ty == CrateType::Executable
-                && sess.crt_static(Some(ty))
-                && !sess.target.crt_static_allows_dylibs)
-        {
-            for &cnum in tcx.crates(()).iter() {
-                if tcx.dep_kind(cnum).macros_only() {
-                    continue;
+            // Staticlibs and static executables must have all static dependencies.
+            // If any are not found, generate some nice pretty errors.
+            if ty == CrateType::Staticlib
+                || (ty == CrateType::Executable
+                    && sess.crt_static(Some(ty))
+                    && !sess.target.crt_static_allows_dylibs)
+            {
+                for &cnum in tcx.crates(()).iter() {
+                    if tcx.dep_kind(cnum).macros_only() {
+                        continue;
+                    }
+                    let src = tcx.used_crate_source(cnum);
+                    if src.rlib.is_some() {
+                        continue;
+                    }
+                    sess.emit_err(RlibRequired { crate_name: tcx.crate_name(cnum) });
                 }
-                let src = tcx.used_crate_source(cnum);
-                if src.rlib.is_some() {
-                    continue;
-                }
-                sess.emit_err(RlibRequired { crate_name: tcx.crate_name(cnum) });
+                return Vec::new();
             }
-            return Vec::new();
         }
+        Linkage::Dynamic | Linkage::IncludedFromDylib => {}
     }
 
     let mut formats = FxHashMap::default();
@@ -283,12 +283,9 @@ fn attempt_static(tcx: TyCtxt<'_>) -> Option<DependencyList> {
     let mut ret = tcx
         .crates(())
         .iter()
-        .map(|&cnum| {
-            if tcx.dep_kind(cnum) == CrateDepKind::Explicit {
-                Linkage::Static
-            } else {
-                Linkage::NotLinked
-            }
+        .map(|&cnum| match tcx.dep_kind(cnum) {
+            CrateDepKind::Explicit => Linkage::Static,
+            CrateDepKind::MacrosOnly | CrateDepKind::Implicit => Linkage::NotLinked,
         })
         .collect::<Vec<_>>();
 
diff --git a/compiler/rustc_metadata/src/errors.rs b/compiler/rustc_metadata/src/errors.rs
index 02c03114e..51b41b5f6 100644
--- a/compiler/rustc_metadata/src/errors.rs
+++ b/compiler/rustc_metadata/src/errors.rs
@@ -9,6 +9,7 @@ use rustc_session::config;
 use rustc_span::{sym, Span, Symbol};
 use rustc_target::spec::{PanicStrategy, TargetTriple};
 
+use crate::fluent_generated as fluent;
 use crate::locator::CrateFlavor;
 
 #[derive(Diagnostic)]
@@ -491,7 +492,7 @@ impl IntoDiagnostic<'_> for MultipleCandidates {
         self,
         handler: &'_ rustc_errors::Handler,
     ) -> rustc_errors::DiagnosticBuilder<'_, ErrorGuaranteed> {
-        let mut diag = handler.struct_err(rustc_errors::fluent::metadata_multiple_candidates);
+        let mut diag = handler.struct_err(fluent::metadata_multiple_candidates);
         diag.set_arg("crate_name", self.crate_name);
         diag.set_arg("flavor", self.flavor);
         diag.code(error_code!(E0464));
@@ -512,14 +513,6 @@ pub struct SymbolConflictsCurrent {
 }
 
 #[derive(Diagnostic)]
-#[diag(metadata_symbol_conflicts_others, code = "E0523")]
-pub struct SymbolConflictsOthers {
-    #[primary_span]
-    pub span: Span,
-    pub crate_name: Symbol,
-}
-
-#[derive(Diagnostic)]
 #[diag(metadata_stable_crate_id_collision)]
 pub struct StableCrateIdCollision {
     #[primary_span]
@@ -598,7 +591,7 @@ impl IntoDiagnostic<'_> for InvalidMetadataFiles {
         self,
         handler: &'_ rustc_errors::Handler,
     ) -> rustc_errors::DiagnosticBuilder<'_, ErrorGuaranteed> {
-        let mut diag = handler.struct_err(rustc_errors::fluent::metadata_invalid_meta_files);
+        let mut diag = handler.struct_err(fluent::metadata_invalid_meta_files);
         diag.set_arg("crate_name", self.crate_name);
         diag.set_arg("add_info", self.add_info);
         diag.code(error_code!(E0786));
@@ -627,7 +620,7 @@ impl IntoDiagnostic<'_> for CannotFindCrate {
         self,
         handler: &'_ rustc_errors::Handler,
     ) -> rustc_errors::DiagnosticBuilder<'_, ErrorGuaranteed> {
-        let mut diag = handler.struct_err(rustc_errors::fluent::metadata_cannot_find_crate);
+        let mut diag = handler.struct_err(fluent::metadata_cannot_find_crate);
         diag.set_arg("crate_name", self.crate_name);
         diag.set_arg("current_crate", self.current_crate);
         diag.set_arg("add_info", self.add_info);
@@ -638,32 +631,32 @@ impl IntoDiagnostic<'_> for CannotFindCrate {
             && self.locator_triple != TargetTriple::from_triple(config::host_triple())
         {
             if self.missing_core {
-                diag.note(rustc_errors::fluent::metadata_target_not_installed);
+                diag.note(fluent::metadata_target_not_installed);
             } else {
-                diag.note(rustc_errors::fluent::metadata_target_no_std_support);
+                diag.note(fluent::metadata_target_no_std_support);
             }
             // NOTE: this suggests using rustup, even though the user may not have it installed.
             // That's because they could choose to install it; or this may give them a hint which
             // target they need to install from their distro.
             if self.missing_core {
-                diag.help(rustc_errors::fluent::metadata_consider_downloading_target);
+                diag.help(fluent::metadata_consider_downloading_target);
             }
             // Suggest using #![no_std]. #[no_core] is unstable and not really supported anyway.
             // NOTE: this is a dummy span if `extern crate std` was injected by the compiler.
             // If it's not a dummy, that means someone added `extern crate std` explicitly and
             // `#![no_std]` won't help.
             if !self.missing_core && self.span.is_dummy() {
-                diag.note(rustc_errors::fluent::metadata_std_required);
+                diag.note(fluent::metadata_std_required);
             }
             if self.is_nightly_build {
-                diag.help(rustc_errors::fluent::metadata_consider_building_std);
+                diag.help(fluent::metadata_consider_building_std);
             }
         } else if self.crate_name == self.profiler_runtime {
-            diag.note(rustc_errors::fluent::metadata_compiler_missing_profiler);
+            diag.note(fluent::metadata_compiler_missing_profiler);
         } else if self.crate_name.as_str().starts_with("rustc_") {
-            diag.help(rustc_errors::fluent::metadata_install_missing_components);
+            diag.help(fluent::metadata_install_missing_components);
         }
-        diag.span_label(self.span, rustc_errors::fluent::metadata_cant_find_crate);
+        diag.span_label(self.span, fluent::metadata_cant_find_crate);
         diag
     }
 }
diff --git a/compiler/rustc_metadata/src/lib.rs b/compiler/rustc_metadata/src/lib.rs
index 1987f88e6..6f6d3731c 100644
--- a/compiler/rustc_metadata/src/lib.rs
+++ b/compiler/rustc_metadata/src/lib.rs
@@ -30,6 +30,8 @@ extern crate rustc_data_structures;
 extern crate tracing;
 
 pub use rmeta::{provide, provide_extern};
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
+use rustc_macros::fluent_messages;
 
 mod dependency_format;
 mod foreign_modules;
@@ -44,3 +46,5 @@ pub mod locator;
 pub use fs::{emit_wrapper_file, METADATA_FILENAME};
 pub use native_libs::find_native_static_library;
 pub use rmeta::{encode_metadata, EncodedMetadata, METADATA_HEADER};
+
+fluent_messages! { "../locales/en-US.ftl" }
diff --git a/compiler/rustc_metadata/src/locator.rs b/compiler/rustc_metadata/src/locator.rs
index 0f5f74007..755a24253 100644
--- a/compiler/rustc_metadata/src/locator.rs
+++ b/compiler/rustc_metadata/src/locator.rs
@@ -213,12 +213,7 @@
 //! metadata::locator or metadata::creader for all the juicy details!
 
 use crate::creader::Library;
-use crate::errors::{
-    CannotFindCrate, CrateLocationUnknownType, DlError, ExternLocationNotExist,
-    ExternLocationNotFile, FoundStaticlib, IncompatibleRustc, InvalidMetadataFiles,
-    LibFilenameForm, MultipleCandidates, NewerCrateVersion, NoCrateWithTriple, NoDylibPlugin,
-    NonAsciiName, StableCrateIdCollision, SymbolConflictsCurrent, SymbolConflictsOthers,
-};
+use crate::errors;
 use crate::rmeta::{rustc_version, MetadataBlob, METADATA_HEADER};
 
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
@@ -950,7 +945,6 @@ pub(crate) enum CrateError {
     ExternLocationNotFile(Symbol, PathBuf),
     MultipleCandidates(Symbol, CrateFlavor, Vec<PathBuf>),
     SymbolConflictsCurrent(Symbol),
-    SymbolConflictsOthers(Symbol),
     StableCrateIdCollision(Symbol, Symbol),
     DlOpen(String),
     DlSym(String),
@@ -980,28 +974,25 @@ impl CrateError {
     pub(crate) fn report(self, sess: &Session, span: Span, missing_core: bool) {
         match self {
             CrateError::NonAsciiName(crate_name) => {
-                sess.emit_err(NonAsciiName { span, crate_name });
+                sess.emit_err(errors::NonAsciiName { span, crate_name });
             }
             CrateError::ExternLocationNotExist(crate_name, loc) => {
-                sess.emit_err(ExternLocationNotExist { span, crate_name, location: &loc });
+                sess.emit_err(errors::ExternLocationNotExist { span, crate_name, location: &loc });
             }
             CrateError::ExternLocationNotFile(crate_name, loc) => {
-                sess.emit_err(ExternLocationNotFile { span, crate_name, location: &loc });
+                sess.emit_err(errors::ExternLocationNotFile { span, crate_name, location: &loc });
             }
             CrateError::MultipleCandidates(crate_name, flavor, candidates) => {
-                sess.emit_err(MultipleCandidates { span, crate_name, flavor, candidates });
+                sess.emit_err(errors::MultipleCandidates { span, crate_name, flavor, candidates });
             }
             CrateError::SymbolConflictsCurrent(root_name) => {
-                sess.emit_err(SymbolConflictsCurrent { span, crate_name: root_name });
-            }
-            CrateError::SymbolConflictsOthers(root_name) => {
-                sess.emit_err(SymbolConflictsOthers { span, crate_name: root_name });
+                sess.emit_err(errors::SymbolConflictsCurrent { span, crate_name: root_name });
             }
             CrateError::StableCrateIdCollision(crate_name0, crate_name1) => {
-                sess.emit_err(StableCrateIdCollision { span, crate_name0, crate_name1 });
+                sess.emit_err(errors::StableCrateIdCollision { span, crate_name0, crate_name1 });
             }
             CrateError::DlOpen(s) | CrateError::DlSym(s) => {
-                sess.emit_err(DlError { span, err: s });
+                sess.emit_err(errors::DlError { span, err: s });
             }
             CrateError::LocatorCombined(locator) => {
                 let crate_name = locator.crate_name;
@@ -1012,8 +1003,12 @@ impl CrateError {
                 if !locator.crate_rejections.via_filename.is_empty() {
                     let mismatches = locator.crate_rejections.via_filename.iter();
                     for CrateMismatch { path, .. } in mismatches {
-                        sess.emit_err(CrateLocationUnknownType { span, path: &path, crate_name });
-                        sess.emit_err(LibFilenameForm {
+                        sess.emit_err(errors::CrateLocationUnknownType {
+                            span,
+                            path: &path,
+                            crate_name,
+                        });
+                        sess.emit_err(errors::LibFilenameForm {
                             span,
                             dll_prefix: &locator.dll_prefix,
                             dll_suffix: &locator.dll_suffix,
@@ -1039,7 +1034,7 @@ impl CrateError {
                             ));
                         }
                     }
-                    sess.emit_err(NewerCrateVersion {
+                    sess.emit_err(errors::NewerCrateVersion {
                         span,
                         crate_name: crate_name,
                         add_info,
@@ -1055,7 +1050,7 @@ impl CrateError {
                             path.display(),
                         ));
                     }
-                    sess.emit_err(NoCrateWithTriple {
+                    sess.emit_err(errors::NoCrateWithTriple {
                         span,
                         crate_name,
                         locator_triple: locator.triple.triple(),
@@ -1071,7 +1066,12 @@ impl CrateError {
                             path.display()
                         ));
                     }
-                    sess.emit_err(FoundStaticlib { span, crate_name, add_info, found_crates });
+                    sess.emit_err(errors::FoundStaticlib {
+                        span,
+                        crate_name,
+                        add_info,
+                        found_crates,
+                    });
                 } else if !locator.crate_rejections.via_version.is_empty() {
                     let mismatches = locator.crate_rejections.via_version.iter();
                     for CrateMismatch { path, got } in mismatches {
@@ -1082,7 +1082,7 @@ impl CrateError {
                             path.display(),
                         ));
                     }
-                    sess.emit_err(IncompatibleRustc {
+                    sess.emit_err(errors::IncompatibleRustc {
                         span,
                         crate_name,
                         add_info,
@@ -1094,14 +1094,14 @@ impl CrateError {
                     for CrateMismatch { path: _, got } in locator.crate_rejections.via_invalid {
                         crate_rejections.push(got);
                     }
-                    sess.emit_err(InvalidMetadataFiles {
+                    sess.emit_err(errors::InvalidMetadataFiles {
                         span,
                         crate_name,
                         add_info,
                         crate_rejections,
                     });
                 } else {
-                    sess.emit_err(CannotFindCrate {
+                    sess.emit_err(errors::CannotFindCrate {
                         span,
                         crate_name,
                         add_info,
@@ -1118,7 +1118,7 @@ impl CrateError {
                 }
             }
             CrateError::NonDylibPlugin(crate_name) => {
-                sess.emit_err(NoDylibPlugin { span, crate_name });
+                sess.emit_err(errors::NoDylibPlugin { span, crate_name });
             }
         }
     }
diff --git a/compiler/rustc_metadata/src/native_libs.rs b/compiler/rustc_metadata/src/native_libs.rs
index 6f05c76e8..d6f68b2e1 100644
--- a/compiler/rustc_metadata/src/native_libs.rs
+++ b/compiler/rustc_metadata/src/native_libs.rs
@@ -13,17 +13,7 @@ use rustc_session::Session;
 use rustc_span::symbol::{sym, Symbol};
 use rustc_target::spec::abi::Abi;
 
-use crate::errors::{
-    AsNeededCompatibility, BundleNeedsStatic, EmptyLinkName, EmptyRenamingTarget,
-    FrameworkOnlyWindows, ImportNameTypeForm, ImportNameTypeRaw, ImportNameTypeX86,
-    IncompatibleWasmLink, InvalidLinkModifier, LibFrameworkApple, LinkCfgForm,
-    LinkCfgSinglePredicate, LinkFrameworkApple, LinkKindForm, LinkModifiersForm, LinkNameForm,
-    LinkOrdinalRawDylib, LinkRequiresName, MissingNativeLibrary, MultipleCfgs,
-    MultipleImportNameType, MultipleKindsInLink, MultipleLinkModifiers, MultipleModifiers,
-    MultipleNamesInLink, MultipleRenamings, MultipleWasmImport, NoLinkModOverride, RawDylibNoNul,
-    RenamingNoLink, UnexpectedLinkArg, UnknownImportNameType, UnknownLinkKind, UnknownLinkModifier,
-    UnsupportedAbi, UnsupportedAbiI686, WasmImportForm, WholeArchiveNeedsStatic,
-};
+use crate::errors;
 
 use std::path::PathBuf;
 
@@ -52,27 +42,28 @@ pub fn find_native_static_library(
         }
     }
 
-    sess.emit_fatal(MissingNativeLibrary::new(name, verbatim));
+    sess.emit_fatal(errors::MissingNativeLibrary::new(name, verbatim));
 }
 
 fn find_bundled_library(
     name: Option<Symbol>,
     verbatim: Option<bool>,
     kind: NativeLibKind,
+    has_cfg: bool,
     sess: &Session,
 ) -> Option<Symbol> {
-    if sess.opts.unstable_opts.packed_bundled_libs &&
-            sess.crate_types().iter().any(|ct| ct == &CrateType::Rlib || ct == &CrateType::Staticlib) &&
-            let NativeLibKind::Static { bundle: Some(true) | None, .. } = kind {
-        find_native_static_library(
-            name.unwrap().as_str(),
-            verbatim.unwrap_or(false),
-            &sess.target_filesearch(PathKind::Native).search_path_dirs(),
-            sess,
-        ).file_name().and_then(|s| s.to_str()).map(Symbol::intern)
-    } else {
-        None
+    if let NativeLibKind::Static { bundle: Some(true) | None, whole_archive } = kind
+        && sess.crate_types().iter().any(|t| matches!(t, &CrateType::Rlib | CrateType::Staticlib))
+        && (sess.opts.unstable_opts.packed_bundled_libs || has_cfg || whole_archive == Some(true))
+    {
+        let verbatim = verbatim.unwrap_or(false);
+        let search_paths = &sess.target_filesearch(PathKind::Native).search_path_dirs();
+        return find_native_static_library(name.unwrap().as_str(), verbatim, search_paths, sess)
+            .file_name()
+            .and_then(|s| s.to_str())
+            .map(Symbol::intern);
     }
+    None
 }
 
 pub(crate) fn collect(tcx: TyCtxt<'_>) -> Vec<NativeLib> {
@@ -107,13 +98,18 @@ impl<'tcx> Collector<'tcx> {
             return;
         };
 
-        if abi == Abi::Rust || abi == Abi::RustIntrinsic || abi == Abi::PlatformIntrinsic {
+        if matches!(abi, Abi::Rust | Abi::RustIntrinsic | Abi::PlatformIntrinsic) {
             return;
         }
 
         // Process all of the #[link(..)]-style arguments
-        let sess = &self.tcx.sess;
+        let sess = self.tcx.sess;
         let features = self.tcx.features();
+
+        if !sess.opts.unstable_opts.link_directives {
+            return;
+        }
+
         for m in self.tcx.hir().attrs(it.hir_id()).iter().filter(|a| a.has_name(sym::link)) {
             let Some(items) = m.meta_item_list() else {
                 continue;
@@ -129,26 +125,26 @@ impl<'tcx> Collector<'tcx> {
                 match item.name_or_empty() {
                     sym::name => {
                         if name.is_some() {
-                            sess.emit_err(MultipleNamesInLink { span: item.span() });
+                            sess.emit_err(errors::MultipleNamesInLink { span: item.span() });
                             continue;
                         }
                         let Some(link_name) = item.value_str() else {
-                            sess.emit_err(LinkNameForm { span: item.span() });
+                            sess.emit_err(errors::LinkNameForm { span: item.span() });
                             continue;
                         };
                         let span = item.name_value_literal_span().unwrap();
                         if link_name.is_empty() {
-                            sess.emit_err(EmptyLinkName { span });
+                            sess.emit_err(errors::EmptyLinkName { span });
                         }
                         name = Some((link_name, span));
                     }
                     sym::kind => {
                         if kind.is_some() {
-                            sess.emit_err(MultipleKindsInLink { span: item.span() });
+                            sess.emit_err(errors::MultipleKindsInLink { span: item.span() });
                             continue;
                         }
                         let Some(link_kind) = item.value_str() else {
-                            sess.emit_err(LinkKindForm { span: item.span() });
+                            sess.emit_err(errors::LinkKindForm { span: item.span() });
                             continue;
                         };
 
@@ -158,13 +154,13 @@ impl<'tcx> Collector<'tcx> {
                             "dylib" => NativeLibKind::Dylib { as_needed: None },
                             "framework" => {
                                 if !sess.target.is_like_osx {
-                                    sess.emit_err(LinkFrameworkApple { span });
+                                    sess.emit_err(errors::LinkFrameworkApple { span });
                                 }
                                 NativeLibKind::Framework { as_needed: None }
                             }
                             "raw-dylib" => {
                                 if !sess.target.is_like_windows {
-                                    sess.emit_err(FrameworkOnlyWindows { span });
+                                    sess.emit_err(errors::FrameworkOnlyWindows { span });
                                 } else if !features.raw_dylib && sess.target.arch == "x86" {
                                     feature_err(
                                         &sess.parse_sess,
@@ -177,7 +173,7 @@ impl<'tcx> Collector<'tcx> {
                                 NativeLibKind::RawDylib
                             }
                             kind => {
-                                sess.emit_err(UnknownLinkKind { span, kind });
+                                sess.emit_err(errors::UnknownLinkKind { span, kind });
                                 continue;
                             }
                         };
@@ -185,26 +181,26 @@ impl<'tcx> Collector<'tcx> {
                     }
                     sym::modifiers => {
                         if modifiers.is_some() {
-                            sess.emit_err(MultipleLinkModifiers { span: item.span() });
+                            sess.emit_err(errors::MultipleLinkModifiers { span: item.span() });
                             continue;
                         }
                         let Some(link_modifiers) = item.value_str() else {
-                            sess.emit_err(LinkModifiersForm { span: item.span() });
+                            sess.emit_err(errors::LinkModifiersForm { span: item.span() });
                             continue;
                         };
                         modifiers = Some((link_modifiers, item.name_value_literal_span().unwrap()));
                     }
                     sym::cfg => {
                         if cfg.is_some() {
-                            sess.emit_err(MultipleCfgs { span: item.span() });
+                            sess.emit_err(errors::MultipleCfgs { span: item.span() });
                             continue;
                         }
                         let Some(link_cfg) = item.meta_item_list() else {
-                            sess.emit_err(LinkCfgForm { span: item.span() });
+                            sess.emit_err(errors::LinkCfgForm { span: item.span() });
                             continue;
                         };
                         let [NestedMetaItem::MetaItem(link_cfg)] = link_cfg else {
-                            sess.emit_err(LinkCfgSinglePredicate { span: item.span() });
+                            sess.emit_err(errors::LinkCfgSinglePredicate { span: item.span() });
                             continue;
                         };
                         if !features.link_cfg {
@@ -220,26 +216,26 @@ impl<'tcx> Collector<'tcx> {
                     }
                     sym::wasm_import_module => {
                         if wasm_import_module.is_some() {
-                            sess.emit_err(MultipleWasmImport { span: item.span() });
+                            sess.emit_err(errors::MultipleWasmImport { span: item.span() });
                             continue;
                         }
                         let Some(link_wasm_import_module) = item.value_str() else {
-                            sess.emit_err(WasmImportForm { span: item.span() });
+                            sess.emit_err(errors::WasmImportForm { span: item.span() });
                             continue;
                         };
                         wasm_import_module = Some((link_wasm_import_module, item.span()));
                     }
                     sym::import_name_type => {
                         if import_name_type.is_some() {
-                            sess.emit_err(MultipleImportNameType { span: item.span() });
+                            sess.emit_err(errors::MultipleImportNameType { span: item.span() });
                             continue;
                         }
                         let Some(link_import_name_type) = item.value_str() else {
-                            sess.emit_err(ImportNameTypeForm { span: item.span() });
+                            sess.emit_err(errors::ImportNameTypeForm { span: item.span() });
                             continue;
                         };
                         if self.tcx.sess.target.arch != "x86" {
-                            sess.emit_err(ImportNameTypeX86 { span: item.span() });
+                            sess.emit_err(errors::ImportNameTypeX86 { span: item.span() });
                             continue;
                         }
 
@@ -248,7 +244,7 @@ impl<'tcx> Collector<'tcx> {
                             "noprefix" => PeImportNameType::NoPrefix,
                             "undecorated" => PeImportNameType::Undecorated,
                             import_name_type => {
-                                sess.emit_err(UnknownImportNameType {
+                                sess.emit_err(errors::UnknownImportNameType {
                                     span: item.span(),
                                     import_name_type,
                                 });
@@ -268,7 +264,7 @@ impl<'tcx> Collector<'tcx> {
                         import_name_type = Some((link_import_name_type, item.span()));
                     }
                     _ => {
-                        sess.emit_err(UnexpectedLinkArg { span: item.span() });
+                        sess.emit_err(errors::UnexpectedLinkArg { span: item.span() });
                     }
                 }
             }
@@ -280,7 +276,7 @@ impl<'tcx> Collector<'tcx> {
                     let (modifier, value) = match modifier.strip_prefix(&['+', '-']) {
                         Some(m) => (m, modifier.starts_with('+')),
                         None => {
-                            sess.emit_err(InvalidLinkModifier { span });
+                            sess.emit_err(errors::InvalidLinkModifier { span });
                             continue;
                         }
                     };
@@ -298,7 +294,7 @@ impl<'tcx> Collector<'tcx> {
                     }
                     let assign_modifier = |dst: &mut Option<bool>| {
                         if dst.is_some() {
-                            sess.emit_err(MultipleModifiers { span, modifier });
+                            sess.emit_err(errors::MultipleModifiers { span, modifier });
                         } else {
                             *dst = Some(value);
                         }
@@ -308,7 +304,7 @@ impl<'tcx> Collector<'tcx> {
                             assign_modifier(bundle)
                         }
                         ("bundle", _) => {
-                            sess.emit_err(BundleNeedsStatic { span });
+                            sess.emit_err(errors::BundleNeedsStatic { span });
                         }
 
                         ("verbatim", _) => assign_modifier(&mut verbatim),
@@ -317,7 +313,7 @@ impl<'tcx> Collector<'tcx> {
                             assign_modifier(whole_archive)
                         }
                         ("whole-archive", _) => {
-                            sess.emit_err(WholeArchiveNeedsStatic { span });
+                            sess.emit_err(errors::WholeArchiveNeedsStatic { span });
                         }
 
                         ("as-needed", Some(NativeLibKind::Dylib { as_needed }))
@@ -326,11 +322,11 @@ impl<'tcx> Collector<'tcx> {
                             assign_modifier(as_needed)
                         }
                         ("as-needed", _) => {
-                            sess.emit_err(AsNeededCompatibility { span });
+                            sess.emit_err(errors::AsNeededCompatibility { span });
                         }
 
                         _ => {
-                            sess.emit_err(UnknownLinkModifier { span, modifier });
+                            sess.emit_err(errors::UnknownLinkModifier { span, modifier });
                         }
                     }
                 }
@@ -338,23 +334,23 @@ impl<'tcx> Collector<'tcx> {
 
             if let Some((_, span)) = wasm_import_module {
                 if name.is_some() || kind.is_some() || modifiers.is_some() || cfg.is_some() {
-                    sess.emit_err(IncompatibleWasmLink { span });
+                    sess.emit_err(errors::IncompatibleWasmLink { span });
                 }
             } else if name.is_none() {
-                sess.emit_err(LinkRequiresName { span: m.span });
+                sess.emit_err(errors::LinkRequiresName { span: m.span });
             }
 
             // Do this outside of the loop so that `import_name_type` can be specified before `kind`.
             if let Some((_, span)) = import_name_type {
                 if kind != Some(NativeLibKind::RawDylib) {
-                    sess.emit_err(ImportNameTypeRaw { span });
+                    sess.emit_err(errors::ImportNameTypeRaw { span });
                 }
             }
 
             let dll_imports = match kind {
                 Some(NativeLibKind::RawDylib) => {
                     if let Some((name, span)) = name && name.as_str().contains('\0') {
-                        sess.emit_err(RawDylibNoNul { span });
+                        sess.emit_err(errors::RawDylibNoNul { span });
                     }
                     foreign_mod_items
                         .iter()
@@ -383,7 +379,9 @@ impl<'tcx> Collector<'tcx> {
                                 .iter()
                                 .find(|a| a.has_name(sym::link_ordinal))
                                 .unwrap();
-                            sess.emit_err(LinkOrdinalRawDylib { span: link_ordinal_attr.span });
+                            sess.emit_err(errors::LinkOrdinalRawDylib {
+                                span: link_ordinal_attr.span,
+                            });
                         }
                     }
 
@@ -393,7 +391,7 @@ impl<'tcx> Collector<'tcx> {
 
             let name = name.map(|(name, _)| name);
             let kind = kind.unwrap_or(NativeLibKind::Unspecified);
-            let filename = find_bundled_library(name, verbatim, kind, sess);
+            let filename = find_bundled_library(name, verbatim, kind, cfg.is_some(), sess);
             self.libs.push(NativeLib {
                 name,
                 filename,
@@ -414,7 +412,7 @@ impl<'tcx> Collector<'tcx> {
         for lib in &self.tcx.sess.opts.libs {
             if let NativeLibKind::Framework { .. } = lib.kind && !self.tcx.sess.target.is_like_osx {
                 // Cannot check this when parsing options because the target is not yet available.
-                self.tcx.sess.emit_err(LibFrameworkApple);
+                self.tcx.sess.emit_err(errors::LibFrameworkApple);
             }
             if let Some(ref new_name) = lib.new_name {
                 let any_duplicate = self
@@ -423,11 +421,11 @@ impl<'tcx> Collector<'tcx> {
                     .filter_map(|lib| lib.name.as_ref())
                     .any(|n| n.as_str() == lib.name);
                 if new_name.is_empty() {
-                    self.tcx.sess.emit_err(EmptyRenamingTarget { lib_name: &lib.name });
+                    self.tcx.sess.emit_err(errors::EmptyRenamingTarget { lib_name: &lib.name });
                 } else if !any_duplicate {
-                    self.tcx.sess.emit_err(RenamingNoLink { lib_name: &lib.name });
+                    self.tcx.sess.emit_err(errors::RenamingNoLink { lib_name: &lib.name });
                 } else if !renames.insert(&lib.name) {
-                    self.tcx.sess.emit_err(MultipleRenamings { lib_name: &lib.name });
+                    self.tcx.sess.emit_err(errors::MultipleRenamings { lib_name: &lib.name });
                 }
             }
         }
@@ -453,12 +451,15 @@ impl<'tcx> Collector<'tcx> {
                             // explicit `:rename` in particular.
                             if lib.has_modifiers() || passed_lib.has_modifiers() {
                                 match lib.foreign_module {
-                                    Some(def_id) => self.tcx.sess.emit_err(NoLinkModOverride {
-                                        span: Some(self.tcx.def_span(def_id)),
-                                    }),
-                                    None => {
-                                        self.tcx.sess.emit_err(NoLinkModOverride { span: None })
+                                    Some(def_id) => {
+                                        self.tcx.sess.emit_err(errors::NoLinkModOverride {
+                                            span: Some(self.tcx.def_span(def_id)),
+                                        })
                                     }
+                                    None => self
+                                        .tcx
+                                        .sess
+                                        .emit_err(errors::NoLinkModOverride { span: None }),
                                 };
                             }
                             if passed_lib.kind != NativeLibKind::Unspecified {
@@ -480,7 +481,7 @@ impl<'tcx> Collector<'tcx> {
                 let name = Some(Symbol::intern(new_name.unwrap_or(&passed_lib.name)));
                 let sess = self.tcx.sess;
                 let filename =
-                    find_bundled_library(name, passed_lib.verbatim, passed_lib.kind, sess);
+                    find_bundled_library(name, passed_lib.verbatim, passed_lib.kind, false, sess);
                 self.libs.push(NativeLib {
                     name,
                     filename,
@@ -503,9 +504,10 @@ impl<'tcx> Collector<'tcx> {
         let argument_types: &List<Ty<'_>> = self.tcx.erase_late_bound_regions(
             self.tcx
                 .type_of(item.id.owner_id)
+                .subst_identity()
                 .fn_sig(self.tcx)
                 .inputs()
-                .map_bound(|slice| self.tcx.mk_type_list(slice.iter())),
+                .map_bound(|slice| self.tcx.mk_type_list(slice)),
         );
 
         argument_types
@@ -542,14 +544,14 @@ impl<'tcx> Collector<'tcx> {
                     DllCallingConvention::Vectorcall(self.i686_arg_list_size(item))
                 }
                 _ => {
-                    self.tcx.sess.emit_fatal(UnsupportedAbiI686 { span: item.span });
+                    self.tcx.sess.emit_fatal(errors::UnsupportedAbiI686 { span: item.span });
                 }
             }
         } else {
             match abi {
                 Abi::C { .. } | Abi::Win64 { .. } | Abi::System { .. } => DllCallingConvention::C,
                 _ => {
-                    self.tcx.sess.emit_fatal(UnsupportedAbi { span: item.span });
+                    self.tcx.sess.emit_fatal(errors::UnsupportedAbi { span: item.span });
                 }
             }
         };
diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs
index 143d8f2f1..b1e59b0a4 100644
--- a/compiler/rustc_metadata/src/rmeta/decoder.rs
+++ b/compiler/rustc_metadata/src/rmeta/decoder.rs
@@ -11,7 +11,7 @@ use rustc_data_structures::sync::{Lock, LockGuard, Lrc, OnceCell};
 use rustc_data_structures::unhash::UnhashMap;
 use rustc_expand::base::{SyntaxExtension, SyntaxExtensionKind};
 use rustc_expand::proc_macro::{AttrProcMacro, BangProcMacro, DeriveProcMacro};
-use rustc_hir::def::{CtorKind, DefKind, Res};
+use rustc_hir::def::{CtorKind, DefKind, DocLinkResMap, Res};
 use rustc_hir::def_id::{CrateNum, DefId, DefIndex, CRATE_DEF_INDEX, LOCAL_CRATE};
 use rustc_hir::definitions::{DefKey, DefPath, DefPathData, DefPathHash};
 use rustc_hir::diagnostic_items::DiagnosticItems;
@@ -654,7 +654,7 @@ impl<'a, 'tcx, T> Decodable<DecodeContext<'a, 'tcx>> for LazyValue<T> {
 impl<'a, 'tcx, T> Decodable<DecodeContext<'a, 'tcx>> for LazyArray<T> {
     fn decode(decoder: &mut DecodeContext<'a, 'tcx>) -> Self {
         let len = decoder.read_usize();
-        if len == 0 { LazyArray::empty() } else { decoder.read_lazy_array(len) }
+        if len == 0 { LazyArray::default() } else { decoder.read_lazy_array(len) }
     }
 }
 
@@ -864,7 +864,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
                 .tables
                 .children
                 .get(self, index)
-                .unwrap_or_else(LazyArray::empty)
+                .expect("fields are not encoded for a variant")
                 .decode(self)
                 .map(|index| ty::FieldDef {
                     did: self.local_def_id(index),
@@ -896,7 +896,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
                 .tables
                 .children
                 .get(self, item_id)
-                .unwrap_or_else(LazyArray::empty)
+                .expect("variants are not encoded for an enum")
                 .decode(self)
                 .filter_map(|index| {
                     let kind = self.def_kind(index);
@@ -910,7 +910,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
             std::iter::once(self.get_variant(&kind, item_id, did)).collect()
         };
 
-        tcx.alloc_adt_def(did, adt_kind, variants, repr)
+        tcx.mk_adt_def(did, adt_kind, variants, repr)
     }
 
     fn get_generics(self, item_id: DefIndex, sess: &Session) -> ty::Generics {
@@ -985,7 +985,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
         let vis = self.get_visibility(id);
         let span = self.get_span(id, sess);
         let macro_rules = match kind {
-            DefKind::Macro(..) => self.root.tables.macro_rules.get(self, id).is_some(),
+            DefKind::Macro(..) => self.root.tables.is_macro_rules.get(self, id),
             _ => false,
         };
 
@@ -1045,7 +1045,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
             .tables
             .fn_arg_names
             .get(self, id)
-            .unwrap_or_else(LazyArray::empty)
+            .expect("argument names not encoded for a function")
             .decode((self, sess))
             .nth(0)
             .map_or(false, |ident| ident.name == kw::SelfLower)
@@ -1060,7 +1060,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
             .tables
             .children
             .get(self, id)
-            .unwrap_or_else(LazyArray::empty)
+            .expect("associated items not encoded for an item")
             .decode((self, sess))
             .map(move |child_index| self.local_def_id(child_index))
     }
@@ -1068,13 +1068,12 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
     fn get_associated_item(self, id: DefIndex, sess: &'a Session) -> ty::AssocItem {
         let name = self.item_name(id);
 
-        let kind = match self.def_kind(id) {
-            DefKind::AssocConst => ty::AssocKind::Const,
-            DefKind::AssocFn => ty::AssocKind::Fn,
-            DefKind::AssocTy => ty::AssocKind::Type,
+        let (kind, has_self) = match self.def_kind(id) {
+            DefKind::AssocConst => (ty::AssocKind::Const, false),
+            DefKind::AssocFn => (ty::AssocKind::Fn, self.get_fn_has_self_parameter(id, sess)),
+            DefKind::AssocTy => (ty::AssocKind::Type, false),
             _ => bug!("cannot get associated-item of `{:?}`", self.def_key(id)),
         };
-        let has_self = self.get_fn_has_self_parameter(id, sess);
         let container = self.root.tables.assoc_container.get(self, id).unwrap();
 
         ty::AssocItem {
@@ -1131,7 +1130,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
             .tables
             .children
             .get(self, id)
-            .unwrap_or_else(LazyArray::empty)
+            .expect("fields not encoded for a struct")
             .decode(self)
             .map(move |index| respan(self.get_span(index, sess), self.item_name(index)))
     }
@@ -1144,7 +1143,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
             .tables
             .children
             .get(self, id)
-            .unwrap_or_else(LazyArray::empty)
+            .expect("fields not encoded for a struct")
             .decode(self)
             .map(move |field_index| self.get_visibility(field_index))
     }
@@ -1159,52 +1158,23 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
                 .tables
                 .inherent_impls
                 .get(self, id)
-                .unwrap_or_else(LazyArray::empty)
                 .decode(self)
                 .map(|index| self.local_def_id(index)),
         )
     }
 
-    /// Decodes all inherent impls in the crate (for rustdoc).
-    fn get_inherent_impls(self) -> impl Iterator<Item = (DefId, DefId)> + 'a {
-        (0..self.root.tables.inherent_impls.size()).flat_map(move |i| {
-            let ty_index = DefIndex::from_usize(i);
-            let ty_def_id = self.local_def_id(ty_index);
-            self.root
-                .tables
-                .inherent_impls
-                .get(self, ty_index)
-                .unwrap_or_else(LazyArray::empty)
-                .decode(self)
-                .map(move |impl_index| (ty_def_id, self.local_def_id(impl_index)))
-        })
-    }
-
     /// Decodes all traits in the crate (for rustdoc and rustc diagnostics).
     fn get_traits(self) -> impl Iterator<Item = DefId> + 'a {
         self.root.traits.decode(self).map(move |index| self.local_def_id(index))
     }
 
     /// Decodes all trait impls in the crate (for rustdoc).
-    fn get_trait_impls(self) -> impl Iterator<Item = (DefId, DefId, Option<SimplifiedType>)> + 'a {
-        self.cdata.trait_impls.iter().flat_map(move |(&(trait_cnum_raw, trait_index), impls)| {
-            let trait_def_id = DefId {
-                krate: self.cnum_map[CrateNum::from_u32(trait_cnum_raw)],
-                index: trait_index,
-            };
-            impls.decode(self).map(move |(impl_index, simplified_self_ty)| {
-                (trait_def_id, self.local_def_id(impl_index), simplified_self_ty)
-            })
+    fn get_trait_impls(self) -> impl Iterator<Item = DefId> + 'a {
+        self.cdata.trait_impls.values().flat_map(move |impls| {
+            impls.decode(self).map(move |(impl_index, _)| self.local_def_id(impl_index))
         })
     }
 
-    fn get_all_incoherent_impls(self) -> impl Iterator<Item = DefId> + 'a {
-        self.cdata
-            .incoherent_impls
-            .values()
-            .flat_map(move |impls| impls.decode(self).map(move |idx| self.local_def_id(idx)))
-    }
-
     fn get_incoherent_impls(self, tcx: TyCtxt<'tcx>, simp: SimplifiedType) -> &'tcx [DefId] {
         if let Some(impls) = self.cdata.incoherent_impls.get(&simp) {
             tcx.arena.alloc_from_iter(impls.decode(self).map(|idx| self.local_def_id(idx)))
@@ -1283,7 +1253,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
     fn get_macro(self, id: DefIndex, sess: &Session) -> ast::MacroDef {
         match self.def_kind(id) {
             DefKind::Macro(_) => {
-                let macro_rules = self.root.tables.macro_rules.get(self, id).is_some();
+                let macro_rules = self.root.tables.is_macro_rules.get(self, id);
                 let body =
                     self.root.tables.macro_definition.get(self, id).unwrap().decode((self, sess));
                 ast::MacroDef { macro_rules, body: ast::ptr::P(body) }
@@ -1322,7 +1292,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
     ) -> DefPathHash {
         *def_path_hashes
             .entry(index)
-            .or_insert_with(|| self.root.tables.def_path_hashes.get(self, index).unwrap())
+            .or_insert_with(|| self.root.tables.def_path_hashes.get(self, index))
     }
 
     #[inline]
@@ -1594,12 +1564,30 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
             })
     }
 
-    fn get_may_have_doc_links(self, index: DefIndex) -> bool {
-        self.root.tables.may_have_doc_links.get(self, index).is_some()
+    fn get_attr_flags(self, index: DefIndex) -> AttrFlags {
+        self.root.tables.attr_flags.get(self, index)
     }
 
     fn get_is_intrinsic(self, index: DefIndex) -> bool {
-        self.root.tables.is_intrinsic.get(self, index).is_some()
+        self.root.tables.is_intrinsic.get(self, index)
+    }
+
+    fn get_doc_link_resolutions(self, index: DefIndex) -> DocLinkResMap {
+        self.root
+            .tables
+            .doc_link_resolutions
+            .get(self, index)
+            .expect("no resolutions for a doc link")
+            .decode(self)
+    }
+
+    fn get_doc_link_traits_in_scope(self, index: DefIndex) -> impl Iterator<Item = DefId> + 'a {
+        self.root
+            .tables
+            .doc_link_traits_in_scope
+            .get(self, index)
+            .expect("no traits in scope for a doc link")
+            .decode(self)
     }
 }
 
diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
index 6fd5bd52a..83a0e833e 100644
--- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
+++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
@@ -1,6 +1,8 @@
 use crate::creader::{CStore, LoadedMacro};
 use crate::foreign_modules;
 use crate::native_libs;
+use crate::rmeta::table::IsDefault;
+use crate::rmeta::AttrFlags;
 
 use rustc_ast as ast;
 use rustc_attr::Deprecation;
@@ -87,6 +89,14 @@ macro_rules! provide_one {
             }
         }
     };
+    ($tcx:ident, $def_id:ident, $other:ident, $cdata:ident, $name:ident => { table_defaulted_array }) => {
+        provide_one! {
+            $tcx, $def_id, $other, $cdata, $name => {
+                let lazy = $cdata.root.tables.$name.get($cdata, $def_id.index);
+                if lazy.is_default() { &[] } else { $tcx.arena.alloc_from_iter(lazy.decode(($cdata, $tcx))) }
+            }
+        }
+    };
     ($tcx:ident, $def_id:ident, $other:ident, $cdata:ident, $name:ident => { table_direct }) => {
         provide_one! {
             $tcx, $def_id, $other, $cdata, $name => {
@@ -104,7 +114,7 @@ macro_rules! provide_one {
         fn $name<'tcx>(
             $tcx: TyCtxt<'tcx>,
             def_id_arg: ty::query::query_keys::$name<'tcx>,
-        ) -> ty::query::query_values::$name<'tcx> {
+        ) -> ty::query::query_provided::$name<'tcx> {
             let _prof_timer =
                 $tcx.prof.generic_activity(concat!("metadata_decode_entry_", stringify!($name)));
 
@@ -120,7 +130,13 @@ macro_rules! provide_one {
                 $tcx.ensure().crate_hash($def_id.krate);
             }
 
-            let $cdata = CStore::from_tcx($tcx).get_crate_data($def_id.krate);
+            let cdata = rustc_data_structures::sync::MappedReadGuard::map(CStore::from_tcx($tcx), |c| {
+                c.get_crate_data($def_id.krate).cdata
+            });
+            let $cdata = crate::creader::CrateMetadataRef {
+                cdata: &cdata,
+                cstore: &CStore::from_tcx($tcx),
+            };
 
             $compute
         }
@@ -186,10 +202,10 @@ impl IntoArgs for (CrateNum, SimplifiedType) {
 }
 
 provide! { tcx, def_id, other, cdata,
-    explicit_item_bounds => { table }
+    explicit_item_bounds => { table_defaulted_array }
     explicit_predicates_of => { table }
     generics_of => { table }
-    inferred_outlives_of => { table }
+    inferred_outlives_of => { table_defaulted_array }
     super_predicates_of => { table }
     type_of => { table }
     variances_of => { table }
@@ -201,6 +217,7 @@ provide! { tcx, def_id, other, cdata,
     thir_abstract_const => { table }
     optimized_mir => { table }
     mir_for_ctfe => { table }
+    mir_generator_witnesses => { table }
     promoted_mir => { table }
     def_span => { table }
     def_ident_span => { table }
@@ -225,12 +242,7 @@ provide! { tcx, def_id, other, cdata,
     deduced_param_attrs => { table }
     is_type_alias_impl_trait => {
         debug_assert_eq!(tcx.def_kind(def_id), DefKind::OpaqueTy);
-        cdata
-            .root
-            .tables
-            .is_type_alias_impl_trait
-            .get(cdata, def_id.index)
-            .is_some()
+        cdata.root.tables.is_type_alias_impl_trait.get(cdata, def_id.index)
     }
     collect_return_position_impl_trait_in_trait_tys => {
         Ok(cdata
@@ -242,6 +254,8 @@ provide! { tcx, def_id, other, cdata,
             .process_decoded(tcx, || panic!("{def_id:?} does not have trait_impl_trait_tys")))
      }
 
+    associated_items_for_impl_trait_in_trait => { table_defaulted_array }
+
     visibility => { cdata.get_visibility(def_id.index) }
     adt_def => { cdata.get_adt_def(def_id.index, tcx) }
     adt_destructor => {
@@ -298,6 +312,7 @@ provide! { tcx, def_id, other, cdata,
     extra_filename => { cdata.root.extra_filename.clone() }
 
     traits_in_crate => { tcx.arena.alloc_from_iter(cdata.get_traits()) }
+    trait_impls_in_crate => { tcx.arena.alloc_from_iter(cdata.get_trait_impls()) }
     implementations_of_trait => { cdata.get_implementations_of_trait(tcx, other) }
     crate_incoherent_impls => { cdata.get_incoherent_impls(tcx, other) }
 
@@ -338,6 +353,11 @@ provide! { tcx, def_id, other, cdata,
     crate_extern_paths => { cdata.source().paths().cloned().collect() }
     expn_that_defined => { cdata.get_expn_that_defined(def_id.index, tcx.sess) }
     generator_diagnostic_data => { cdata.get_generator_diagnostic_data(tcx, def_id.index) }
+    is_doc_hidden => { cdata.get_attr_flags(def_id.index).contains(AttrFlags::IS_DOC_HIDDEN) }
+    doc_link_resolutions => { tcx.arena.alloc(cdata.get_doc_link_resolutions(def_id.index)) }
+    doc_link_traits_in_scope => {
+        tcx.arena.alloc_from_iter(cdata.get_doc_link_traits_in_scope(def_id.index))
+    }
 }
 
 pub(in crate::rmeta) fn provide(providers: &mut Providers) {
@@ -425,7 +445,7 @@ pub(in crate::rmeta) fn provide(providers: &mut Providers) {
                         return;
                     }
 
-                    if ty::util::is_doc_hidden(tcx, parent) {
+                    if tcx.is_doc_hidden(parent) {
                         fallback_map.push((def_id, parent));
                         return;
                     }
@@ -597,42 +617,6 @@ impl CStore {
     ) -> Span {
         self.get_crate_data(cnum).get_proc_macro_quoted_span(id, sess)
     }
-
-    /// Decodes all trait impls in the crate (for rustdoc).
-    pub fn trait_impls_in_crate_untracked(
-        &self,
-        cnum: CrateNum,
-    ) -> impl Iterator<Item = (DefId, DefId, Option<SimplifiedType>)> + '_ {
-        self.get_crate_data(cnum).get_trait_impls()
-    }
-
-    /// Decodes all inherent impls in the crate (for rustdoc).
-    pub fn inherent_impls_in_crate_untracked(
-        &self,
-        cnum: CrateNum,
-    ) -> impl Iterator<Item = (DefId, DefId)> + '_ {
-        self.get_crate_data(cnum).get_inherent_impls()
-    }
-
-    /// Decodes all incoherent inherent impls in the crate (for rustdoc).
-    pub fn incoherent_impls_in_crate_untracked(
-        &self,
-        cnum: CrateNum,
-    ) -> impl Iterator<Item = DefId> + '_ {
-        self.get_crate_data(cnum).get_all_incoherent_impls()
-    }
-
-    pub fn associated_item_def_ids_untracked<'a>(
-        &'a self,
-        def_id: DefId,
-        sess: &'a Session,
-    ) -> impl Iterator<Item = DefId> + 'a {
-        self.get_crate_data(def_id.krate).get_associated_item_def_ids(def_id.index, sess)
-    }
-
-    pub fn may_have_doc_links_untracked(&self, def_id: DefId) -> bool {
-        self.get_crate_data(def_id.krate).get_may_have_doc_links(def_id.index)
-    }
 }
 
 impl CrateStore for CStore {
diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs
index 8f7a61b72..3ab01f780 100644
--- a/compiler/rustc_metadata/src/rmeta/encoder.rs
+++ b/compiler/rustc_metadata/src/rmeta/encoder.rs
@@ -38,7 +38,6 @@ use rustc_span::symbol::{sym, Symbol};
 use rustc_span::{
     self, DebuggerVisualizerFile, ExternalSource, FileName, SourceFile, Span, SyntaxContext,
 };
-use rustc_target::abi::VariantIdx;
 use std::borrow::Borrow;
 use std::collections::hash_map::Entry;
 use std::hash::Hash;
@@ -76,13 +75,13 @@ pub(super) struct EncodeContext<'a, 'tcx> {
     symbol_table: FxHashMap<Symbol, usize>,
 }
 
-/// If the current crate is a proc-macro, returns early with `LazyArray::empty()`.
+/// If the current crate is a proc-macro, returns early with `LazyArray::default()`.
 /// This is useful for skipping the encoding of things that aren't needed
 /// for proc-macro crates.
 macro_rules! empty_proc_macro {
     ($self:ident) => {
         if $self.is_proc_macro {
-            return LazyArray::empty();
+            return LazyArray::default();
         }
     };
 }
@@ -365,25 +364,35 @@ impl<'a, 'tcx> TyEncoder for EncodeContext<'a, 'tcx> {
     }
 }
 
-// Shorthand for `$self.$tables.$table.set($def_id.index, $self.lazy_value($value))`, which would
+// Shorthand for `$self.$tables.$table.set_some($def_id.index, $self.lazy_value($value))`, which would
 // normally need extra variables to avoid errors about multiple mutable borrows.
 macro_rules! record {
     ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{
         {
             let value = $value;
             let lazy = $self.lazy(value);
-            $self.$tables.$table.set($def_id.index, lazy);
+            $self.$tables.$table.set_some($def_id.index, lazy);
         }
     }};
 }
 
-// Shorthand for `$self.$tables.$table.set($def_id.index, $self.lazy_value($value))`, which would
+// Shorthand for `$self.$tables.$table.set_some($def_id.index, $self.lazy_value($value))`, which would
 // normally need extra variables to avoid errors about multiple mutable borrows.
 macro_rules! record_array {
     ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{
         {
             let value = $value;
             let lazy = $self.lazy_array(value);
+            $self.$tables.$table.set_some($def_id.index, lazy);
+        }
+    }};
+}
+
+macro_rules! record_defaulted_array {
+    ($self:ident.$tables:ident.$table:ident[$def_id:expr] <- $value:expr) => {{
+        {
+            let value = $value;
+            let lazy = $self.lazy_array(value);
             $self.$tables.$table.set($def_id.index, lazy);
         }
     }};
@@ -467,13 +476,13 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             {
                 let def_key = self.lazy(table.def_key(def_index));
                 let def_path_hash = table.def_path_hash(def_index);
-                self.tables.def_keys.set(def_index, def_key);
+                self.tables.def_keys.set_some(def_index, def_key);
                 self.tables.def_path_hashes.set(def_index, def_path_hash);
             }
         } else {
             for (def_index, def_key, def_path_hash) in table.enumerated_keys_and_path_hashes() {
                 let def_key = self.lazy(def_key);
-                self.tables.def_keys.set(def_index, def_key);
+                self.tables.def_keys.set_some(def_index, def_key);
                 self.tables.def_path_hashes.set(def_index, *def_path_hash);
             }
         }
@@ -483,7 +492,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         self.lazy(DefPathHashMapRef::BorrowedFromTcx(self.tcx.def_path_hash_to_def_index_map()))
     }
 
-    fn encode_source_map(&mut self) -> LazyTable<u32, LazyValue<rustc_span::SourceFile>> {
+    fn encode_source_map(&mut self) -> LazyTable<u32, Option<LazyValue<rustc_span::SourceFile>>> {
         let source_map = self.tcx.sess.source_map();
         let all_source_files = source_map.files();
 
@@ -548,7 +557,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
 
             let on_disk_index: u32 =
                 on_disk_index.try_into().expect("cannot export more than U32_MAX files");
-            adapted.set(on_disk_index, self.lazy(source_file));
+            adapted.set_some(on_disk_index, self.lazy(source_file));
         }
 
         adapted.encode(&mut self.opaque)
@@ -760,36 +769,50 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
     }
 }
 
+struct AnalyzeAttrState {
+    is_exported: bool,
+    is_doc_hidden: bool,
+}
+
 /// Returns whether an attribute needs to be recorded in metadata, that is, if it's usable and
 /// useful in downstream crates. Local-only attributes are an obvious example, but some
 /// rustdoc-specific attributes can equally be of use while documenting the current crate only.
 ///
 /// Removing these superfluous attributes speeds up compilation by making the metadata smaller.
 ///
-/// Note: the `is_def_id_public` parameter is used to cache whether the given `DefId` has a public
+/// Note: the `is_exported` parameter is used to cache whether the given `DefId` has a public
 /// visibility: this is a piece of data that can be computed once per defid, and not once per
 /// attribute. Some attributes would only be usable downstream if they are public.
 #[inline]
-fn should_encode_attr(
-    tcx: TyCtxt<'_>,
-    attr: &Attribute,
-    def_id: LocalDefId,
-    is_def_id_public: &mut Option<bool>,
-) -> bool {
+fn analyze_attr(attr: &Attribute, state: &mut AnalyzeAttrState) -> bool {
+    let mut should_encode = false;
     if rustc_feature::is_builtin_only_local(attr.name_or_empty()) {
         // Attributes marked local-only don't need to be encoded for downstream crates.
-        false
     } else if attr.doc_str().is_some() {
-        // We keep all public doc comments because they might be "imported" into downstream crates
-        // if they use `#[doc(inline)]` to copy an item's documentation into their own.
-        *is_def_id_public.get_or_insert_with(|| tcx.effective_visibilities(()).is_exported(def_id))
+        // We keep all doc comments reachable to rustdoc because they might be "imported" into
+        // downstream crates if they use `#[doc(inline)]` to copy an item's documentation into
+        // their own.
+        if state.is_exported {
+            should_encode = true;
+        }
     } else if attr.has_name(sym::doc) {
-        // If this is a `doc` attribute, and it's marked `inline` (as in `#[doc(inline)]`), we can
-        // remove it. It won't be inlinable in downstream crates.
-        attr.meta_item_list().map(|l| l.iter().any(|l| !l.has_name(sym::inline))).unwrap_or(false)
+        // If this is a `doc` attribute that doesn't have anything except maybe `inline` (as in
+        // `#[doc(inline)]`), then we can remove it. It won't be inlinable in downstream crates.
+        if let Some(item_list) = attr.meta_item_list() {
+            for item in item_list {
+                if !item.has_name(sym::inline) {
+                    should_encode = true;
+                    if item.has_name(sym::hidden) {
+                        state.is_doc_hidden = true;
+                        break;
+                    }
+                }
+            }
+        }
     } else {
-        true
+        should_encode = true;
     }
+    should_encode
 }
 
 fn should_encode_visibility(def_kind: DefKind) -> bool {
@@ -815,7 +838,7 @@ fn should_encode_visibility(def_kind: DefKind) -> bool {
         | DefKind::ForeignMod
         | DefKind::OpaqueTy
         | DefKind::ImplTraitPlaceholder
-        | DefKind::Impl
+        | DefKind::Impl { .. }
         | DefKind::Field => true,
         DefKind::TyParam
         | DefKind::ConstParam
@@ -850,7 +873,7 @@ fn should_encode_stability(def_kind: DefKind) -> bool {
         | DefKind::ImplTraitPlaceholder
         | DefKind::Enum
         | DefKind::Union
-        | DefKind::Impl
+        | DefKind::Impl { .. }
         | DefKind::Trait
         | DefKind::TraitAlias
         | DefKind::Macro(..)
@@ -928,7 +951,7 @@ fn should_encode_variances(def_kind: DefKind) -> bool {
         | DefKind::Const
         | DefKind::ForeignMod
         | DefKind::TyAlias
-        | DefKind::Impl
+        | DefKind::Impl { .. }
         | DefKind::Trait
         | DefKind::TraitAlias
         | DefKind::Macro(..)
@@ -965,7 +988,7 @@ fn should_encode_generics(def_kind: DefKind) -> bool {
         | DefKind::InlineConst
         | DefKind::OpaqueTy
         | DefKind::ImplTraitPlaceholder
-        | DefKind::Impl
+        | DefKind::Impl { .. }
         | DefKind::Field
         | DefKind::TyParam
         | DefKind::Closure
@@ -995,7 +1018,7 @@ fn should_encode_type(tcx: TyCtxt<'_>, def_id: LocalDefId, def_kind: DefKind) ->
         | DefKind::TyAlias
         | DefKind::OpaqueTy
         | DefKind::ForeignTy
-        | DefKind::Impl
+        | DefKind::Impl { .. }
         | DefKind::AssocFn
         | DefKind::AssocConst
         | DefKind::Closure
@@ -1058,7 +1081,7 @@ fn should_encode_const(def_kind: DefKind) -> bool {
         | DefKind::OpaqueTy
         | DefKind::ImplTraitPlaceholder
         | DefKind::ForeignTy
-        | DefKind::Impl
+        | DefKind::Impl { .. }
         | DefKind::AssocFn
         | DefKind::Closure
         | DefKind::Generator
@@ -1078,49 +1101,40 @@ fn should_encode_const(def_kind: DefKind) -> bool {
     }
 }
 
-fn should_encode_trait_impl_trait_tys(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
-    if tcx.def_kind(def_id) != DefKind::AssocFn {
-        return false;
-    }
-
-    let Some(item) = tcx.opt_associated_item(def_id) else { return false; };
-    if item.container != ty::AssocItemContainer::ImplContainer {
-        return false;
+// We only encode impl trait in trait when using `lower-impl-trait-in-trait-to-assoc-ty` unstable
+// option.
+fn should_encode_fn_impl_trait_in_trait<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool {
+    if tcx.sess.opts.unstable_opts.lower_impl_trait_in_trait_to_assoc_ty
+        && let Some(assoc_item) = tcx.opt_associated_item(def_id)
+        && assoc_item.container == ty::AssocItemContainer::TraitContainer
+        && assoc_item.kind == ty::AssocKind::Fn
+    {
+        true
+    } else {
+        false
     }
-
-    let Some(trait_item_def_id) = item.trait_item_def_id else { return false; };
-
-    // FIXME(RPITIT): This does a somewhat manual walk through the signature
-    // of the trait fn to look for any RPITITs, but that's kinda doing a lot
-    // of work. We can probably remove this when we refactor RPITITs to be
-    // associated types.
-    tcx.fn_sig(trait_item_def_id).skip_binder().output().walk().any(|arg| {
-        if let ty::GenericArgKind::Type(ty) = arg.unpack()
-            && let ty::Alias(ty::Projection, data) = ty.kind()
-            && tcx.def_kind(data.def_id) == DefKind::ImplTraitPlaceholder
-        {
-            true
-        } else {
-            false
-        }
-    })
 }
 
 impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
     fn encode_attrs(&mut self, def_id: LocalDefId) {
         let tcx = self.tcx;
-        let mut is_public: Option<bool> = None;
-
-        let mut attrs = tcx
-            .hir()
-            .attrs(tcx.hir().local_def_id_to_hir_id(def_id))
+        let mut state = AnalyzeAttrState {
+            is_exported: tcx.effective_visibilities(()).is_exported(def_id),
+            is_doc_hidden: false,
+        };
+        let attr_iter = tcx
+            .opt_local_def_id_to_hir_id(def_id)
+            .map_or(Default::default(), |hir_id| tcx.hir().attrs(hir_id))
             .iter()
-            .filter(move |attr| should_encode_attr(tcx, attr, def_id, &mut is_public));
+            .filter(|attr| analyze_attr(attr, &mut state));
+
+        record_array!(self.tables.attributes[def_id.to_def_id()] <- attr_iter);
 
-        record_array!(self.tables.attributes[def_id.to_def_id()] <- attrs.clone());
-        if attrs.any(|attr| attr.may_have_doc_links()) {
-            self.tables.may_have_doc_links.set(def_id.local_def_index, ());
+        let mut attr_flags = AttrFlags::empty();
+        if state.is_doc_hidden {
+            attr_flags |= AttrFlags::IS_DOC_HIDDEN;
         }
+        self.tables.attr_flags.set(def_id.local_def_index, attr_flags);
     }
 
     fn encode_def_ids(&mut self) {
@@ -1132,7 +1146,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             let def_id = local_id.to_def_id();
             let def_kind = tcx.opt_def_kind(local_id);
             let Some(def_kind) = def_kind else { continue };
-            self.tables.opt_def_kind.set(def_id.index, def_kind);
+            self.tables.opt_def_kind.set_some(def_id.index, def_kind);
             let def_span = tcx.def_span(local_id);
             record!(self.tables.def_span[def_id] <- def_span);
             self.encode_attrs(local_id);
@@ -1163,9 +1177,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 record!(self.tables.generics_of[def_id] <- g);
                 record!(self.tables.explicit_predicates_of[def_id] <- self.tcx.explicit_predicates_of(def_id));
                 let inferred_outlives = self.tcx.inferred_outlives_of(def_id);
-                if !inferred_outlives.is_empty() {
-                    record_array!(self.tables.inferred_outlives_of[def_id] <- inferred_outlives);
-                }
+                record_defaulted_array!(self.tables.inferred_outlives_of[def_id] <- inferred_outlives);
             }
             if should_encode_type(tcx, local_id, def_kind) {
                 record!(self.tables.type_of[def_id] <- self.tcx.type_of(def_id));
@@ -1178,70 +1190,85 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 record!(self.tables.super_predicates_of[def_id] <- self.tcx.super_predicates_of(def_id));
             }
             if let DefKind::Enum | DefKind::Struct | DefKind::Union = def_kind {
-                let params_in_repr = self.tcx.params_in_repr(def_id);
-                record!(self.tables.params_in_repr[def_id] <- params_in_repr);
+                self.encode_info_for_adt(def_id);
             }
-            if should_encode_trait_impl_trait_tys(tcx, def_id)
+            if tcx.impl_method_has_trait_impl_trait_tys(def_id)
                 && let Ok(table) = self.tcx.collect_return_position_impl_trait_in_trait_tys(def_id)
             {
                 record!(self.tables.trait_impl_trait_tys[def_id] <- table);
             }
+            if should_encode_fn_impl_trait_in_trait(tcx, def_id) {
+                let table = tcx.associated_items_for_impl_trait_in_trait(def_id);
+                record_defaulted_array!(self.tables.associated_items_for_impl_trait_in_trait[def_id] <- table);
+            }
         }
+
         let inherent_impls = tcx.with_stable_hashing_context(|hcx| {
             tcx.crate_inherent_impls(()).inherent_impls.to_sorted(&hcx, true)
         });
-
-        for (def_id, implementations) in inherent_impls {
-            if implementations.is_empty() {
-                continue;
-            }
-            record_array!(self.tables.inherent_impls[def_id.to_def_id()] <- implementations.iter().map(|&def_id| {
+        for (def_id, impls) in inherent_impls {
+            record_defaulted_array!(self.tables.inherent_impls[def_id.to_def_id()] <- impls.iter().map(|def_id| {
                 assert!(def_id.is_local());
                 def_id.index
             }));
         }
+
+        for (def_id, res_map) in &tcx.resolutions(()).doc_link_resolutions {
+            record!(self.tables.doc_link_resolutions[def_id.to_def_id()] <- res_map);
+        }
+
+        for (def_id, traits) in &tcx.resolutions(()).doc_link_traits_in_scope {
+            record_array!(self.tables.doc_link_traits_in_scope[def_id.to_def_id()] <- traits);
+        }
     }
 
-    fn encode_enum_variant_info(&mut self, def: ty::AdtDef<'tcx>, index: VariantIdx) {
+    #[instrument(level = "trace", skip(self))]
+    fn encode_info_for_adt(&mut self, def_id: DefId) {
         let tcx = self.tcx;
-        let variant = &def.variant(index);
-        let def_id = variant.def_id;
-        debug!("EncodeContext::encode_enum_variant_info({:?})", def_id);
-
-        let data = VariantData {
-            discr: variant.discr,
-            ctor: variant.ctor.map(|(kind, def_id)| (kind, def_id.index)),
-            is_non_exhaustive: variant.is_field_list_non_exhaustive(),
-        };
+        let adt_def = tcx.adt_def(def_id);
+        record!(self.tables.repr_options[def_id] <- adt_def.repr());
+
+        let params_in_repr = self.tcx.params_in_repr(def_id);
+        record!(self.tables.params_in_repr[def_id] <- params_in_repr);
 
-        record!(self.tables.variant_data[def_id] <- data);
-        self.tables.constness.set(def_id.index, hir::Constness::Const);
-        record_array!(self.tables.children[def_id] <- variant.fields.iter().map(|f| {
-            assert!(f.did.is_local());
-            f.did.index
-        }));
-        if let Some((CtorKind::Fn, ctor_def_id)) = variant.ctor {
-            // FIXME(eddyb) encode signature only in `encode_enum_variant_ctor`.
-            record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(ctor_def_id));
+        if adt_def.is_enum() {
+            record_array!(self.tables.children[def_id] <- iter::from_generator(||
+                for variant in tcx.adt_def(def_id).variants() {
+                    yield variant.def_id.index;
+                    // Encode constructors which take a separate slot in value namespace.
+                    if let Some(ctor_def_id) = variant.ctor_def_id() {
+                        yield ctor_def_id.index;
+                    }
+                }
+            ));
+        } else {
+            // For non-enum, there is only one variant, and its def_id is the adt's.
+            debug_assert_eq!(adt_def.variants().len(), 1);
+            debug_assert_eq!(adt_def.non_enum_variant().def_id, def_id);
+            // Therefore, the loop over variants will encode its fields as the adt's children.
         }
-    }
 
-    fn encode_enum_variant_ctor(&mut self, def: ty::AdtDef<'tcx>, index: VariantIdx) {
-        let variant = &def.variant(index);
-        let Some((ctor_kind, def_id)) = variant.ctor else { return };
-        debug!("EncodeContext::encode_enum_variant_ctor({:?})", def_id);
+        for variant in adt_def.variants().iter() {
+            let data = VariantData {
+                discr: variant.discr,
+                ctor: variant.ctor.map(|(kind, def_id)| (kind, def_id.index)),
+                is_non_exhaustive: variant.is_field_list_non_exhaustive(),
+            };
+            record!(self.tables.variant_data[variant.def_id] <- data);
 
-        // FIXME(eddyb) encode only the `CtorKind` for constructors.
-        let data = VariantData {
-            discr: variant.discr,
-            ctor: Some((ctor_kind, def_id.index)),
-            is_non_exhaustive: variant.is_field_list_non_exhaustive(),
-        };
+            self.tables.constness.set_some(variant.def_id.index, hir::Constness::Const);
+            record_array!(self.tables.children[variant.def_id] <- variant.fields.iter().map(|f| {
+                assert!(f.did.is_local());
+                f.did.index
+            }));
 
-        record!(self.tables.variant_data[def_id] <- data);
-        self.tables.constness.set(def_id.index, hir::Constness::Const);
-        if ctor_kind == CtorKind::Fn {
-            record!(self.tables.fn_sig[def_id] <- self.tcx.fn_sig(def_id));
+            if let Some((CtorKind::Fn, ctor_def_id)) = variant.ctor {
+                self.tables.constness.set_some(ctor_def_id.index, hir::Constness::Const);
+                let fn_sig = tcx.fn_sig(ctor_def_id);
+                record!(self.tables.fn_sig[ctor_def_id] <- fn_sig);
+                // FIXME only encode signature for ctor_def_id
+                record!(self.tables.fn_sig[variant.def_id] <- fn_sig);
+            }
         }
     }
 
@@ -1275,8 +1302,8 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                         hir::ItemKind::Struct(ref vdata, _) => {
                             yield item_id.owner_id.def_id.local_def_index;
                             // Encode constructors which take a separate slot in value namespace.
-                            if let Some(ctor_hir_id) = vdata.ctor_hir_id() {
-                                yield tcx.hir().local_def_id(ctor_hir_id).local_def_index;
+                            if let Some(ctor_def_id) = vdata.ctor_def_id() {
+                                yield ctor_def_id.local_def_index;
                             }
                         }
                         _ if tcx.def_key(item_id.owner_id.to_def_id()).get_opt_name().is_some() => {
@@ -1294,31 +1321,10 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         }
     }
 
-    fn encode_struct_ctor(&mut self, adt_def: ty::AdtDef<'tcx>) {
-        let variant = adt_def.non_enum_variant();
-        let Some((ctor_kind, def_id)) = variant.ctor else { return };
-        debug!("EncodeContext::encode_struct_ctor({:?})", def_id);
-
-        let data = VariantData {
-            discr: variant.discr,
-            ctor: Some((ctor_kind, def_id.index)),
-            is_non_exhaustive: variant.is_field_list_non_exhaustive(),
-        };
-
-        record!(self.tables.repr_options[def_id] <- adt_def.repr());
-        record!(self.tables.variant_data[def_id] <- data);
-        self.tables.constness.set(def_id.index, hir::Constness::Const);
-        if ctor_kind == CtorKind::Fn {
-            record!(self.tables.fn_sig[def_id] <- self.tcx.fn_sig(def_id));
-        }
-    }
-
     fn encode_explicit_item_bounds(&mut self, def_id: DefId) {
         debug!("EncodeContext::encode_explicit_item_bounds({:?})", def_id);
         let bounds = self.tcx.explicit_item_bounds(def_id);
-        if !bounds.is_empty() {
-            record_array!(self.tables.explicit_item_bounds[def_id] <- bounds);
-        }
+        record_defaulted_array!(self.tables.explicit_item_bounds[def_id] <- bounds);
     }
 
     fn encode_info_for_trait_item(&mut self, def_id: DefId) {
@@ -1326,16 +1332,16 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         let tcx = self.tcx;
 
         let impl_defaultness = tcx.impl_defaultness(def_id.expect_local());
-        self.tables.impl_defaultness.set(def_id.index, impl_defaultness);
+        self.tables.impl_defaultness.set_some(def_id.index, impl_defaultness);
         let trait_item = tcx.associated_item(def_id);
-        self.tables.assoc_container.set(def_id.index, trait_item.container);
+        self.tables.assoc_container.set_some(def_id.index, trait_item.container);
 
         match trait_item.kind {
             ty::AssocKind::Const => {}
             ty::AssocKind::Fn => {
                 record_array!(self.tables.fn_arg_names[def_id] <- tcx.fn_arg_names(def_id));
-                self.tables.asyncness.set(def_id.index, tcx.asyncness(def_id));
-                self.tables.constness.set(def_id.index, hir::Constness::NotConst);
+                self.tables.asyncness.set_some(def_id.index, tcx.asyncness(def_id));
+                self.tables.constness.set_some(def_id.index, hir::Constness::NotConst);
             }
             ty::AssocKind::Type => {
                 self.encode_explicit_item_bounds(def_id);
@@ -1351,14 +1357,14 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         let tcx = self.tcx;
 
         let ast_item = self.tcx.hir().expect_impl_item(def_id.expect_local());
-        self.tables.impl_defaultness.set(def_id.index, ast_item.defaultness);
+        self.tables.impl_defaultness.set_some(def_id.index, ast_item.defaultness);
         let impl_item = self.tcx.associated_item(def_id);
-        self.tables.assoc_container.set(def_id.index, impl_item.container);
+        self.tables.assoc_container.set_some(def_id.index, impl_item.container);
 
         match impl_item.kind {
             ty::AssocKind::Fn => {
                 let hir::ImplItemKind::Fn(ref sig, body) = ast_item.kind else { bug!() };
-                self.tables.asyncness.set(def_id.index, sig.header.asyncness);
+                self.tables.asyncness.set_some(def_id.index, sig.header.asyncness);
                 record_array!(self.tables.fn_arg_names[def_id] <- self.tcx.hir().body_param_names(body));
                 // Can be inside `impl const Trait`, so using sig.header.constness is not reliable
                 let constness = if self.tcx.is_const_fn_raw(def_id) {
@@ -1366,18 +1372,16 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 } else {
                     hir::Constness::NotConst
                 };
-                self.tables.constness.set(def_id.index, constness);
+                self.tables.constness.set_some(def_id.index, constness);
             }
             ty::AssocKind::Const | ty::AssocKind::Type => {}
         }
         if let Some(trait_item_def_id) = impl_item.trait_item_def_id {
-            self.tables.trait_item_def_id.set(def_id.index, trait_item_def_id.into());
+            self.tables.trait_item_def_id.set_some(def_id.index, trait_item_def_id.into());
         }
         if impl_item.kind == ty::AssocKind::Fn {
             record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id));
-            if tcx.is_intrinsic(def_id) {
-                self.tables.is_intrinsic.set(def_id.index, ());
-            }
+            self.tables.is_intrinsic.set(def_id.index, tcx.is_intrinsic(def_id));
         }
     }
 
@@ -1398,6 +1402,10 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             debug!("EntryBuilder::encode_mir({:?})", def_id);
             if encode_opt {
                 record!(self.tables.optimized_mir[def_id.to_def_id()] <- tcx.optimized_mir(def_id));
+
+                if tcx.sess.opts.unstable_opts.drop_tracking_mir && let DefKind::Generator = self.tcx.def_kind(def_id) {
+                    record!(self.tables.mir_generator_witnesses[def_id.to_def_id()] <- tcx.mir_generator_witnesses(def_id));
+                }
             }
             if encode_const {
                 record!(self.tables.mir_for_ctfe[def_id.to_def_id()] <- tcx.mir_for_ctfe(def_id));
@@ -1502,14 +1510,12 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
 
         match item.kind {
             hir::ItemKind::Fn(ref sig, .., body) => {
-                self.tables.asyncness.set(def_id.index, sig.header.asyncness);
+                self.tables.asyncness.set_some(def_id.index, sig.header.asyncness);
                 record_array!(self.tables.fn_arg_names[def_id] <- self.tcx.hir().body_param_names(body));
-                self.tables.constness.set(def_id.index, sig.header.constness);
+                self.tables.constness.set_some(def_id.index, sig.header.constness);
             }
             hir::ItemKind::Macro(ref macro_def, _) => {
-                if macro_def.macro_rules {
-                    self.tables.macro_rules.set(def_id.index, ());
-                }
+                self.tables.is_macro_rules.set(def_id.index, macro_def.macro_rules);
                 record!(self.tables.macro_definition[def_id] <- &*macro_def.body);
             }
             hir::ItemKind::Mod(ref m) => {
@@ -1517,47 +1523,21 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             }
             hir::ItemKind::OpaqueTy(ref opaque) => {
                 self.encode_explicit_item_bounds(def_id);
-                if matches!(opaque.origin, hir::OpaqueTyOrigin::TyAlias) {
-                    self.tables.is_type_alias_impl_trait.set(def_id.index, ());
-                }
-            }
-            hir::ItemKind::Enum(..) => {
-                let adt_def = self.tcx.adt_def(def_id);
-                record!(self.tables.repr_options[def_id] <- adt_def.repr());
-            }
-            hir::ItemKind::Struct(..) => {
-                let adt_def = self.tcx.adt_def(def_id);
-                record!(self.tables.repr_options[def_id] <- adt_def.repr());
-                self.tables.constness.set(def_id.index, hir::Constness::Const);
-
-                let variant = adt_def.non_enum_variant();
-                record!(self.tables.variant_data[def_id] <- VariantData {
-                    discr: variant.discr,
-                    ctor: variant.ctor.map(|(kind, def_id)| (kind, def_id.index)),
-                    is_non_exhaustive: variant.is_field_list_non_exhaustive(),
-                });
-            }
-            hir::ItemKind::Union(..) => {
-                let adt_def = self.tcx.adt_def(def_id);
-                record!(self.tables.repr_options[def_id] <- adt_def.repr());
-
-                let variant = adt_def.non_enum_variant();
-                record!(self.tables.variant_data[def_id] <- VariantData {
-                    discr: variant.discr,
-                    ctor: variant.ctor.map(|(kind, def_id)| (kind, def_id.index)),
-                    is_non_exhaustive: variant.is_field_list_non_exhaustive(),
-                });
+                self.tables
+                    .is_type_alias_impl_trait
+                    .set(def_id.index, matches!(opaque.origin, hir::OpaqueTyOrigin::TyAlias));
             }
             hir::ItemKind::Impl(hir::Impl { defaultness, constness, .. }) => {
-                self.tables.impl_defaultness.set(def_id.index, *defaultness);
-                self.tables.constness.set(def_id.index, *constness);
+                self.tables.impl_defaultness.set_some(def_id.index, *defaultness);
+                self.tables.constness.set_some(def_id.index, *constness);
 
-                let trait_ref = self.tcx.impl_trait_ref(def_id).map(ty::EarlyBinder::skip_binder);
+                let trait_ref = self.tcx.impl_trait_ref(def_id);
                 if let Some(trait_ref) = trait_ref {
+                    let trait_ref = trait_ref.skip_binder();
                     let trait_def = self.tcx.trait_def(trait_ref.def_id);
                     if let Ok(mut an) = trait_def.ancestors(self.tcx, def_id) {
                         if let Some(specialization_graph::Node::Impl(parent)) = an.nth(1) {
-                            self.tables.impl_parent.set(def_id.index, parent.into());
+                            self.tables.impl_parent.set_some(def_id.index, parent.into());
                         }
                     }
 
@@ -1571,7 +1551,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 }
 
                 let polarity = self.tcx.impl_polarity(def_id);
-                self.tables.impl_polarity.set(def_id.index, polarity);
+                self.tables.impl_polarity.set_some(def_id.index, polarity);
             }
             hir::ItemKind::Trait(..) => {
                 let trait_def = self.tcx.trait_def(def_id);
@@ -1586,31 +1566,15 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             }
             hir::ItemKind::Static(..)
             | hir::ItemKind::Const(..)
+            | hir::ItemKind::Enum(..)
+            | hir::ItemKind::Struct(..)
+            | hir::ItemKind::Union(..)
             | hir::ItemKind::ForeignMod { .. }
             | hir::ItemKind::GlobalAsm(..)
             | hir::ItemKind::TyAlias(..) => {}
         };
         // FIXME(eddyb) there should be a nicer way to do this.
         match item.kind {
-            hir::ItemKind::Enum(..) => {
-                record_array!(self.tables.children[def_id] <- iter::from_generator(||
-                    for variant in tcx.adt_def(def_id).variants() {
-                        yield variant.def_id.index;
-                        // Encode constructors which take a separate slot in value namespace.
-                        if let Some(ctor_def_id) = variant.ctor_def_id() {
-                            yield ctor_def_id.index;
-                        }
-                    }
-                ))
-            }
-            hir::ItemKind::Struct(..) | hir::ItemKind::Union(..) => {
-                record_array!(self.tables.children[def_id] <-
-                    self.tcx.adt_def(def_id).non_enum_variant().fields.iter().map(|f| {
-                        assert!(f.did.is_local());
-                        f.did.index
-                    })
-                )
-            }
             hir::ItemKind::Impl { .. } | hir::ItemKind::Trait(..) => {
                 let associated_item_def_ids = self.tcx.associated_item_def_ids(def_id);
                 record_array!(self.tables.children[def_id] <-
@@ -1624,9 +1588,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         }
         if let hir::ItemKind::Fn(..) = item.kind {
             record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id));
-            if tcx.is_intrinsic(def_id) {
-                self.tables.is_intrinsic.set(def_id.index, ());
-            }
+            self.tables.is_intrinsic.set(def_id.index, tcx.is_intrinsic(def_id));
         }
         if let hir::ItemKind::Impl { .. } = item.kind {
             if let Some(trait_ref) = self.tcx.impl_trait_ref(def_id) {
@@ -1638,17 +1600,6 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         // so it's easier to do that here then to wait until we would encounter
         // normally in the visitor walk.
         match item.kind {
-            hir::ItemKind::Enum(..) => {
-                let def = self.tcx.adt_def(item.owner_id.to_def_id());
-                for (i, _) in def.variants().iter_enumerated() {
-                    self.encode_enum_variant_info(def, i);
-                    self.encode_enum_variant_ctor(def, i);
-                }
-            }
-            hir::ItemKind::Struct(..) => {
-                let def = self.tcx.adt_def(item.owner_id.to_def_id());
-                self.encode_struct_ctor(def);
-            }
             hir::ItemKind::Impl { .. } => {
                 for &trait_item_def_id in
                     self.tcx.associated_item_def_ids(item.owner_id.to_def_id()).iter()
@@ -1670,7 +1621,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
     #[instrument(level = "debug", skip(self))]
     fn encode_info_for_closure(&mut self, def_id: LocalDefId) {
         // NOTE(eddyb) `tcx.type_of(def_id)` isn't used because it's fully generic,
-        // including on the signature, which is inferred in `typeck.
+        // including on the signature, which is inferred in `typeck`.
         let typeck_result: &'tcx ty::TypeckResults<'tcx> = self.tcx.typeck(def_id);
         let hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id);
         let ty = typeck_result.node_type(hir_id);
@@ -1684,8 +1635,8 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
 
             ty::Closure(_, substs) => {
                 let constness = self.tcx.constness(def_id.to_def_id());
-                self.tables.constness.set(def_id.to_def_id().index, constness);
-                record!(self.tables.fn_sig[def_id.to_def_id()] <- substs.as_closure().sig());
+                self.tables.constness.set_some(def_id.to_def_id().index, constness);
+                record!(self.tables.fn_sig[def_id.to_def_id()] <- ty::EarlyBinder(substs.as_closure().sig()));
             }
 
             _ => bug!("closure that is neither generator nor closure"),
@@ -1712,12 +1663,12 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         self.hygiene_ctxt.encode(
             &mut (&mut *self, &mut syntax_contexts, &mut expn_data_table, &mut expn_hash_table),
             |(this, syntax_contexts, _, _), index, ctxt_data| {
-                syntax_contexts.set(index, this.lazy(ctxt_data));
+                syntax_contexts.set_some(index, this.lazy(ctxt_data));
             },
             |(this, _, expn_data_table, expn_hash_table), index, expn_data, hash| {
                 if let Some(index) = index.as_local() {
-                    expn_data_table.set(index.as_raw(), this.lazy(expn_data));
-                    expn_hash_table.set(index.as_raw(), this.lazy(hash));
+                    expn_data_table.set_some(index.as_raw(), this.lazy(expn_data));
+                    expn_hash_table.set_some(index.as_raw(), this.lazy(hash));
                 }
             },
         );
@@ -1742,10 +1693,10 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             let spans = self.tcx.sess.parse_sess.proc_macro_quoted_spans();
             for (i, span) in spans.into_iter().enumerate() {
                 let span = self.lazy(span);
-                self.tables.proc_macro_quoted_spans.set(i, span);
+                self.tables.proc_macro_quoted_spans.set_some(i, span);
             }
 
-            self.tables.opt_def_kind.set(LOCAL_CRATE.as_def_id().index, DefKind::Mod);
+            self.tables.opt_def_kind.set_some(LOCAL_CRATE.as_def_id().index, DefKind::Mod);
             record!(self.tables.def_span[LOCAL_CRATE.as_def_id()] <- tcx.def_span(LOCAL_CRATE.as_def_id()));
             self.encode_attrs(LOCAL_CRATE.as_def_id().expect_local());
             let vis = tcx.local_visibility(CRATE_DEF_ID).map_id(|def_id| def_id.local_def_index);
@@ -1754,6 +1705,12 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 record!(self.tables.lookup_stability[LOCAL_CRATE.as_def_id()] <- stability);
             }
             self.encode_deprecation(LOCAL_CRATE.as_def_id());
+            if let Some(res_map) = tcx.resolutions(()).doc_link_resolutions.get(&CRATE_DEF_ID) {
+                record!(self.tables.doc_link_resolutions[LOCAL_CRATE.as_def_id()] <- res_map);
+            }
+            if let Some(traits) = tcx.resolutions(()).doc_link_traits_in_scope.get(&CRATE_DEF_ID) {
+                record_array!(self.tables.doc_link_traits_in_scope[LOCAL_CRATE.as_def_id()] <- traits);
+            }
 
             // Normally, this information is encoded when we walk the items
             // defined in this crate. However, we skip doing that for proc-macro crates,
@@ -1787,8 +1744,8 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 def_key.disambiguated_data.data = DefPathData::MacroNs(name);
 
                 let def_id = id.to_def_id();
-                self.tables.opt_def_kind.set(def_id.index, DefKind::Macro(macro_kind));
-                self.tables.proc_macro.set(def_id.index, macro_kind);
+                self.tables.opt_def_kind.set_some(def_id.index, DefKind::Macro(macro_kind));
+                self.tables.proc_macro.set_some(def_id.index, macro_kind);
                 self.encode_attrs(id);
                 record!(self.tables.def_keys[def_id] <- def_key);
                 record!(self.tables.def_ident_span[def_id] <- span);
@@ -1894,7 +1851,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             FxHashMap::default();
 
         for id in tcx.hir().items() {
-            if matches!(tcx.def_kind(id.owner_id), DefKind::Impl) {
+            if matches!(tcx.def_kind(id.owner_id), DefKind::Impl { .. }) {
                 if let Some(trait_ref) = tcx.impl_trait_ref(id.owner_id) {
                     let trait_ref = trait_ref.subst_identity();
 
@@ -2003,7 +1960,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 Linkage::Static => Some(LinkagePreference::RequireStatic),
             }));
         }
-        LazyArray::empty()
+        LazyArray::default()
     }
 
     fn encode_info_for_foreign_item(&mut self, def_id: DefId, nitem: &hir::ForeignItem<'_>) {
@@ -2013,22 +1970,20 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
 
         match nitem.kind {
             hir::ForeignItemKind::Fn(_, ref names, _) => {
-                self.tables.asyncness.set(def_id.index, hir::IsAsync::NotAsync);
+                self.tables.asyncness.set_some(def_id.index, hir::IsAsync::NotAsync);
                 record_array!(self.tables.fn_arg_names[def_id] <- *names);
                 let constness = if self.tcx.is_const_fn_raw(def_id) {
                     hir::Constness::Const
                 } else {
                     hir::Constness::NotConst
                 };
-                self.tables.constness.set(def_id.index, constness);
+                self.tables.constness.set_some(def_id.index, constness);
                 record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id));
             }
             hir::ForeignItemKind::Static(..) | hir::ForeignItemKind::Type => {}
         }
         if let hir::ForeignItemKind::Fn(..) = nitem.kind {
-            if tcx.is_intrinsic(def_id) {
-                self.tables.is_intrinsic.set(def_id.index, ());
-            }
+            self.tables.is_intrinsic.set(def_id.index, tcx.is_intrinsic(def_id));
         }
     }
 }
@@ -2266,6 +2221,18 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>, path: &Path) {
 
 pub fn provide(providers: &mut Providers) {
     *providers = Providers {
+        doc_link_resolutions: |tcx, def_id| {
+            tcx.resolutions(())
+                .doc_link_resolutions
+                .get(&def_id.expect_local())
+                .expect("no resolutions for a doc link")
+        },
+        doc_link_traits_in_scope: |tcx, def_id| {
+            tcx.resolutions(())
+                .doc_link_traits_in_scope
+                .get(&def_id.expect_local())
+                .expect("no traits in scope for a doc link")
+        },
         traits_in_crate: |tcx, cnum| {
             assert_eq!(cnum, LOCAL_CRATE);
 
@@ -2280,6 +2247,22 @@ pub fn provide(providers: &mut Providers) {
             traits.sort_by_cached_key(|&def_id| tcx.def_path_hash(def_id));
             tcx.arena.alloc_slice(&traits)
         },
+        trait_impls_in_crate: |tcx, cnum| {
+            assert_eq!(cnum, LOCAL_CRATE);
+
+            let mut trait_impls = Vec::new();
+            for id in tcx.hir().items() {
+                if matches!(tcx.def_kind(id.owner_id), DefKind::Impl { .. })
+                    && tcx.impl_trait_ref(id.owner_id).is_some()
+                {
+                    trait_impls.push(id.owner_id.to_def_id())
+                }
+            }
+
+            // Bring everything into deterministic order.
+            trait_impls.sort_by_cached_key(|&def_id| tcx.def_path_hash(def_id));
+            tcx.arena.alloc_slice(&trait_impls)
+        },
 
         ..*providers
     }
diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs
index 5066dbbb9..a7ec2d790 100644
--- a/compiler/rustc_metadata/src/rmeta/mod.rs
+++ b/compiler/rustc_metadata/src/rmeta/mod.rs
@@ -9,7 +9,7 @@ use rustc_attr as attr;
 use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::MetadataRef;
 use rustc_hir as hir;
-use rustc_hir::def::{CtorKind, DefKind};
+use rustc_hir::def::{CtorKind, DefKind, DocLinkResMap};
 use rustc_hir::def_id::{CrateNum, DefId, DefIndex, DefPathHash, StableCrateId};
 use rustc_hir::definitions::DefKey;
 use rustc_hir::lang_items::LangItem;
@@ -18,7 +18,7 @@ use rustc_index::vec::IndexVec;
 use rustc_middle::metadata::ModChild;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo};
-use rustc_middle::middle::resolve_lifetime::ObjectLifetimeDefault;
+use rustc_middle::middle::resolve_bound_vars::ObjectLifetimeDefault;
 use rustc_middle::mir;
 use rustc_middle::ty::fast_reject::SimplifiedType;
 use rustc_middle::ty::query::Providers;
@@ -115,14 +115,16 @@ impl<T: ParameterizedOverTcx> ParameterizedOverTcx for LazyArray<T> {
     type Value<'tcx> = LazyArray<T::Value<'tcx>>;
 }
 
+impl<T> Default for LazyArray<T> {
+    fn default() -> LazyArray<T> {
+        LazyArray::from_position_and_num_elems(NonZeroUsize::new(1).unwrap(), 0)
+    }
+}
+
 impl<T> LazyArray<T> {
     fn from_position_and_num_elems(position: NonZeroUsize, num_elems: usize) -> LazyArray<T> {
         LazyArray { position, num_elems, _marker: PhantomData }
     }
-
-    fn empty() -> LazyArray<T> {
-        LazyArray::from_position_and_num_elems(NonZeroUsize::new(1).unwrap(), 0)
-    }
 }
 
 /// A list of lazily-decoded values, with the added capability of random access.
@@ -185,9 +187,9 @@ enum LazyState {
     Previous(NonZeroUsize),
 }
 
-type SyntaxContextTable = LazyTable<u32, LazyValue<SyntaxContextData>>;
-type ExpnDataTable = LazyTable<ExpnIndex, LazyValue<ExpnData>>;
-type ExpnHashTable = LazyTable<ExpnIndex, LazyValue<ExpnHash>>;
+type SyntaxContextTable = LazyTable<u32, Option<LazyValue<SyntaxContextData>>>;
+type ExpnDataTable = LazyTable<ExpnIndex, Option<LazyValue<ExpnData>>>;
+type ExpnHashTable = LazyTable<ExpnIndex, Option<LazyValue<ExpnHash>>>;
 
 #[derive(MetadataEncodable, MetadataDecodable)]
 pub(crate) struct ProcMacroData {
@@ -253,7 +255,7 @@ pub(crate) struct CrateRoot {
 
     def_path_hash_map: LazyValue<DefPathHashMapRef<'static>>,
 
-    source_map: LazyTable<u32, LazyValue<rustc_span::SourceFile>>,
+    source_map: LazyTable<u32, Option<LazyValue<rustc_span::SourceFile>>>,
 
     compiler_builtins: bool,
     needs_allocator: bool,
@@ -315,21 +317,27 @@ pub(crate) struct IncoherentImpls {
 
 /// Define `LazyTables` and `TableBuilders` at the same time.
 macro_rules! define_tables {
-    ($($name:ident: Table<$IDX:ty, $T:ty>),+ $(,)?) => {
+    (
+        - defaulted: $($name1:ident: Table<$IDX1:ty, $T1:ty>,)+
+        - optional: $($name2:ident: Table<$IDX2:ty, $T2:ty>,)+
+    ) => {
         #[derive(MetadataEncodable, MetadataDecodable)]
         pub(crate) struct LazyTables {
-            $($name: LazyTable<$IDX, $T>),+
+            $($name1: LazyTable<$IDX1, $T1>,)+
+            $($name2: LazyTable<$IDX2, Option<$T2>>,)+
         }
 
         #[derive(Default)]
         struct TableBuilders {
-            $($name: TableBuilder<$IDX, $T>),+
+            $($name1: TableBuilder<$IDX1, $T1>,)+
+            $($name2: TableBuilder<$IDX2, Option<$T2>>,)+
         }
 
         impl TableBuilders {
             fn encode(&self, buf: &mut FileEncoder) -> LazyTables {
                 LazyTables {
-                    $($name: self.$name.encode(buf)),+
+                    $($name1: self.$name1.encode(buf),)+
+                    $($name2: self.$name2.encode(buf),)+
                 }
             }
         }
@@ -337,9 +345,20 @@ macro_rules! define_tables {
 }
 
 define_tables! {
+- defaulted:
+    is_intrinsic: Table<DefIndex, bool>,
+    is_macro_rules: Table<DefIndex, bool>,
+    is_type_alias_impl_trait: Table<DefIndex, bool>,
+    attr_flags: Table<DefIndex, AttrFlags>,
+    def_path_hashes: Table<DefIndex, DefPathHash>,
+    explicit_item_bounds: Table<DefIndex, LazyArray<(ty::Predicate<'static>, Span)>>,
+    inferred_outlives_of: Table<DefIndex, LazyArray<(ty::Clause<'static>, Span)>>,
+    inherent_impls: Table<DefIndex, LazyArray<DefIndex>>,
+    associated_items_for_impl_trait_in_trait: Table<DefIndex, LazyArray<DefId>>,
+
+- optional:
     attributes: Table<DefIndex, LazyArray<ast::Attribute>>,
     children: Table<DefIndex, LazyArray<DefIndex>>,
-
     opt_def_kind: Table<DefIndex, DefKind>,
     visibility: Table<DefIndex, LazyValue<ty::Visibility<DefIndex>>>,
     def_span: Table<DefIndex, LazyValue<Span>>,
@@ -348,29 +367,25 @@ define_tables! {
     lookup_const_stability: Table<DefIndex, LazyValue<attr::ConstStability>>,
     lookup_default_body_stability: Table<DefIndex, LazyValue<attr::DefaultBodyStability>>,
     lookup_deprecation_entry: Table<DefIndex, LazyValue<attr::Deprecation>>,
-    // As an optimization, a missing entry indicates an empty `&[]`.
-    explicit_item_bounds: Table<DefIndex, LazyArray<(ty::Predicate<'static>, Span)>>,
     explicit_predicates_of: Table<DefIndex, LazyValue<ty::GenericPredicates<'static>>>,
     generics_of: Table<DefIndex, LazyValue<ty::Generics>>,
-    // As an optimization, a missing entry indicates an empty `&[]`.
-    inferred_outlives_of: Table<DefIndex, LazyArray<(ty::Clause<'static>, Span)>>,
     super_predicates_of: Table<DefIndex, LazyValue<ty::GenericPredicates<'static>>>,
-    type_of: Table<DefIndex, LazyValue<Ty<'static>>>,
+    type_of: Table<DefIndex, LazyValue<ty::EarlyBinder<Ty<'static>>>>,
     variances_of: Table<DefIndex, LazyArray<ty::Variance>>,
-    fn_sig: Table<DefIndex, LazyValue<ty::PolyFnSig<'static>>>,
+    fn_sig: Table<DefIndex, LazyValue<ty::EarlyBinder<ty::PolyFnSig<'static>>>>,
     codegen_fn_attrs: Table<DefIndex, LazyValue<CodegenFnAttrs>>,
     impl_trait_ref: Table<DefIndex, LazyValue<ty::EarlyBinder<ty::TraitRef<'static>>>>,
     const_param_default: Table<DefIndex, LazyValue<ty::EarlyBinder<rustc_middle::ty::Const<'static>>>>,
     object_lifetime_default: Table<DefIndex, LazyValue<ObjectLifetimeDefault>>,
     optimized_mir: Table<DefIndex, LazyValue<mir::Body<'static>>>,
     mir_for_ctfe: Table<DefIndex, LazyValue<mir::Body<'static>>>,
+    mir_generator_witnesses: Table<DefIndex, LazyValue<mir::GeneratorLayout<'static>>>,
     promoted_mir: Table<DefIndex, LazyValue<IndexVec<mir::Promoted, mir::Body<'static>>>>,
     // FIXME(compiler-errors): Why isn't this a LazyArray?
     thir_abstract_const: Table<DefIndex, LazyValue<ty::Const<'static>>>,
     impl_parent: Table<DefIndex, RawDefId>,
     impl_polarity: Table<DefIndex, ty::ImplPolarity>,
     constness: Table<DefIndex, hir::Constness>,
-    is_intrinsic: Table<DefIndex, ()>,
     impl_defaultness: Table<DefIndex, hir::Defaultness>,
     // FIXME(eddyb) perhaps compute this on the fly if cheap enough?
     coerce_unsized_info: Table<DefIndex, LazyValue<ty::adjustment::CoerceUnsizedInfo>>,
@@ -380,9 +395,7 @@ define_tables! {
     fn_arg_names: Table<DefIndex, LazyArray<Ident>>,
     generator_kind: Table<DefIndex, LazyValue<hir::GeneratorKind>>,
     trait_def: Table<DefIndex, LazyValue<ty::TraitDef>>,
-
     trait_item_def_id: Table<DefIndex, RawDefId>,
-    inherent_impls: Table<DefIndex, LazyArray<DefIndex>>,
     expn_that_defined: Table<DefIndex, LazyValue<ExpnId>>,
     unused_generic_params: Table<DefIndex, LazyValue<UnusedGenericParams>>,
     params_in_repr: Table<DefIndex, LazyValue<BitSet<u32>>>,
@@ -392,22 +405,17 @@ define_tables! {
     // `DefPathTable` up front, since we may only ever use a few
     // definitions from any given crate.
     def_keys: Table<DefIndex, LazyValue<DefKey>>,
-    def_path_hashes: Table<DefIndex, DefPathHash>,
     proc_macro_quoted_spans: Table<usize, LazyValue<Span>>,
     generator_diagnostic_data: Table<DefIndex, LazyValue<GeneratorDiagnosticData<'static>>>,
-    may_have_doc_links: Table<DefIndex, ()>,
     variant_data: Table<DefIndex, LazyValue<VariantData>>,
     assoc_container: Table<DefIndex, ty::AssocItemContainer>,
-    // Slot is full when macro is macro_rules.
-    macro_rules: Table<DefIndex, ()>,
     macro_definition: Table<DefIndex, LazyValue<ast::DelimArgs>>,
     proc_macro: Table<DefIndex, MacroKind>,
     module_reexports: Table<DefIndex, LazyArray<ModChild>>,
     deduced_param_attrs: Table<DefIndex, LazyArray<DeducedParamAttrs>>,
-    // Slot is full when opaque is TAIT.
-    is_type_alias_impl_trait: Table<DefIndex, ()>,
-
     trait_impl_trait_tys: Table<DefIndex, LazyValue<FxHashMap<DefId, Ty<'static>>>>,
+    doc_link_resolutions: Table<DefIndex, LazyValue<DocLinkResMap>>,
+    doc_link_traits_in_scope: Table<DefIndex, LazyArray<DefId>>,
 }
 
 #[derive(TyEncodable, TyDecodable)]
@@ -418,6 +426,13 @@ struct VariantData {
     is_non_exhaustive: bool,
 }
 
+bitflags::bitflags! {
+    #[derive(Default)]
+    pub struct AttrFlags: u8 {
+        const IS_DOC_HIDDEN = 1 << 0;
+    }
+}
+
 // Tags used for encoding Spans:
 const TAG_VALID_SPAN_LOCAL: u8 = 0;
 const TAG_VALID_SPAN_FOREIGN: u8 = 1;
@@ -440,4 +455,5 @@ trivially_parameterized_over_tcx! {
     IncoherentImpls,
     CrateRoot,
     CrateDep,
+    AttrFlags,
 }
diff --git a/compiler/rustc_metadata/src/rmeta/table.rs b/compiler/rustc_metadata/src/rmeta/table.rs
index 716655c7f..b89d48ec1 100644
--- a/compiler/rustc_metadata/src/rmeta/table.rs
+++ b/compiler/rustc_metadata/src/rmeta/table.rs
@@ -10,18 +10,61 @@ use rustc_span::hygiene::MacroKind;
 use std::marker::PhantomData;
 use std::num::NonZeroUsize;
 
+pub(super) trait IsDefault: Default {
+    fn is_default(&self) -> bool;
+}
+
+impl<T> IsDefault for Option<T> {
+    fn is_default(&self) -> bool {
+        self.is_none()
+    }
+}
+
+impl IsDefault for AttrFlags {
+    fn is_default(&self) -> bool {
+        self.is_empty()
+    }
+}
+
+impl IsDefault for bool {
+    fn is_default(&self) -> bool {
+        !self
+    }
+}
+
+impl IsDefault for u32 {
+    fn is_default(&self) -> bool {
+        *self == 0
+    }
+}
+
+impl<T> IsDefault for LazyArray<T> {
+    fn is_default(&self) -> bool {
+        self.num_elems == 0
+    }
+}
+
+impl IsDefault for DefPathHash {
+    fn is_default(&self) -> bool {
+        self.0 == Fingerprint::ZERO
+    }
+}
+
 /// Helper trait, for encoding to, and decoding from, a fixed number of bytes.
 /// Used mainly for Lazy positions and lengths.
 /// Unchecked invariant: `Self::default()` should encode as `[0; BYTE_LEN]`,
 /// but this has no impact on safety.
-pub(super) trait FixedSizeEncoding: Default {
+pub(super) trait FixedSizeEncoding: IsDefault {
     /// This should be `[u8; BYTE_LEN]`;
+    /// Cannot use an associated `const BYTE_LEN: usize` instead due to const eval limitations.
     type ByteArray;
 
     fn from_bytes(b: &Self::ByteArray) -> Self;
     fn write_to_bytes(self, b: &mut Self::ByteArray);
 }
 
+/// This implementation is not used generically, but for reading/writing
+/// concrete `u32` fields in `Lazy*` structures, which may be zero.
 impl FixedSizeEncoding for u32 {
     type ByteArray = [u8; 4];
 
@@ -57,7 +100,7 @@ macro_rules! fixed_size_enum {
             fn write_to_bytes(self, b: &mut [u8;1]) {
                 use $ty::*;
                 b[0] = match self {
-                    None => 0,
+                    None => unreachable!(),
                     $(Some($($pat)*) => 1 + ${index()},)*
                 }
             }
@@ -93,7 +136,8 @@ fixed_size_enum! {
         ( Field                                    )
         ( LifetimeParam                            )
         ( GlobalAsm                                )
-        ( Impl                                     )
+        ( Impl { of_trait: false }                 )
+        ( Impl { of_trait: true }                  )
         ( Closure                                  )
         ( Generator                                )
         ( Static(ast::Mutability::Not)             )
@@ -154,20 +198,18 @@ fixed_size_enum! {
 }
 
 // We directly encode `DefPathHash` because a `LazyValue` would incur a 25% cost.
-impl FixedSizeEncoding for Option<DefPathHash> {
+impl FixedSizeEncoding for DefPathHash {
     type ByteArray = [u8; 16];
 
     #[inline]
     fn from_bytes(b: &[u8; 16]) -> Self {
-        Some(DefPathHash(Fingerprint::from_le_bytes(*b)))
+        DefPathHash(Fingerprint::from_le_bytes(*b))
     }
 
     #[inline]
     fn write_to_bytes(self, b: &mut [u8; 16]) {
-        let Some(DefPathHash(fingerprint)) = self else {
-            panic!("Trying to encode absent DefPathHash.")
-        };
-        *b = fingerprint.to_le_bytes();
+        debug_assert!(!self.is_default());
+        *b = self.0.to_le_bytes();
     }
 }
 
@@ -178,17 +220,17 @@ impl FixedSizeEncoding for Option<RawDefId> {
     #[inline]
     fn from_bytes(b: &[u8; 8]) -> Self {
         let krate = u32::from_le_bytes(b[0..4].try_into().unwrap());
-        let index = u32::from_le_bytes(b[4..8].try_into().unwrap());
         if krate == 0 {
             return None;
         }
+        let index = u32::from_le_bytes(b[4..8].try_into().unwrap());
         Some(RawDefId { krate: krate - 1, index })
     }
 
     #[inline]
     fn write_to_bytes(self, b: &mut [u8; 8]) {
         match self {
-            None => *b = [0; 8],
+            None => unreachable!(),
             Some(RawDefId { krate, index }) => {
                 // CrateNum is less than `CrateNum::MAX_AS_U32`.
                 debug_assert!(krate < u32::MAX);
@@ -199,17 +241,33 @@ impl FixedSizeEncoding for Option<RawDefId> {
     }
 }
 
-impl FixedSizeEncoding for Option<()> {
+impl FixedSizeEncoding for AttrFlags {
+    type ByteArray = [u8; 1];
+
+    #[inline]
+    fn from_bytes(b: &[u8; 1]) -> Self {
+        AttrFlags::from_bits_truncate(b[0])
+    }
+
+    #[inline]
+    fn write_to_bytes(self, b: &mut [u8; 1]) {
+        debug_assert!(!self.is_default());
+        b[0] = self.bits();
+    }
+}
+
+impl FixedSizeEncoding for bool {
     type ByteArray = [u8; 1];
 
     #[inline]
     fn from_bytes(b: &[u8; 1]) -> Self {
-        (b[0] != 0).then(|| ())
+        b[0] != 0
     }
 
     #[inline]
     fn write_to_bytes(self, b: &mut [u8; 1]) {
-        b[0] = self.is_some() as u8
+        debug_assert!(!self.is_default());
+        b[0] = self as u8
     }
 }
 
@@ -227,76 +285,115 @@ impl<T> FixedSizeEncoding for Option<LazyValue<T>> {
 
     #[inline]
     fn write_to_bytes(self, b: &mut [u8; 4]) {
-        let position = self.map_or(0, |lazy| lazy.position.get());
+        match self {
+            None => unreachable!(),
+            Some(lazy) => {
+                let position = lazy.position.get();
+                let position: u32 = position.try_into().unwrap();
+                position.write_to_bytes(b)
+            }
+        }
+    }
+}
+
+impl<T> LazyArray<T> {
+    #[inline]
+    fn write_to_bytes_impl(self, b: &mut [u8; 8]) {
+        let ([position_bytes, meta_bytes],[])= b.as_chunks_mut::<4>() else { panic!() };
+
+        let position = self.position.get();
         let position: u32 = position.try_into().unwrap();
-        position.write_to_bytes(b)
+        position.write_to_bytes(position_bytes);
+
+        let len = self.num_elems;
+        let len: u32 = len.try_into().unwrap();
+        len.write_to_bytes(meta_bytes);
+    }
+
+    fn from_bytes_impl(position_bytes: &[u8; 4], meta_bytes: &[u8; 4]) -> Option<LazyArray<T>> {
+        let position = NonZeroUsize::new(u32::from_bytes(position_bytes) as usize)?;
+        let len = u32::from_bytes(meta_bytes) as usize;
+        Some(LazyArray::from_position_and_num_elems(position, len))
     }
 }
 
-impl<T> FixedSizeEncoding for Option<LazyArray<T>> {
+impl<T> FixedSizeEncoding for LazyArray<T> {
     type ByteArray = [u8; 8];
 
     #[inline]
     fn from_bytes(b: &[u8; 8]) -> Self {
-        let ([ref position_bytes, ref meta_bytes],[])= b.as_chunks::<4>() else { panic!() };
-        let position = NonZeroUsize::new(u32::from_bytes(position_bytes) as usize)?;
-        let len = u32::from_bytes(meta_bytes) as usize;
-        Some(LazyArray::from_position_and_num_elems(position, len))
+        let ([position_bytes, meta_bytes],[])= b.as_chunks::<4>() else { panic!() };
+        if *meta_bytes == [0; 4] {
+            return Default::default();
+        }
+        LazyArray::from_bytes_impl(position_bytes, meta_bytes).unwrap()
     }
 
     #[inline]
     fn write_to_bytes(self, b: &mut [u8; 8]) {
-        let ([ref mut position_bytes, ref mut meta_bytes],[])= b.as_chunks_mut::<4>() else { panic!() };
+        assert!(!self.is_default());
+        self.write_to_bytes_impl(b)
+    }
+}
 
-        let position = self.map_or(0, |lazy| lazy.position.get());
-        let position: u32 = position.try_into().unwrap();
-        position.write_to_bytes(position_bytes);
+impl<T> FixedSizeEncoding for Option<LazyArray<T>> {
+    type ByteArray = [u8; 8];
 
-        let len = self.map_or(0, |lazy| lazy.num_elems);
-        let len: u32 = len.try_into().unwrap();
-        len.write_to_bytes(meta_bytes);
+    #[inline]
+    fn from_bytes(b: &[u8; 8]) -> Self {
+        let ([position_bytes, meta_bytes],[])= b.as_chunks::<4>() else { panic!() };
+        LazyArray::from_bytes_impl(position_bytes, meta_bytes)
+    }
+
+    #[inline]
+    fn write_to_bytes(self, b: &mut [u8; 8]) {
+        match self {
+            None => unreachable!(),
+            Some(lazy) => lazy.write_to_bytes_impl(b),
+        }
     }
 }
 
 /// Helper for constructing a table's serialization (also see `Table`).
-pub(super) struct TableBuilder<I: Idx, T>
-where
-    Option<T>: FixedSizeEncoding,
-{
-    blocks: IndexVec<I, <Option<T> as FixedSizeEncoding>::ByteArray>,
+pub(super) struct TableBuilder<I: Idx, T: FixedSizeEncoding> {
+    blocks: IndexVec<I, T::ByteArray>,
     _marker: PhantomData<T>,
 }
 
-impl<I: Idx, T> Default for TableBuilder<I, T>
-where
-    Option<T>: FixedSizeEncoding,
-{
+impl<I: Idx, T: FixedSizeEncoding> Default for TableBuilder<I, T> {
     fn default() -> Self {
         TableBuilder { blocks: Default::default(), _marker: PhantomData }
     }
 }
 
-impl<I: Idx, T> TableBuilder<I, T>
+impl<I: Idx, const N: usize, T> TableBuilder<I, Option<T>>
 where
-    Option<T>: FixedSizeEncoding,
+    Option<T>: FixedSizeEncoding<ByteArray = [u8; N]>,
 {
-    pub(crate) fn set<const N: usize>(&mut self, i: I, value: T)
-    where
-        Option<T>: FixedSizeEncoding<ByteArray = [u8; N]>,
-    {
-        // FIXME(eddyb) investigate more compact encodings for sparse tables.
-        // On the PR @michaelwoerister mentioned:
-        // > Space requirements could perhaps be optimized by using the HAMT `popcnt`
-        // > trick (i.e. divide things into buckets of 32 or 64 items and then
-        // > store bit-masks of which item in each bucket is actually serialized).
-        self.blocks.ensure_contains_elem(i, || [0; N]);
-        Some(value).write_to_bytes(&mut self.blocks[i]);
-    }
-
-    pub(crate) fn encode<const N: usize>(&self, buf: &mut FileEncoder) -> LazyTable<I, T>
-    where
-        Option<T>: FixedSizeEncoding<ByteArray = [u8; N]>,
-    {
+    pub(crate) fn set_some(&mut self, i: I, value: T) {
+        self.set(i, Some(value))
+    }
+}
+
+impl<I: Idx, const N: usize, T: FixedSizeEncoding<ByteArray = [u8; N]>> TableBuilder<I, T> {
+    /// Sets the table value if it is not default.
+    /// ATTENTION: For optimization default values are simply ignored by this function, because
+    /// right now metadata tables never need to reset non-default values to default. If such need
+    /// arises in the future then a new method (e.g. `clear` or `reset`) will need to be introduced
+    /// for doing that explicitly.
+    pub(crate) fn set(&mut self, i: I, value: T) {
+        if !value.is_default() {
+            // FIXME(eddyb) investigate more compact encodings for sparse tables.
+            // On the PR @michaelwoerister mentioned:
+            // > Space requirements could perhaps be optimized by using the HAMT `popcnt`
+            // > trick (i.e. divide things into buckets of 32 or 64 items and then
+            // > store bit-masks of which item in each bucket is actually serialized).
+            self.blocks.ensure_contains_elem(i, || [0; N]);
+            value.write_to_bytes(&mut self.blocks[i]);
+        }
+    }
+
+    pub(crate) fn encode(&self, buf: &mut FileEncoder) -> LazyTable<I, T> {
         let pos = buf.position();
         for block in &self.blocks {
             buf.emit_raw_bytes(block);
@@ -309,34 +406,24 @@ where
     }
 }
 
-impl<I: Idx, T: ParameterizedOverTcx> LazyTable<I, T>
+impl<I: Idx, const N: usize, T: FixedSizeEncoding<ByteArray = [u8; N]> + ParameterizedOverTcx>
+    LazyTable<I, T>
 where
-    Option<T>: FixedSizeEncoding,
+    for<'tcx> T::Value<'tcx>: FixedSizeEncoding<ByteArray = [u8; N]>,
 {
     /// Given the metadata, extract out the value at a particular index (if any).
     #[inline(never)]
-    pub(super) fn get<'a, 'tcx, M: Metadata<'a, 'tcx>, const N: usize>(
-        &self,
-        metadata: M,
-        i: I,
-    ) -> Option<T::Value<'tcx>>
-    where
-        Option<T::Value<'tcx>>: FixedSizeEncoding<ByteArray = [u8; N]>,
-    {
+    pub(super) fn get<'a, 'tcx, M: Metadata<'a, 'tcx>>(&self, metadata: M, i: I) -> T::Value<'tcx> {
         debug!("LazyTable::lookup: index={:?} len={:?}", i, self.encoded_size);
 
         let start = self.position.get();
         let bytes = &metadata.blob()[start..start + self.encoded_size];
         let (bytes, []) = bytes.as_chunks::<N>() else { panic!() };
-        let bytes = bytes.get(i.index())?;
-        FixedSizeEncoding::from_bytes(bytes)
+        bytes.get(i.index()).map_or_else(Default::default, FixedSizeEncoding::from_bytes)
     }
 
     /// Size of the table in entries, including possible gaps.
-    pub(super) fn size<const N: usize>(&self) -> usize
-    where
-        for<'tcx> Option<T::Value<'tcx>>: FixedSizeEncoding<ByteArray = [u8; N]>,
-    {
+    pub(super) fn size(&self) -> usize {
         self.encoded_size / N
     }
 }
diff --git a/compiler/rustc_middle/Cargo.toml b/compiler/rustc_middle/Cargo.toml
index 543bd56a2..a2b78cc29 100644
--- a/compiler/rustc_middle/Cargo.toml
+++ b/compiler/rustc_middle/Cargo.toml
@@ -34,7 +34,7 @@ rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 rustc_type_ir = { path = "../rustc_type_ir" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-thin-vec = "0.2.9"
+thin-vec = "0.2.12"
 tracing = "0.1"
 
 [features]
diff --git a/compiler/rustc_error_messages/locales/en-US/middle.ftl b/compiler/rustc_middle/locales/en-US.ftl
index 4f4e5c6a2..4f4e5c6a2 100644
--- a/compiler/rustc_error_messages/locales/en-US/middle.ftl
+++ b/compiler/rustc_middle/locales/en-US.ftl
diff --git a/compiler/rustc_middle/src/arena.rs b/compiler/rustc_middle/src/arena.rs
index f816d6145..62e44b629 100644
--- a/compiler/rustc_middle/src/arena.rs
+++ b/compiler/rustc_middle/src/arena.rs
@@ -8,7 +8,7 @@
 macro_rules! arena_types {
     ($macro:path) => (
         $macro!([
-            [] layout: rustc_target::abi::LayoutS<rustc_target::abi::VariantIdx>,
+            [] layout: rustc_target::abi::LayoutS,
             [] fn_abi: rustc_target::abi::call::FnAbi<'tcx, rustc_middle::ty::Ty<'tcx>>,
             // AdtDef are interned and compared by address
             [decode] adt_def: rustc_middle::ty::AdtDefData,
@@ -35,6 +35,8 @@ macro_rules! arena_types {
                 rustc_data_structures::sync::Lrc<rustc_ast::Crate>,
             )>,
             [] output_filenames: std::sync::Arc<rustc_session::config::OutputFilenames>,
+            [] metadata_loader: rustc_data_structures::steal::Steal<Box<rustc_session::cstore::MetadataLoaderDyn>>,
+            [] crate_for_resolver: rustc_data_structures::steal::Steal<rustc_ast::ast::Crate>,
             [] resolutions: rustc_middle::ty::ResolverGlobalCtxt,
             [decode] unsafety_check_result: rustc_middle::mir::UnsafetyCheckResult,
             [decode] code_region: rustc_middle::mir::coverage::CodeRegion,
@@ -105,13 +107,16 @@ macro_rules! arena_types {
             // (during lowering) and the `librustc_middle` arena (for decoding MIR)
             [decode] asm_template: rustc_ast::InlineAsmTemplatePiece,
             [decode] used_trait_imports: rustc_data_structures::unord::UnordSet<rustc_hir::def_id::LocalDefId>,
-            [decode] is_late_bound_map: rustc_data_structures::fx::FxIndexSet<rustc_hir::def_id::LocalDefId>,
+            [decode] is_late_bound_map: rustc_data_structures::fx::FxIndexSet<rustc_hir::ItemLocalId>,
             [decode] impl_source: rustc_middle::traits::ImplSource<'tcx, ()>,
 
             [] dep_kind: rustc_middle::dep_graph::DepKindStruct<'tcx>,
 
             [decode] trait_impl_trait_tys: rustc_data_structures::fx::FxHashMap<rustc_hir::def_id::DefId, rustc_middle::ty::Ty<'tcx>>,
             [] bit_set_u32: rustc_index::bit_set::BitSet<u32>,
+            [] external_constraints: rustc_middle::traits::solve::ExternalConstraintsData<'tcx>,
+            [decode] doc_link_resolutions: rustc_hir::def::DocLinkResMap,
+            [] closure_kind_origin: (rustc_span::Span, rustc_middle::hir::place::Place<'tcx>),
         ]);
     )
 }
diff --git a/compiler/rustc_middle/src/dep_graph/mod.rs b/compiler/rustc_middle/src/dep_graph/mod.rs
index 2e62bebc8..84510fe21 100644
--- a/compiler/rustc_middle/src/dep_graph/mod.rs
+++ b/compiler/rustc_middle/src/dep_graph/mod.rs
@@ -55,7 +55,7 @@ impl rustc_query_system::dep_graph::DepKind for DepKind {
         ty::tls::with_context(|icx| {
             let icx = ty::tls::ImplicitCtxt { task_deps, ..icx.clone() };
 
-            ty::tls::enter_context(&icx, |_| op())
+            ty::tls::enter_context(&icx, op)
         })
     }
 
@@ -74,8 +74,8 @@ impl<'tcx> DepContext for TyCtxt<'tcx> {
     type DepKind = DepKind;
 
     #[inline]
-    fn with_stable_hashing_context<R>(&self, f: impl FnOnce(StableHashingContext<'_>) -> R) -> R {
-        TyCtxt::with_stable_hashing_context(*self, f)
+    fn with_stable_hashing_context<R>(self, f: impl FnOnce(StableHashingContext<'_>) -> R) -> R {
+        TyCtxt::with_stable_hashing_context(self, f)
     }
 
     #[inline]
diff --git a/compiler/rustc_middle/src/hir/map/mod.rs b/compiler/rustc_middle/src/hir/map/mod.rs
index 9e63c2bd2..4b5bacac8 100644
--- a/compiler/rustc_middle/src/hir/map/mod.rs
+++ b/compiler/rustc_middle/src/hir/map/mod.rs
@@ -18,24 +18,30 @@ use rustc_span::Span;
 use rustc_target::spec::abi::Abi;
 
 #[inline]
-pub fn associated_body(node: Node<'_>) -> Option<BodyId> {
+pub fn associated_body(node: Node<'_>) -> Option<(LocalDefId, BodyId)> {
     match node {
         Node::Item(Item {
+            owner_id,
             kind: ItemKind::Const(_, body) | ItemKind::Static(.., body) | ItemKind::Fn(.., body),
             ..
         })
         | Node::TraitItem(TraitItem {
+            owner_id,
             kind:
                 TraitItemKind::Const(_, Some(body)) | TraitItemKind::Fn(_, TraitFn::Provided(body)),
             ..
         })
         | Node::ImplItem(ImplItem {
+            owner_id,
             kind: ImplItemKind::Const(_, body) | ImplItemKind::Fn(_, body),
             ..
-        })
-        | Node::Expr(Expr { kind: ExprKind::Closure(Closure { body, .. }), .. }) => Some(*body),
+        }) => Some((owner_id.def_id, *body)),
+
+        Node::Expr(Expr { kind: ExprKind::Closure(Closure { def_id, body, .. }), .. }) => {
+            Some((*def_id, *body))
+        }
 
-        Node::AnonConst(constant) => Some(constant.body),
+        Node::AnonConst(constant) => Some((constant.def_id, constant.body)),
 
         _ => None,
     }
@@ -43,7 +49,7 @@ pub fn associated_body(node: Node<'_>) -> Option<BodyId> {
 
 fn is_body_owner(node: Node<'_>, hir_id: HirId) -> bool {
     match associated_body(node) {
-        Some(b) => b.hir_id == hir_id,
+        Some((_, b)) => b.hir_id == hir_id,
         None => false,
     }
 }
@@ -154,10 +160,6 @@ impl<'hir> Map<'hir> {
         self.tcx.definitions_untracked().def_key(def_id)
     }
 
-    pub fn def_path_from_hir_id(self, id: HirId) -> Option<DefPath> {
-        self.opt_local_def_id(id).map(|def_id| self.def_path(def_id))
-    }
-
     pub fn def_path(self, def_id: LocalDefId) -> DefPath {
         // Accessing the DefPath is ok, since it is part of DefPathHash.
         self.tcx.definitions_untracked().def_path(def_id)
@@ -170,32 +172,6 @@ impl<'hir> Map<'hir> {
     }
 
     #[inline]
-    #[track_caller]
-    pub fn local_def_id(self, hir_id: HirId) -> LocalDefId {
-        self.opt_local_def_id(hir_id).unwrap_or_else(|| {
-            bug!(
-                "local_def_id: no entry for `{:?}`, which has a map of `{:?}`",
-                hir_id,
-                self.find(hir_id)
-            )
-        })
-    }
-
-    #[inline]
-    pub fn opt_local_def_id(self, hir_id: HirId) -> Option<LocalDefId> {
-        if hir_id.local_id == ItemLocalId::new(0) {
-            Some(hir_id.owner.def_id)
-        } else {
-            self.tcx
-                .hir_owner_nodes(hir_id.owner)
-                .as_owner()?
-                .local_id_to_def_id
-                .get(&hir_id.local_id)
-                .copied()
-        }
-    }
-
-    #[inline]
     pub fn local_def_id_to_hir_id(self, def_id: LocalDefId) -> HirId {
         self.tcx.local_def_id_to_hir_id(def_id)
     }
@@ -227,7 +203,7 @@ impl<'hir> Map<'hir> {
                 ItemKind::Use(..) => DefKind::Use,
                 ItemKind::ForeignMod { .. } => DefKind::ForeignMod,
                 ItemKind::GlobalAsm(..) => DefKind::GlobalAsm,
-                ItemKind::Impl { .. } => DefKind::Impl,
+                ItemKind::Impl(impl_) => DefKind::Impl { of_trait: impl_.of_trait.is_some() },
             },
             Node::ForeignItem(item) => match item.kind {
                 ForeignItemKind::Fn(..) => DefKind::Fn,
@@ -314,7 +290,7 @@ impl<'hir> Map<'hir> {
     #[track_caller]
     pub fn parent_id(self, hir_id: HirId) -> HirId {
         self.opt_parent_id(hir_id)
-            .unwrap_or_else(|| bug!("No parent for node {:?}", self.node_to_string(hir_id)))
+            .unwrap_or_else(|| bug!("No parent for node {}", self.node_to_string(hir_id)))
     }
 
     pub fn get_parent(self, hir_id: HirId) -> Node<'hir> {
@@ -410,8 +386,8 @@ impl<'hir> Map<'hir> {
     #[track_caller]
     pub fn enclosing_body_owner(self, hir_id: HirId) -> LocalDefId {
         for (_, node) in self.parent_iter(hir_id) {
-            if let Some(body) = associated_body(node) {
-                return self.body_owner_def_id(body);
+            if let Some((def_id, _)) = associated_body(node) {
+                return def_id;
             }
         }
 
@@ -427,14 +403,17 @@ impl<'hir> Map<'hir> {
         parent
     }
 
-    pub fn body_owner_def_id(self, id: BodyId) -> LocalDefId {
-        self.local_def_id(self.body_owner(id))
+    pub fn body_owner_def_id(self, BodyId { hir_id }: BodyId) -> LocalDefId {
+        let parent = self.parent_id(hir_id);
+        associated_body(self.get(parent)).unwrap().0
     }
 
     /// Given a `LocalDefId`, returns the `BodyId` associated with it,
     /// if the node is a body owner, otherwise returns `None`.
     pub fn maybe_body_owned_by(self, id: LocalDefId) -> Option<BodyId> {
-        self.find_by_def_id(id).and_then(associated_body)
+        let node = self.find_by_def_id(id)?;
+        let (_, body_id) = associated_body(node)?;
+        Some(body_id)
     }
 
     /// Given a body owner's id, returns the `BodyId` associated with it.
@@ -870,6 +849,13 @@ impl<'hir> Map<'hir> {
         }
     }
 
+    pub fn get_fn_output(self, def_id: LocalDefId) -> Option<&'hir FnRetTy<'hir>> {
+        match self.tcx.hir_owner(OwnerId { def_id }) {
+            Some(Owner { node, .. }) => node.fn_decl().map(|fn_decl| &fn_decl.output),
+            _ => None,
+        }
+    }
+
     pub fn expect_variant(self, id: HirId) -> &'hir Variant<'hir> {
         match self.find(id) {
             Some(Node::Variant(variant)) => variant,
@@ -917,6 +903,11 @@ impl<'hir> Map<'hir> {
     }
 
     #[inline]
+    pub fn ident(self, id: HirId) -> Ident {
+        self.opt_ident(id).unwrap()
+    }
+
+    #[inline]
     pub fn opt_name(self, id: HirId) -> Option<Symbol> {
         self.opt_ident(id).map(|ident| ident.name)
     }
@@ -1083,7 +1074,7 @@ impl<'hir> Map<'hir> {
     }
 
     pub fn span_if_local(self, id: DefId) -> Option<Span> {
-        if id.is_local() { Some(self.tcx.def_span(id)) } else { None }
+        id.is_local().then(|| self.tcx.def_span(id))
     }
 
     pub fn res_span(self, res: Res) -> Option<Span> {
@@ -1212,12 +1203,10 @@ fn upstream_crates(tcx: TyCtxt<'_>) -> Vec<(StableCrateId, Svh)> {
 }
 
 fn hir_id_to_string(map: Map<'_>, id: HirId) -> String {
-    let id_str = format!(" (hir_id={})", id);
-
     let path_str = |def_id: LocalDefId| map.tcx.def_path_str(def_id.to_def_id());
 
     let span_str = || map.tcx.sess.source_map().span_to_snippet(map.span(id)).unwrap_or_default();
-    let node_str = |prefix| format!("{} {}{}", prefix, span_str(), id_str);
+    let node_str = |prefix| format!("{id} ({prefix} `{}`)", span_str());
 
     match map.find(id) {
         Some(Node::Item(item)) => {
@@ -1246,10 +1235,10 @@ fn hir_id_to_string(map: Map<'_>, id: HirId) -> String {
                 ItemKind::TraitAlias(..) => "trait alias",
                 ItemKind::Impl { .. } => "impl",
             };
-            format!("{} {}{}", item_str, path_str(item.owner_id.def_id), id_str)
+            format!("{id} ({item_str} {})", path_str(item.owner_id.def_id))
         }
         Some(Node::ForeignItem(item)) => {
-            format!("foreign item {}{}", path_str(item.owner_id.def_id), id_str)
+            format!("{id} (foreign item {})", path_str(item.owner_id.def_id))
         }
         Some(Node::ImplItem(ii)) => {
             let kind = match ii.kind {
@@ -1257,7 +1246,7 @@ fn hir_id_to_string(map: Map<'_>, id: HirId) -> String {
                 ImplItemKind::Fn(..) => "method",
                 ImplItemKind::Type(_) => "assoc type",
             };
-            format!("{} {} in {}{}", kind, ii.ident, path_str(ii.owner_id.def_id), id_str)
+            format!("{id} ({kind} `{}` in {})", ii.ident, path_str(ii.owner_id.def_id))
         }
         Some(Node::TraitItem(ti)) => {
             let kind = match ti.kind {
@@ -1266,13 +1255,13 @@ fn hir_id_to_string(map: Map<'_>, id: HirId) -> String {
                 TraitItemKind::Type(..) => "assoc type",
             };
 
-            format!("{} {} in {}{}", kind, ti.ident, path_str(ti.owner_id.def_id), id_str)
+            format!("{id} ({kind} `{}` in {})", ti.ident, path_str(ti.owner_id.def_id))
         }
         Some(Node::Variant(ref variant)) => {
-            format!("variant {} in {}{}", variant.ident, path_str(variant.def_id), id_str)
+            format!("{id} (variant `{}` in {})", variant.ident, path_str(variant.def_id))
         }
         Some(Node::Field(ref field)) => {
-            format!("field {} in {}{}", field.ident, path_str(field.def_id), id_str)
+            format!("{id} (field `{}` in {})", field.ident, path_str(field.def_id))
         }
         Some(Node::AnonConst(_)) => node_str("const"),
         Some(Node::Expr(_)) => node_str("expr"),
@@ -1290,16 +1279,15 @@ fn hir_id_to_string(map: Map<'_>, id: HirId) -> String {
         Some(Node::Infer(_)) => node_str("infer"),
         Some(Node::Local(_)) => node_str("local"),
         Some(Node::Ctor(ctor)) => format!(
-            "ctor {}{}",
+            "{id} (ctor {})",
             ctor.ctor_def_id().map_or("<missing path>".into(), |def_id| path_str(def_id)),
-            id_str
         ),
         Some(Node::Lifetime(_)) => node_str("lifetime"),
         Some(Node::GenericParam(ref param)) => {
-            format!("generic_param {}{}", path_str(param.def_id), id_str)
+            format!("{id} (generic_param {})", path_str(param.def_id))
         }
-        Some(Node::Crate(..)) => String::from("root_crate"),
-        None => format!("unknown node{}", id_str),
+        Some(Node::Crate(..)) => String::from("(root_crate)"),
+        None => format!("{id} (unknown node)"),
     }
 }
 
diff --git a/compiler/rustc_middle/src/hir/mod.rs b/compiler/rustc_middle/src/hir/mod.rs
index dedc65f4c..c9da711e5 100644
--- a/compiler/rustc_middle/src/hir/mod.rs
+++ b/compiler/rustc_middle/src/hir/mod.rs
@@ -64,13 +64,17 @@ impl ModuleItems {
         self.foreign_items.iter().copied()
     }
 
-    pub fn definitions(&self) -> impl Iterator<Item = LocalDefId> + '_ {
+    pub fn owners(&self) -> impl Iterator<Item = OwnerId> + '_ {
         self.items
             .iter()
-            .map(|id| id.owner_id.def_id)
-            .chain(self.trait_items.iter().map(|id| id.owner_id.def_id))
-            .chain(self.impl_items.iter().map(|id| id.owner_id.def_id))
-            .chain(self.foreign_items.iter().map(|id| id.owner_id.def_id))
+            .map(|id| id.owner_id)
+            .chain(self.trait_items.iter().map(|id| id.owner_id))
+            .chain(self.impl_items.iter().map(|id| id.owner_id))
+            .chain(self.foreign_items.iter().map(|id| id.owner_id))
+    }
+
+    pub fn definitions(&self) -> impl Iterator<Item = LocalDefId> + '_ {
+        self.owners().map(|id| id.def_id)
     }
 
     pub fn par_items(&self, f: impl Fn(ItemId) + Send + Sync) {
@@ -104,7 +108,7 @@ impl<'tcx> TyCtxt<'tcx> {
         self.impl_trait_ref(def_id)
             .map(|t| t.subst_identity())
             .map(ImplSubject::Trait)
-            .unwrap_or_else(|| ImplSubject::Inherent(self.type_of(def_id)))
+            .unwrap_or_else(|| ImplSubject::Inherent(self.type_of(def_id).subst_identity()))
     }
 }
 
@@ -121,13 +125,13 @@ pub fn provide(providers: &mut Providers) {
         let node = owner.node();
         Some(Owner { node, hash_without_bodies: owner.nodes.hash_without_bodies })
     };
-    providers.local_def_id_to_hir_id = |tcx, id| {
+    providers.opt_local_def_id_to_hir_id = |tcx, id| {
         let owner = tcx.hir_crate(()).owners[id].map(|_| ());
-        match owner {
+        Some(match owner {
             MaybeOwner::Owner(_) => HirId::make_owner(id),
             MaybeOwner::Phantom => bug!("No HirId for {:?}", id),
             MaybeOwner::NonOwner(hir_id) => hir_id,
-        }
+        })
     };
     providers.hir_owner_nodes = |tcx, id| tcx.hir_crate(()).owners[id.def_id].map(|i| &i.nodes);
     providers.hir_owner_parent = |tcx, id| {
@@ -173,6 +177,7 @@ pub fn provide(providers: &mut Providers) {
         }
     };
     providers.opt_def_kind = |tcx, def_id| tcx.hir().opt_def_kind(def_id.expect_local());
+    providers.opt_rpitit_info = |_, _| None;
     providers.all_local_trait_impls = |tcx, ()| &tcx.resolutions(()).trait_impls;
     providers.expn_that_defined = |tcx, id| {
         let id = id.expect_local();
diff --git a/compiler/rustc_middle/src/infer/canonical.rs b/compiler/rustc_middle/src/infer/canonical.rs
index 43583b572..7f8fc1774 100644
--- a/compiler/rustc_middle/src/infer/canonical.rs
+++ b/compiler/rustc_middle/src/infer/canonical.rs
@@ -25,10 +25,8 @@ use crate::infer::MemberConstraint;
 use crate::mir::ConstraintCategory;
 use crate::ty::subst::GenericArg;
 use crate::ty::{self, BoundVar, List, Region, Ty, TyCtxt};
-use rustc_index::vec::IndexVec;
 use rustc_macros::HashStable;
 use smallvec::SmallVec;
-use std::iter;
 use std::ops::Index;
 
 /// A "canonicalized" type `V` is one where all free inference
@@ -44,12 +42,12 @@ pub struct Canonical<'tcx, V> {
 
 pub type CanonicalVarInfos<'tcx> = &'tcx List<CanonicalVarInfo<'tcx>>;
 
-impl<'tcx> ty::TypeFoldable<'tcx> for CanonicalVarInfos<'tcx> {
-    fn try_fold_with<F: ty::FallibleTypeFolder<'tcx>>(
+impl<'tcx> ty::TypeFoldable<TyCtxt<'tcx>> for CanonicalVarInfos<'tcx> {
+    fn try_fold_with<F: ty::FallibleTypeFolder<TyCtxt<'tcx>>>(
         self,
         folder: &mut F,
     ) -> Result<Self, F::Error> {
-        ty::util::fold_list(self, folder, |tcx, v| tcx.intern_canonical_var_infos(v))
+        ty::util::fold_list(self, folder, |tcx, v| tcx.mk_canonical_var_infos(v))
     }
 }
 
@@ -62,23 +60,23 @@ impl<'tcx> ty::TypeFoldable<'tcx> for CanonicalVarInfos<'tcx> {
 /// vectors with the original values that were replaced by canonical
 /// variables. You will need to supply it later to instantiate the
 /// canonicalized query response.
-#[derive(Clone, Debug, PartialEq, Eq, Hash, TyDecodable, TyEncodable)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, TyDecodable, TyEncodable)]
 #[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct CanonicalVarValues<'tcx> {
-    pub var_values: IndexVec<BoundVar, GenericArg<'tcx>>,
+    pub var_values: ty::SubstsRef<'tcx>,
 }
 
 impl CanonicalVarValues<'_> {
     pub fn is_identity(&self) -> bool {
-        self.var_values.iter_enumerated().all(|(bv, arg)| match arg.unpack() {
+        self.var_values.iter().enumerate().all(|(bv, arg)| match arg.unpack() {
             ty::GenericArgKind::Lifetime(r) => {
-                matches!(*r, ty::ReLateBound(ty::INNERMOST, br) if br.var == bv)
+                matches!(*r, ty::ReLateBound(ty::INNERMOST, br) if br.var.as_usize() == bv)
             }
             ty::GenericArgKind::Type(ty) => {
-                matches!(*ty.kind(), ty::Bound(ty::INNERMOST, bt) if bt.var == bv)
+                matches!(*ty.kind(), ty::Bound(ty::INNERMOST, bt) if bt.var.as_usize() == bv)
             }
             ty::GenericArgKind::Const(ct) => {
-                matches!(ct.kind(), ty::ConstKind::Bound(ty::INNERMOST, bc) if bc == bv)
+                matches!(ct.kind(), ty::ConstKind::Bound(ty::INNERMOST, bc) if bc.as_usize() == bv)
             }
         })
     }
@@ -125,6 +123,11 @@ impl<'tcx> CanonicalVarInfo<'tcx> {
         self.kind.universe()
     }
 
+    #[must_use]
+    pub fn with_updated_universe(self, ui: ty::UniverseIndex) -> CanonicalVarInfo<'tcx> {
+        CanonicalVarInfo { kind: self.kind.with_updated_universe(ui) }
+    }
+
     pub fn is_existential(&self) -> bool {
         match self.kind {
             CanonicalVarKind::Ty(_) => true,
@@ -135,6 +138,28 @@ impl<'tcx> CanonicalVarInfo<'tcx> {
             CanonicalVarKind::PlaceholderConst(_, _) => false,
         }
     }
+
+    pub fn is_region(&self) -> bool {
+        match self.kind {
+            CanonicalVarKind::Region(_) | CanonicalVarKind::PlaceholderRegion(_) => true,
+            CanonicalVarKind::Ty(_)
+            | CanonicalVarKind::PlaceholderTy(_)
+            | CanonicalVarKind::Const(_, _)
+            | CanonicalVarKind::PlaceholderConst(_, _) => false,
+        }
+    }
+
+    pub fn expect_anon_placeholder(self) -> u32 {
+        match self.kind {
+            CanonicalVarKind::Ty(_)
+            | CanonicalVarKind::Region(_)
+            | CanonicalVarKind::Const(_, _) => bug!("expected placeholder: {self:?}"),
+
+            CanonicalVarKind::PlaceholderRegion(placeholder) => placeholder.name.expect_anon(),
+            CanonicalVarKind::PlaceholderTy(placeholder) => placeholder.name.expect_anon(),
+            CanonicalVarKind::PlaceholderConst(placeholder, _) => placeholder.name.as_u32(),
+        }
+    }
 }
 
 /// Describes the "kind" of the canonical variable. This is a "kind"
@@ -179,6 +204,38 @@ impl<'tcx> CanonicalVarKind<'tcx> {
             CanonicalVarKind::PlaceholderConst(placeholder, _) => placeholder.universe,
         }
     }
+
+    /// Replaces the universe of this canonical variable with `ui`.
+    ///
+    /// In case this is a float or int variable, this causes an ICE if
+    /// the updated universe is not the root.
+    pub fn with_updated_universe(self, ui: ty::UniverseIndex) -> CanonicalVarKind<'tcx> {
+        match self {
+            CanonicalVarKind::Ty(kind) => match kind {
+                CanonicalTyVarKind::General(_) => {
+                    CanonicalVarKind::Ty(CanonicalTyVarKind::General(ui))
+                }
+                CanonicalTyVarKind::Int | CanonicalTyVarKind::Float => {
+                    assert_eq!(ui, ty::UniverseIndex::ROOT);
+                    CanonicalVarKind::Ty(kind)
+                }
+            },
+            CanonicalVarKind::PlaceholderTy(placeholder) => {
+                CanonicalVarKind::PlaceholderTy(ty::Placeholder { universe: ui, ..placeholder })
+            }
+            CanonicalVarKind::Region(_) => CanonicalVarKind::Region(ui),
+            CanonicalVarKind::PlaceholderRegion(placeholder) => {
+                CanonicalVarKind::PlaceholderRegion(ty::Placeholder { universe: ui, ..placeholder })
+            }
+            CanonicalVarKind::Const(_, ty) => CanonicalVarKind::Const(ui, ty),
+            CanonicalVarKind::PlaceholderConst(placeholder, ty) => {
+                CanonicalVarKind::PlaceholderConst(
+                    ty::Placeholder { universe: ui, ..placeholder },
+                    ty,
+                )
+            }
+        }
+    }
 }
 
 /// Rust actually has more than one category of type variables;
@@ -215,7 +272,8 @@ pub struct QueryResponse<'tcx, R> {
     pub value: R,
 }
 
-#[derive(Clone, Debug, Default, HashStable, TypeFoldable, TypeVisitable, Lift)]
+#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct QueryRegionConstraints<'tcx> {
     pub outlives: Vec<QueryOutlivesConstraint<'tcx>>,
     pub member_constraints: Vec<MemberConstraint<'tcx>>,
@@ -326,10 +384,8 @@ impl<'tcx, V> Canonical<'tcx, V> {
     }
 }
 
-pub type QueryOutlivesConstraint<'tcx> = (
-    ty::Binder<'tcx, ty::OutlivesPredicate<GenericArg<'tcx>, Region<'tcx>>>,
-    ConstraintCategory<'tcx>,
-);
+pub type QueryOutlivesConstraint<'tcx> =
+    (ty::OutlivesPredicate<GenericArg<'tcx>, Region<'tcx>>, ConstraintCategory<'tcx>);
 
 TrivialTypeTraversalAndLiftImpls! {
     for <'tcx> {
@@ -339,57 +395,57 @@ TrivialTypeTraversalAndLiftImpls! {
 }
 
 impl<'tcx> CanonicalVarValues<'tcx> {
+    // Given a list of canonical variables, construct a set of values which are
+    // the identity response.
+    pub fn make_identity(
+        tcx: TyCtxt<'tcx>,
+        infos: CanonicalVarInfos<'tcx>,
+    ) -> CanonicalVarValues<'tcx> {
+        CanonicalVarValues {
+            var_values: tcx.mk_substs_from_iter(infos.iter().enumerate().map(
+                |(i, info)| -> ty::GenericArg<'tcx> {
+                    match info.kind {
+                        CanonicalVarKind::Ty(_) | CanonicalVarKind::PlaceholderTy(_) => {
+                            tcx.mk_bound(ty::INNERMOST, ty::BoundVar::from_usize(i).into()).into()
+                        }
+                        CanonicalVarKind::Region(_) | CanonicalVarKind::PlaceholderRegion(_) => {
+                            let br = ty::BoundRegion {
+                                var: ty::BoundVar::from_usize(i),
+                                kind: ty::BrAnon(i as u32, None),
+                            };
+                            tcx.mk_re_late_bound(ty::INNERMOST, br).into()
+                        }
+                        CanonicalVarKind::Const(_, ty)
+                        | CanonicalVarKind::PlaceholderConst(_, ty) => tcx
+                            .mk_const(
+                                ty::ConstKind::Bound(ty::INNERMOST, ty::BoundVar::from_usize(i)),
+                                ty,
+                            )
+                            .into(),
+                    }
+                },
+            )),
+        }
+    }
+
     /// Creates dummy var values which should not be used in a
     /// canonical response.
     pub fn dummy() -> CanonicalVarValues<'tcx> {
-        CanonicalVarValues { var_values: Default::default() }
+        CanonicalVarValues { var_values: ty::List::empty() }
     }
 
     #[inline]
     pub fn len(&self) -> usize {
         self.var_values.len()
     }
-
-    /// Makes an identity substitution from this one: each bound var
-    /// is matched to the same bound var, preserving the original kinds.
-    /// For example, if we have:
-    /// `self.var_values == [Type(u32), Lifetime('a), Type(u64)]`
-    /// we'll return a substitution `subst` with:
-    /// `subst.var_values == [Type(^0), Lifetime(^1), Type(^2)]`.
-    pub fn make_identity(&self, tcx: TyCtxt<'tcx>) -> Self {
-        use crate::ty::subst::GenericArgKind;
-
-        CanonicalVarValues {
-            var_values: iter::zip(&self.var_values, 0..)
-                .map(|(kind, i)| match kind.unpack() {
-                    GenericArgKind::Type(..) => {
-                        tcx.mk_ty(ty::Bound(ty::INNERMOST, ty::BoundVar::from_u32(i).into())).into()
-                    }
-                    GenericArgKind::Lifetime(..) => {
-                        let br = ty::BoundRegion {
-                            var: ty::BoundVar::from_u32(i),
-                            kind: ty::BrAnon(i, None),
-                        };
-                        tcx.mk_region(ty::ReLateBound(ty::INNERMOST, br)).into()
-                    }
-                    GenericArgKind::Const(ct) => tcx
-                        .mk_const(
-                            ty::ConstKind::Bound(ty::INNERMOST, ty::BoundVar::from_u32(i)),
-                            ct.ty(),
-                        )
-                        .into(),
-                })
-                .collect(),
-        }
-    }
 }
 
 impl<'a, 'tcx> IntoIterator for &'a CanonicalVarValues<'tcx> {
     type Item = GenericArg<'tcx>;
-    type IntoIter = ::std::iter::Cloned<::std::slice::Iter<'a, GenericArg<'tcx>>>;
+    type IntoIter = ::std::iter::Copied<::std::slice::Iter<'a, GenericArg<'tcx>>>;
 
     fn into_iter(self) -> Self::IntoIter {
-        self.var_values.iter().cloned()
+        self.var_values.iter()
     }
 }
 
@@ -397,6 +453,6 @@ impl<'tcx> Index<BoundVar> for CanonicalVarValues<'tcx> {
     type Output = GenericArg<'tcx>;
 
     fn index(&self, value: BoundVar) -> &GenericArg<'tcx> {
-        &self.var_values[value]
+        &self.var_values[value.as_usize()]
     }
 }
diff --git a/compiler/rustc_middle/src/infer/mod.rs b/compiler/rustc_middle/src/infer/mod.rs
index 38868c210..2db59f37f 100644
--- a/compiler/rustc_middle/src/infer/mod.rs
+++ b/compiler/rustc_middle/src/infer/mod.rs
@@ -12,7 +12,8 @@ use rustc_span::Span;
 /// ```text
 /// R0 member of [O1..On]
 /// ```
-#[derive(Debug, Clone, HashStable, TypeFoldable, TypeVisitable, Lift)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct MemberConstraint<'tcx> {
     /// The `DefId` and substs of the opaque type causing this constraint.
     /// Used for error reporting.
diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs
index 7e4063c2f..c33b9d84e 100644
--- a/compiler/rustc_middle/src/lib.rs
+++ b/compiler/rustc_middle/src/lib.rs
@@ -34,6 +34,7 @@
 #![feature(get_mut_unchecked)]
 #![feature(if_let_guard)]
 #![feature(iter_from_generator)]
+#![feature(local_key_cell_methods)]
 #![feature(negative_impls)]
 #![feature(never_type)]
 #![feature(extern_types)]
@@ -43,21 +44,21 @@
 #![feature(min_specialization)]
 #![feature(trusted_len)]
 #![feature(type_alias_impl_trait)]
+#![feature(strict_provenance)]
 #![feature(associated_type_bounds)]
 #![feature(rustc_attrs)]
 #![feature(control_flow_enum)]
-#![feature(associated_type_defaults)]
 #![feature(trusted_step)]
 #![feature(try_blocks)]
 #![feature(try_reserve_kind)]
 #![feature(nonzero_ops)]
-#![feature(unwrap_infallible)]
 #![feature(decl_macro)]
 #![feature(drain_filter)]
 #![feature(intra_doc_pointers)]
 #![feature(yeet_expr)]
 #![feature(result_option_inspect)]
 #![feature(const_option)]
+#![feature(trait_alias)]
 #![recursion_limit = "512"]
 #![allow(rustc::potential_query_instability)]
 
@@ -72,6 +73,9 @@ extern crate tracing;
 #[macro_use]
 extern crate smallvec;
 
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
+use rustc_macros::fluent_messages;
+
 #[cfg(test)]
 mod tests;
 
@@ -104,3 +108,5 @@ pub mod util {
 
 // Allows macros to refer to this crate as `::rustc_middle`
 extern crate self as rustc_middle;
+
+fluent_messages! { "../locales/en-US.ftl" }
diff --git a/compiler/rustc_middle/src/macros.rs b/compiler/rustc_middle/src/macros.rs
index 250f3d079..a8d71ce03 100644
--- a/compiler/rustc_middle/src/macros.rs
+++ b/compiler/rustc_middle/src/macros.rs
@@ -69,8 +69,8 @@ macro_rules! CloneLiftImpls {
 macro_rules! TrivialTypeTraversalImpls {
     (for <$tcx:lifetime> { $($ty:ty,)+ }) => {
         $(
-            impl<$tcx> $crate::ty::fold::TypeFoldable<$tcx> for $ty {
-                fn try_fold_with<F: $crate::ty::fold::FallibleTypeFolder<$tcx>>(
+            impl<$tcx> $crate::ty::fold::TypeFoldable<$crate::ty::TyCtxt<$tcx>> for $ty {
+                fn try_fold_with<F: $crate::ty::fold::FallibleTypeFolder<$crate::ty::TyCtxt<$tcx>>>(
                     self,
                     _: &mut F,
                 ) -> ::std::result::Result<Self, F::Error> {
@@ -78,7 +78,7 @@ macro_rules! TrivialTypeTraversalImpls {
                 }
 
                 #[inline]
-                fn fold_with<F: $crate::ty::fold::TypeFolder<$tcx>>(
+                fn fold_with<F: $crate::ty::fold::TypeFolder<$crate::ty::TyCtxt<$tcx>>>(
                     self,
                     _: &mut F,
                 ) -> Self {
@@ -86,9 +86,9 @@ macro_rules! TrivialTypeTraversalImpls {
                 }
             }
 
-            impl<$tcx> $crate::ty::visit::TypeVisitable<$tcx> for $ty {
+            impl<$tcx> $crate::ty::visit::TypeVisitable<$crate::ty::TyCtxt<$tcx>> for $ty {
                 #[inline]
-                fn visit_with<F: $crate::ty::visit::TypeVisitor<$tcx>>(
+                fn visit_with<F: $crate::ty::visit::TypeVisitor<$crate::ty::TyCtxt<$tcx>>>(
                     &self,
                     _: &mut F)
                     -> ::std::ops::ControlFlow<F::BreakTy>
@@ -101,7 +101,7 @@ macro_rules! TrivialTypeTraversalImpls {
 
     ($($ty:ty,)+) => {
         TrivialTypeTraversalImpls! {
-            for <'tcx> {
+            for<'tcx> {
                 $($ty,)+
             }
         }
@@ -115,145 +115,3 @@ macro_rules! TrivialTypeTraversalAndLiftImpls {
         CloneLiftImpls! { $($t)* }
     }
 }
-
-#[macro_export]
-macro_rules! EnumTypeTraversalImpl {
-    (impl<$($p:tt),*> TypeFoldable<$tcx:tt> for $s:path {
-        $($variants:tt)*
-    } $(where $($wc:tt)*)*) => {
-        impl<$($p),*> $crate::ty::fold::TypeFoldable<$tcx> for $s
-            $(where $($wc)*)*
-        {
-            fn try_fold_with<V: $crate::ty::fold::FallibleTypeFolder<$tcx>>(
-                self,
-                folder: &mut V,
-            ) -> ::std::result::Result<Self, V::Error> {
-                EnumTypeTraversalImpl!(@FoldVariants(self, folder) input($($variants)*) output())
-            }
-        }
-    };
-
-    (impl<$($p:tt),*> TypeVisitable<$tcx:tt> for $s:path {
-        $($variants:tt)*
-    } $(where $($wc:tt)*)*) => {
-        impl<$($p),*> $crate::ty::visit::TypeVisitable<$tcx> for $s
-            $(where $($wc)*)*
-        {
-            fn visit_with<V: $crate::ty::visit::TypeVisitor<$tcx>>(
-                &self,
-                visitor: &mut V,
-            ) -> ::std::ops::ControlFlow<V::BreakTy> {
-                EnumTypeTraversalImpl!(@VisitVariants(self, visitor) input($($variants)*) output())
-            }
-        }
-    };
-
-    (@FoldVariants($this:expr, $folder:expr) input() output($($output:tt)*)) => {
-        Ok(match $this {
-            $($output)*
-        })
-    };
-
-    (@FoldVariants($this:expr, $folder:expr)
-     input( ($variant:path) ( $($variant_arg:ident),* ) , $($input:tt)*)
-     output( $($output:tt)*) ) => {
-        EnumTypeTraversalImpl!(
-            @FoldVariants($this, $folder)
-                input($($input)*)
-                output(
-                    $variant ( $($variant_arg),* ) => {
-                        $variant (
-                            $($crate::ty::fold::TypeFoldable::try_fold_with($variant_arg, $folder)?),*
-                        )
-                    }
-                    $($output)*
-                )
-        )
-    };
-
-    (@FoldVariants($this:expr, $folder:expr)
-     input( ($variant:path) { $($variant_arg:ident),* $(,)? } , $($input:tt)*)
-     output( $($output:tt)*) ) => {
-        EnumTypeTraversalImpl!(
-            @FoldVariants($this, $folder)
-                input($($input)*)
-                output(
-                    $variant { $($variant_arg),* } => {
-                        $variant {
-                            $($variant_arg: $crate::ty::fold::TypeFoldable::fold_with(
-                                $variant_arg, $folder
-                            )?),* }
-                    }
-                    $($output)*
-                )
-        )
-    };
-
-    (@FoldVariants($this:expr, $folder:expr)
-     input( ($variant:path), $($input:tt)*)
-     output( $($output:tt)*) ) => {
-        EnumTypeTraversalImpl!(
-            @FoldVariants($this, $folder)
-                input($($input)*)
-                output(
-                    $variant => { $variant }
-                    $($output)*
-                )
-        )
-    };
-
-    (@VisitVariants($this:expr, $visitor:expr) input() output($($output:tt)*)) => {
-        match $this {
-            $($output)*
-        }
-    };
-
-    (@VisitVariants($this:expr, $visitor:expr)
-     input( ($variant:path) ( $($variant_arg:ident),* ) , $($input:tt)*)
-     output( $($output:tt)*) ) => {
-        EnumTypeTraversalImpl!(
-            @VisitVariants($this, $visitor)
-                input($($input)*)
-                output(
-                    $variant ( $($variant_arg),* ) => {
-                        $($crate::ty::visit::TypeVisitable::visit_with(
-                            $variant_arg, $visitor
-                        )?;)*
-                        ::std::ops::ControlFlow::Continue(())
-                    }
-                    $($output)*
-                )
-        )
-    };
-
-    (@VisitVariants($this:expr, $visitor:expr)
-     input( ($variant:path) { $($variant_arg:ident),* $(,)? } , $($input:tt)*)
-     output( $($output:tt)*) ) => {
-        EnumTypeTraversalImpl!(
-            @VisitVariants($this, $visitor)
-                input($($input)*)
-                output(
-                    $variant { $($variant_arg),* } => {
-                        $($crate::ty::visit::TypeVisitable::visit_with(
-                            $variant_arg, $visitor
-                        )?;)*
-                        ::std::ops::ControlFlow::Continue(())
-                    }
-                    $($output)*
-                )
-        )
-    };
-
-    (@VisitVariants($this:expr, $visitor:expr)
-     input( ($variant:path), $($input:tt)*)
-     output( $($output:tt)*) ) => {
-        EnumTypeTraversalImpl!(
-            @VisitVariants($this, $visitor)
-                input($($input)*)
-                output(
-                    $variant => { ::std::ops::ControlFlow::Continue(()) }
-                    $($output)*
-                )
-        )
-    };
-}
diff --git a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
index bea884c85..c4601a1fb 100644
--- a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
+++ b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
@@ -91,7 +91,8 @@ bitflags! {
         /// the MIR `InstrumentCoverage` pass and not added to the coverage map
         /// during codegen.
         const NO_COVERAGE               = 1 << 15;
-        /// `#[used(linker)]`: indicates that LLVM nor the linker can eliminate this function.
+        /// `#[used(linker)]`:
+        /// indicates that neither LLVM nor the linker will eliminate this function.
         const USED_LINKER               = 1 << 16;
         /// `#[rustc_deallocator]`: a hint to LLVM that the function only deallocates memory.
         const DEALLOCATOR               = 1 << 17;
diff --git a/compiler/rustc_middle/src/middle/mod.rs b/compiler/rustc_middle/src/middle/mod.rs
index 8dc68b1f5..0b6774f1b 100644
--- a/compiler/rustc_middle/src/middle/mod.rs
+++ b/compiler/rustc_middle/src/middle/mod.rs
@@ -29,7 +29,7 @@ pub mod lib_features {
 pub mod limits;
 pub mod privacy;
 pub mod region;
-pub mod resolve_lifetime;
+pub mod resolve_bound_vars;
 pub mod stability;
 
 pub fn provide(providers: &mut crate::ty::query::Providers) {
diff --git a/compiler/rustc_middle/src/middle/privacy.rs b/compiler/rustc_middle/src/middle/privacy.rs
index 0e18ba73d..893bf54b8 100644
--- a/compiler/rustc_middle/src/middle/privacy.rs
+++ b/compiler/rustc_middle/src/middle/privacy.rs
@@ -194,11 +194,6 @@ impl EffectiveVisibilities {
     }
 }
 
-pub trait IntoDefIdTree {
-    type Tree: DefIdTree;
-    fn tree(self) -> Self::Tree;
-}
-
 impl<Id: Eq + Hash> EffectiveVisibilities<Id> {
     pub fn iter(&self) -> impl Iterator<Item = (&Id, &EffectiveVisibility)> {
         self.map.iter()
@@ -217,25 +212,21 @@ impl<Id: Eq + Hash> EffectiveVisibilities<Id> {
         self.map.entry(id).or_insert_with(|| EffectiveVisibility::from_vis(lazy_private_vis()))
     }
 
-    pub fn update<T: IntoDefIdTree>(
+    pub fn update(
         &mut self,
         id: Id,
         nominal_vis: Visibility,
-        lazy_private_vis: impl FnOnce(T) -> (Visibility, T),
+        lazy_private_vis: impl FnOnce() -> Visibility,
         inherited_effective_vis: EffectiveVisibility,
         level: Level,
-        mut into_tree: T,
+        tree: impl DefIdTree,
     ) -> bool {
         let mut changed = false;
-        let mut current_effective_vis = match self.map.get(&id).copied() {
-            Some(eff_vis) => eff_vis,
-            None => {
-                let private_vis;
-                (private_vis, into_tree) = lazy_private_vis(into_tree);
-                EffectiveVisibility::from_vis(private_vis)
-            }
-        };
-        let tree = into_tree.tree();
+        let mut current_effective_vis = self
+            .map
+            .get(&id)
+            .copied()
+            .unwrap_or_else(|| EffectiveVisibility::from_vis(lazy_private_vis()));
 
         let mut inherited_effective_vis_at_prev_level = *inherited_effective_vis.at_level(level);
         let mut calculated_effective_vis = inherited_effective_vis_at_prev_level;
diff --git a/compiler/rustc_middle/src/middle/resolve_lifetime.rs b/compiler/rustc_middle/src/middle/resolve_bound_vars.rs
index c3bf1c717..c59704fc0 100644
--- a/compiler/rustc_middle/src/middle/resolve_lifetime.rs
+++ b/compiler/rustc_middle/src/middle/resolve_bound_vars.rs
@@ -1,18 +1,20 @@
-//! Name resolution for lifetimes: type declarations.
+//! Name resolution for lifetimes and late-bound type and const variables: type declarations.
 
 use crate::ty;
 
 use rustc_data_structures::fx::FxHashMap;
+use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def_id::DefId;
 use rustc_hir::{ItemLocalId, OwnerId};
 use rustc_macros::HashStable;
 
 #[derive(Clone, Copy, PartialEq, Eq, Hash, TyEncodable, TyDecodable, Debug, HashStable)]
-pub enum Region {
-    Static,
-    EarlyBound(/* lifetime decl */ DefId),
-    LateBound(ty::DebruijnIndex, /* late-bound index */ u32, /* lifetime decl */ DefId),
+pub enum ResolvedArg {
+    StaticLifetime,
+    EarlyBound(/* decl */ DefId),
+    LateBound(ty::DebruijnIndex, /* late-bound index */ u32, /* decl */ DefId),
     Free(DefId, /* lifetime decl */ DefId),
+    Error(ErrorGuaranteed),
 }
 
 /// A set containing, at most, one known element.
@@ -46,10 +48,10 @@ pub enum ObjectLifetimeDefault {
 /// Maps the id of each lifetime reference to the lifetime decl
 /// that it corresponds to.
 #[derive(Default, HashStable, Debug)]
-pub struct ResolveLifetimes {
+pub struct ResolveBoundVars {
     /// Maps from every use of a named (not anonymous) lifetime to a
     /// `Region` describing how that region is bound
-    pub defs: FxHashMap<OwnerId, FxHashMap<ItemLocalId, Region>>,
+    pub defs: FxHashMap<OwnerId, FxHashMap<ItemLocalId, ResolvedArg>>,
 
     pub late_bound_vars: FxHashMap<OwnerId, FxHashMap<ItemLocalId, Vec<ty::BoundVariableKind>>>,
 }
diff --git a/compiler/rustc_middle/src/middle/stability.rs b/compiler/rustc_middle/src/middle/stability.rs
index 0836f236e..354c84e22 100644
--- a/compiler/rustc_middle/src/middle/stability.rs
+++ b/compiler/rustc_middle/src/middle/stability.rs
@@ -255,7 +255,7 @@ fn late_report_deprecation(
     let method_span = method_span.unwrap_or(span);
     tcx.struct_span_lint_hir(lint, hir_id, method_span, message, |diag| {
         if let hir::Node::Expr(_) = tcx.hir().get(hir_id) {
-            let kind = tcx.def_kind(def_id).descr(def_id);
+            let kind = tcx.def_descr(def_id);
             deprecation_suggestion(diag, kind, suggestion, method_span);
         }
         diag
@@ -392,7 +392,7 @@ impl<'tcx> TyCtxt<'tcx> {
                     let lint = deprecation_lint(is_in_effect);
                     if self.lint_level_at_node(lint, id).0 != Level::Allow {
                         let def_path = with_no_trimmed_paths!(self.def_path_str(def_id));
-                        let def_kind = self.def_kind(def_id).descr(def_id);
+                        let def_kind = self.def_descr(def_id);
 
                         late_report_deprecation(
                             self,
diff --git a/compiler/rustc_middle/src/mir/basic_blocks.rs b/compiler/rustc_middle/src/mir/basic_blocks.rs
index 752cbdeae..b93871769 100644
--- a/compiler/rustc_middle/src/mir/basic_blocks.rs
+++ b/compiler/rustc_middle/src/mir/basic_blocks.rs
@@ -1,41 +1,46 @@
-use crate::mir::graph_cyclic_cache::GraphIsCyclicCache;
-use crate::mir::predecessors::{PredecessorCache, Predecessors};
-use crate::mir::switch_sources::{SwitchSourceCache, SwitchSources};
-use crate::mir::traversal::PostorderCache;
-use crate::mir::{BasicBlock, BasicBlockData, Successors, START_BLOCK};
+use crate::mir::traversal::Postorder;
+use crate::mir::{BasicBlock, BasicBlockData, Successors, Terminator, TerminatorKind, START_BLOCK};
 
+use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::graph;
 use rustc_data_structures::graph::dominators::{dominators, Dominators};
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+use rustc_data_structures::sync::OnceCell;
 use rustc_index::vec::IndexVec;
+use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
+use smallvec::SmallVec;
 
 #[derive(Clone, TyEncodable, TyDecodable, Debug, HashStable, TypeFoldable, TypeVisitable)]
 pub struct BasicBlocks<'tcx> {
     basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>,
-    predecessor_cache: PredecessorCache,
-    switch_source_cache: SwitchSourceCache,
-    is_cyclic: GraphIsCyclicCache,
-    postorder_cache: PostorderCache,
+    cache: Cache,
+}
+
+// Typically 95%+ of basic blocks have 4 or fewer predecessors.
+pub type Predecessors = IndexVec<BasicBlock, SmallVec<[BasicBlock; 4]>>;
+
+pub type SwitchSources = FxHashMap<(BasicBlock, BasicBlock), SmallVec<[Option<u128>; 1]>>;
+
+#[derive(Clone, Default, Debug)]
+struct Cache {
+    predecessors: OnceCell<Predecessors>,
+    switch_sources: OnceCell<SwitchSources>,
+    is_cyclic: OnceCell<bool>,
+    postorder: OnceCell<Vec<BasicBlock>>,
 }
 
 impl<'tcx> BasicBlocks<'tcx> {
     #[inline]
     pub fn new(basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>) -> Self {
-        BasicBlocks {
-            basic_blocks,
-            predecessor_cache: PredecessorCache::new(),
-            switch_source_cache: SwitchSourceCache::new(),
-            is_cyclic: GraphIsCyclicCache::new(),
-            postorder_cache: PostorderCache::new(),
-        }
+        BasicBlocks { basic_blocks, cache: Cache::default() }
     }
 
     /// Returns true if control-flow graph contains a cycle reachable from the `START_BLOCK`.
     #[inline]
     pub fn is_cfg_cyclic(&self) -> bool {
-        self.is_cyclic.is_cyclic(self)
+        *self.cache.is_cyclic.get_or_init(|| graph::is_cyclic(self))
     }
 
-    #[inline]
     pub fn dominators(&self) -> Dominators<BasicBlock> {
         dominators(&self)
     }
@@ -43,20 +48,46 @@ impl<'tcx> BasicBlocks<'tcx> {
     /// Returns predecessors for each basic block.
     #[inline]
     pub fn predecessors(&self) -> &Predecessors {
-        self.predecessor_cache.compute(&self.basic_blocks)
+        self.cache.predecessors.get_or_init(|| {
+            let mut preds = IndexVec::from_elem(SmallVec::new(), &self.basic_blocks);
+            for (bb, data) in self.basic_blocks.iter_enumerated() {
+                if let Some(term) = &data.terminator {
+                    for succ in term.successors() {
+                        preds[succ].push(bb);
+                    }
+                }
+            }
+            preds
+        })
     }
 
     /// Returns basic blocks in a postorder.
     #[inline]
     pub fn postorder(&self) -> &[BasicBlock] {
-        self.postorder_cache.compute(&self.basic_blocks)
+        self.cache.postorder.get_or_init(|| {
+            Postorder::new(&self.basic_blocks, START_BLOCK).map(|(bb, _)| bb).collect()
+        })
     }
 
     /// `switch_sources()[&(target, switch)]` returns a list of switch
     /// values that lead to a `target` block from a `switch` block.
     #[inline]
     pub fn switch_sources(&self) -> &SwitchSources {
-        self.switch_source_cache.compute(&self.basic_blocks)
+        self.cache.switch_sources.get_or_init(|| {
+            let mut switch_sources: SwitchSources = FxHashMap::default();
+            for (bb, data) in self.basic_blocks.iter_enumerated() {
+                if let Some(Terminator {
+                    kind: TerminatorKind::SwitchInt { targets, .. }, ..
+                }) = &data.terminator
+                {
+                    for (value, target) in targets.iter() {
+                        switch_sources.entry((target, bb)).or_default().push(Some(value));
+                    }
+                    switch_sources.entry((targets.otherwise(), bb)).or_default().push(None);
+                }
+            }
+            switch_sources
+        })
     }
 
     /// Returns mutable reference to basic blocks. Invalidates CFG cache.
@@ -88,10 +119,7 @@ impl<'tcx> BasicBlocks<'tcx> {
     /// All other methods that allow you to mutate the basic blocks also call this method
     /// themselves, thereby avoiding any risk of accidentally cache invalidation.
     pub fn invalidate_cfg_cache(&mut self) {
-        self.predecessor_cache.invalidate();
-        self.switch_source_cache.invalidate();
-        self.is_cyclic.invalidate();
-        self.postorder_cache.invalidate();
+        self.cache = Cache::default();
     }
 }
 
@@ -145,3 +173,24 @@ impl<'tcx> graph::WithPredecessors for BasicBlocks<'tcx> {
         self.predecessors()[node].iter().copied()
     }
 }
+
+TrivialTypeTraversalAndLiftImpls! {
+    Cache,
+}
+
+impl<S: Encoder> Encodable<S> for Cache {
+    #[inline]
+    fn encode(&self, _s: &mut S) {}
+}
+
+impl<D: Decoder> Decodable<D> for Cache {
+    #[inline]
+    fn decode(_: &mut D) -> Self {
+        Default::default()
+    }
+}
+
+impl<CTX> HashStable<CTX> for Cache {
+    #[inline]
+    fn hash_stable(&self, _: &mut CTX, _: &mut StableHasher) {}
+}
diff --git a/compiler/rustc_middle/src/mir/coverage.rs b/compiler/rustc_middle/src/mir/coverage.rs
index e7bb3ab0b..db24dae11 100644
--- a/compiler/rustc_middle/src/mir/coverage.rs
+++ b/compiler/rustc_middle/src/mir/coverage.rs
@@ -135,7 +135,10 @@ impl Debug for CoverageKind {
                 "Expression({:?}) = {} {} {}",
                 id.index(),
                 lhs.index(),
-                if *op == Op::Add { "+" } else { "-" },
+                match op {
+                    Op::Add => "+",
+                    Op::Subtract => "-",
+                },
                 rhs.index(),
             ),
             Unreachable => write!(fmt, "Unreachable"),
diff --git a/compiler/rustc_middle/src/mir/graph_cyclic_cache.rs b/compiler/rustc_middle/src/mir/graph_cyclic_cache.rs
deleted file mode 100644
index f97bf2883..000000000
--- a/compiler/rustc_middle/src/mir/graph_cyclic_cache.rs
+++ /dev/null
@@ -1,63 +0,0 @@
-use rustc_data_structures::graph::{
-    self, DirectedGraph, WithNumNodes, WithStartNode, WithSuccessors,
-};
-use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_data_structures::sync::OnceCell;
-use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
-
-/// Helper type to cache the result of `graph::is_cyclic`.
-#[derive(Clone, Debug)]
-pub(super) struct GraphIsCyclicCache {
-    cache: OnceCell<bool>,
-}
-
-impl GraphIsCyclicCache {
-    #[inline]
-    pub(super) fn new() -> Self {
-        GraphIsCyclicCache { cache: OnceCell::new() }
-    }
-
-    pub(super) fn is_cyclic<G>(&self, graph: &G) -> bool
-    where
-        G: ?Sized + DirectedGraph + WithStartNode + WithSuccessors + WithNumNodes,
-    {
-        *self.cache.get_or_init(|| graph::is_cyclic(graph))
-    }
-
-    /// Invalidates the cache.
-    #[inline]
-    pub(super) fn invalidate(&mut self) {
-        // Invalidating the cache requires mutating the MIR, which in turn requires a unique
-        // reference (`&mut`) to the `mir::Body`. Because of this, we can assume that all
-        // callers of `invalidate` have a unique reference to the MIR and thus to the
-        // cache. This means we never need to do synchronization when `invalidate` is called,
-        // we can simply reinitialize the `OnceCell`.
-        self.cache = OnceCell::new();
-    }
-}
-
-impl<S: Encoder> Encodable<S> for GraphIsCyclicCache {
-    #[inline]
-    fn encode(&self, s: &mut S) {
-        Encodable::encode(&(), s);
-    }
-}
-
-impl<D: Decoder> Decodable<D> for GraphIsCyclicCache {
-    #[inline]
-    fn decode(d: &mut D) -> Self {
-        let () = Decodable::decode(d);
-        Self::new()
-    }
-}
-
-impl<CTX> HashStable<CTX> for GraphIsCyclicCache {
-    #[inline]
-    fn hash_stable(&self, _: &mut CTX, _: &mut StableHasher) {
-        // do nothing
-    }
-}
-
-TrivialTypeTraversalAndLiftImpls! {
-    GraphIsCyclicCache,
-}
diff --git a/compiler/rustc_middle/src/mir/graphviz.rs b/compiler/rustc_middle/src/mir/graphviz.rs
index 5de56dad0..cf6d46e1e 100644
--- a/compiler/rustc_middle/src/mir/graphviz.rs
+++ b/compiler/rustc_middle/src/mir/graphviz.rs
@@ -110,7 +110,7 @@ fn write_graph_label<'tcx, W: std::fmt::Write>(
         let decl = &body.local_decls[local];
 
         write!(w, "let ")?;
-        if decl.mutability == Mutability::Mut {
+        if decl.mutability.is_mut() {
             write!(w, "mut ")?;
         }
 
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation.rs b/compiler/rustc_middle/src/mir/interpret/allocation.rs
index 221105ac4..48375ed30 100644
--- a/compiler/rustc_middle/src/mir/interpret/allocation.rs
+++ b/compiler/rustc_middle/src/mir/interpret/allocation.rs
@@ -8,7 +8,8 @@ mod tests;
 use std::borrow::Cow;
 use std::fmt;
 use std::hash;
-use std::ops::Range;
+use std::hash::Hash;
+use std::ops::{Deref, DerefMut, Range};
 use std::ptr;
 
 use either::{Left, Right};
@@ -29,6 +30,39 @@ use provenance_map::*;
 
 pub use init_mask::{InitChunk, InitChunkIter};
 
+/// Functionality required for the bytes of an `Allocation`.
+pub trait AllocBytes:
+    Clone + fmt::Debug + Eq + PartialEq + Hash + Deref<Target = [u8]> + DerefMut<Target = [u8]>
+{
+    /// Adjust the bytes to the specified alignment -- by default, this is a no-op.
+    fn adjust_to_align(self, _align: Align) -> Self;
+
+    /// Create an `AllocBytes` from a slice of `u8`.
+    fn from_bytes<'a>(slice: impl Into<Cow<'a, [u8]>>, _align: Align) -> Self;
+
+    /// Create a zeroed `AllocBytes` of the specified size and alignment;
+    /// call the callback error handler if there is an error in allocating the memory.
+    fn zeroed(size: Size, _align: Align) -> Option<Self>;
+}
+
+// Default `bytes` for `Allocation` is a `Box<[u8]>`.
+impl AllocBytes for Box<[u8]> {
+    fn adjust_to_align(self, _align: Align) -> Self {
+        self
+    }
+
+    fn from_bytes<'a>(slice: impl Into<Cow<'a, [u8]>>, _align: Align) -> Self {
+        Box::<[u8]>::from(slice.into())
+    }
+
+    fn zeroed(size: Size, _align: Align) -> Option<Self> {
+        let bytes = Box::<[u8]>::try_new_zeroed_slice(size.bytes_usize()).ok()?;
+        // SAFETY: the box was zero-allocated, which is a valid initial value for Box<[u8]>
+        let bytes = unsafe { bytes.assume_init() };
+        Some(bytes)
+    }
+}
+
 /// This type represents an Allocation in the Miri/CTFE core engine.
 ///
 /// Its public API is rather low-level, working directly with allocation offsets and a custom error
@@ -38,10 +72,10 @@ pub use init_mask::{InitChunk, InitChunkIter};
 // hashed. (see the `Hash` impl below for more details), so the impl is not derived.
 #[derive(Clone, Eq, PartialEq, TyEncodable, TyDecodable)]
 #[derive(HashStable)]
-pub struct Allocation<Prov: Provenance = AllocId, Extra = ()> {
+pub struct Allocation<Prov: Provenance = AllocId, Extra = (), Bytes = Box<[u8]>> {
     /// The actual bytes of the allocation.
     /// Note that the bytes of a pointer represent the offset of the pointer.
-    bytes: Box<[u8]>,
+    bytes: Bytes,
     /// Maps from byte addresses to extra provenance data for each pointer.
     /// Only the first byte of a pointer is inserted into the map; i.e.,
     /// every entry in this map applies to `pointer_size` consecutive bytes starting
@@ -220,14 +254,27 @@ impl AllocRange {
 }
 
 // The constructors are all without extra; the extra gets added by a machine hook later.
-impl<Prov: Provenance> Allocation<Prov> {
+impl<Prov: Provenance, Bytes: AllocBytes> Allocation<Prov, (), Bytes> {
+    /// Creates an allocation from an existing `Bytes` value - this is needed for miri FFI support
+    pub fn from_raw_bytes(bytes: Bytes, align: Align, mutability: Mutability) -> Self {
+        let size = Size::from_bytes(bytes.len());
+        Self {
+            bytes,
+            provenance: ProvenanceMap::new(),
+            init_mask: InitMask::new(size, true),
+            align,
+            mutability,
+            extra: (),
+        }
+    }
+
     /// Creates an allocation initialized by the given bytes
     pub fn from_bytes<'a>(
         slice: impl Into<Cow<'a, [u8]>>,
         align: Align,
         mutability: Mutability,
     ) -> Self {
-        let bytes = Box::<[u8]>::from(slice.into());
+        let bytes = Bytes::from_bytes(slice, align);
         let size = Size::from_bytes(bytes.len());
         Self {
             bytes,
@@ -248,7 +295,7 @@ impl<Prov: Provenance> Allocation<Prov> {
     ///
     /// If `panic_on_fail` is true, this will never return `Err`.
     pub fn uninit<'tcx>(size: Size, align: Align, panic_on_fail: bool) -> InterpResult<'tcx, Self> {
-        let bytes = Box::<[u8]>::try_new_zeroed_slice(size.bytes_usize()).map_err(|_| {
+        let bytes = Bytes::zeroed(size, align).ok_or_else(|| {
             // This results in an error that can happen non-deterministically, since the memory
             // available to the compiler can change between runs. Normally queries are always
             // deterministic. However, we can be non-deterministic here because all uses of const
@@ -262,8 +309,7 @@ impl<Prov: Provenance> Allocation<Prov> {
             });
             InterpError::ResourceExhaustion(ResourceExhaustionInfo::MemoryExhausted)
         })?;
-        // SAFETY: the box was zero-allocated, which is a valid initial value for Box<[u8]>
-        let bytes = unsafe { bytes.assume_init() };
+
         Ok(Allocation {
             bytes,
             provenance: ProvenanceMap::new(),
@@ -275,7 +321,7 @@ impl<Prov: Provenance> Allocation<Prov> {
     }
 }
 
-impl Allocation {
+impl<Bytes: AllocBytes> Allocation<AllocId, (), Bytes> {
     /// Adjust allocation from the ones in tcx to a custom Machine instance
     /// with a different Provenance and Extra type.
     pub fn adjust_from_tcx<Prov: Provenance, Extra, Err>(
@@ -283,9 +329,11 @@ impl Allocation {
         cx: &impl HasDataLayout,
         extra: Extra,
         mut adjust_ptr: impl FnMut(Pointer<AllocId>) -> Result<Pointer<Prov>, Err>,
-    ) -> Result<Allocation<Prov, Extra>, Err> {
-        // Compute new pointer provenance, which also adjusts the bytes.
-        let mut bytes = self.bytes;
+    ) -> Result<Allocation<Prov, Extra, Bytes>, Err> {
+        // Compute new pointer provenance, which also adjusts the bytes, and realign the pointer if
+        // necessary.
+        let mut bytes = self.bytes.adjust_to_align(self.align);
+
         let mut new_provenance = Vec::with_capacity(self.provenance.ptrs().len());
         let ptr_size = cx.data_layout().pointer_size.bytes_usize();
         let endian = cx.data_layout().endian;
@@ -311,7 +359,7 @@ impl Allocation {
 }
 
 /// Raw accessors. Provide access to otherwise private bytes.
-impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
+impl<Prov: Provenance, Extra, Bytes: AllocBytes> Allocation<Prov, Extra, Bytes> {
     pub fn len(&self) -> usize {
         self.bytes.len()
     }
@@ -340,7 +388,11 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
 }
 
 /// Byte accessors.
-impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
+impl<Prov: Provenance, Extra, Bytes: AllocBytes> Allocation<Prov, Extra, Bytes> {
+    pub fn base_addr(&self) -> *const u8 {
+        self.bytes.as_ptr()
+    }
+
     /// This is the entirely abstraction-violating way to just grab the raw bytes without
     /// caring about provenance or initialization.
     ///
@@ -412,7 +464,7 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
 }
 
 /// Reading and writing.
-impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
+impl<Prov: Provenance, Extra, Bytes: AllocBytes> Allocation<Prov, Extra, Bytes> {
     /// Sets the init bit for the given range.
     fn mark_init(&mut self, range: AllocRange, is_init: bool) {
         if range.size.bytes() == 0 {
diff --git a/compiler/rustc_middle/src/mir/interpret/error.rs b/compiler/rustc_middle/src/mir/interpret/error.rs
index bd9cd53e1..c5137cf06 100644
--- a/compiler/rustc_middle/src/mir/interpret/error.rs
+++ b/compiler/rustc_middle/src/mir/interpret/error.rs
@@ -323,7 +323,7 @@ impl fmt::Display for UndefinedBehaviorInfo {
                 write!(
                     f,
                     "{msg}{pointer} is a dangling pointer (it has no provenance)",
-                    pointer = Pointer::<Option<AllocId>>::from_addr(*i),
+                    pointer = Pointer::<Option<AllocId>>::from_addr_invalid(*i),
                 )
             }
             AlignmentCheckFailed { required, has } => write!(
@@ -430,8 +430,10 @@ pub enum ResourceExhaustionInfo {
     ///
     /// The exact limit is set by the `const_eval_limit` attribute.
     StepLimitReached,
-    /// There is not enough memory to perform an allocation.
+    /// There is not enough memory (on the host) to perform an allocation.
     MemoryExhausted,
+    /// The address space (of the target) is full.
+    AddressSpaceFull,
 }
 
 impl fmt::Display for ResourceExhaustionInfo {
@@ -447,6 +449,9 @@ impl fmt::Display for ResourceExhaustionInfo {
             MemoryExhausted => {
                 write!(f, "tried to allocate more memory than available to compiler")
             }
+            AddressSpaceFull => {
+                write!(f, "there are no more free addresses in the address space")
+            }
         }
     }
 }
diff --git a/compiler/rustc_middle/src/mir/interpret/mod.rs b/compiler/rustc_middle/src/mir/interpret/mod.rs
index 5f425a287..1766d7a66 100644
--- a/compiler/rustc_middle/src/mir/interpret/mod.rs
+++ b/compiler/rustc_middle/src/mir/interpret/mod.rs
@@ -110,7 +110,7 @@ use rustc_hir::def_id::DefId;
 use rustc_macros::HashStable;
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_serialize::{Decodable, Encodable};
-use rustc_target::abi::Endian;
+use rustc_target::abi::{AddressSpace, Endian, HasDataLayout};
 
 use crate::mir;
 use crate::ty::codec::{TyDecoder, TyEncoder};
@@ -127,8 +127,8 @@ pub use self::error::{
 pub use self::value::{get_slice_bytes, ConstAlloc, ConstValue, Scalar};
 
 pub use self::allocation::{
-    alloc_range, AllocError, AllocRange, AllocResult, Allocation, ConstAllocation, InitChunk,
-    InitChunkIter,
+    alloc_range, AllocBytes, AllocError, AllocRange, AllocResult, Allocation, ConstAllocation,
+    InitChunk, InitChunkIter,
 };
 
 pub use self::pointer::{Pointer, PointerArithmetic, Provenance};
@@ -438,6 +438,17 @@ impl<'tcx> GlobalAlloc<'tcx> {
             _ => bug!("expected vtable, got {:?}", self),
         }
     }
+
+    /// The address space that this `GlobalAlloc` should be placed in.
+    #[inline]
+    pub fn address_space(&self, cx: &impl HasDataLayout) -> AddressSpace {
+        match self {
+            GlobalAlloc::Function(..) => cx.data_layout().instruction_address_space,
+            GlobalAlloc::Static(..) | GlobalAlloc::Memory(..) | GlobalAlloc::VTable(..) => {
+                AddressSpace::DATA
+            }
+        }
+    }
 }
 
 pub(crate) struct AllocMap<'tcx> {
diff --git a/compiler/rustc_middle/src/mir/interpret/pointer.rs b/compiler/rustc_middle/src/mir/interpret/pointer.rs
index b08309910..60927eed8 100644
--- a/compiler/rustc_middle/src/mir/interpret/pointer.rs
+++ b/compiler/rustc_middle/src/mir/interpret/pointer.rs
@@ -19,29 +19,29 @@ pub trait PointerArithmetic: HasDataLayout {
 
     #[inline(always)]
     fn max_size_of_val(&self) -> Size {
-        Size::from_bytes(self.machine_isize_max())
+        Size::from_bytes(self.target_isize_max())
     }
 
     #[inline]
-    fn machine_usize_max(&self) -> u64 {
+    fn target_usize_max(&self) -> u64 {
         self.pointer_size().unsigned_int_max().try_into().unwrap()
     }
 
     #[inline]
-    fn machine_isize_min(&self) -> i64 {
+    fn target_isize_min(&self) -> i64 {
         self.pointer_size().signed_int_min().try_into().unwrap()
     }
 
     #[inline]
-    fn machine_isize_max(&self) -> i64 {
+    fn target_isize_max(&self) -> i64 {
         self.pointer_size().signed_int_max().try_into().unwrap()
     }
 
     #[inline]
-    fn machine_usize_to_isize(&self, val: u64) -> i64 {
+    fn target_usize_to_isize(&self, val: u64) -> i64 {
         let val = val as i64;
         // Now wrap-around into the machine_isize range.
-        if val > self.machine_isize_max() {
+        if val > self.target_isize_max() {
             // This can only happen if the ptr size is < 64, so we know max_usize_plus_1 fits into
             // i64.
             debug_assert!(self.pointer_size().bits() < 64);
@@ -76,11 +76,11 @@ pub trait PointerArithmetic: HasDataLayout {
         let n = i.unsigned_abs();
         if i >= 0 {
             let (val, over) = self.overflowing_offset(val, n);
-            (val, over || i > self.machine_isize_max())
+            (val, over || i > self.target_isize_max())
         } else {
             let res = val.overflowing_sub(n);
             let (val, over) = self.truncate_to_ptr(res);
-            (val, over || i < self.machine_isize_min())
+            (val, over || i < self.target_isize_min())
         }
     }
 
@@ -251,14 +251,16 @@ impl<Prov> Pointer<Option<Prov>> {
 }
 
 impl<Prov> Pointer<Option<Prov>> {
+    /// Creates a pointer to the given address, with invalid provenance (i.e., cannot be used for
+    /// any memory access).
     #[inline(always)]
-    pub fn from_addr(addr: u64) -> Self {
+    pub fn from_addr_invalid(addr: u64) -> Self {
         Pointer { provenance: None, offset: Size::from_bytes(addr) }
     }
 
     #[inline(always)]
     pub fn null() -> Self {
-        Pointer::from_addr(0)
+        Pointer::from_addr_invalid(0)
     }
 }
 
diff --git a/compiler/rustc_middle/src/mir/interpret/queries.rs b/compiler/rustc_middle/src/mir/interpret/queries.rs
index b6c6e9d55..856d821a5 100644
--- a/compiler/rustc_middle/src/mir/interpret/queries.rs
+++ b/compiler/rustc_middle/src/mir/interpret/queries.rs
@@ -2,7 +2,7 @@ use super::{ErrorHandled, EvalToConstValueResult, EvalToValTreeResult, GlobalId}
 
 use crate::mir;
 use crate::ty::subst::InternalSubsts;
-use crate::ty::visit::TypeVisitable;
+use crate::ty::visit::TypeVisitableExt;
 use crate::ty::{self, query::TyCtxtAt, query::TyCtxtEnsure, TyCtxt};
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
diff --git a/compiler/rustc_middle/src/mir/interpret/value.rs b/compiler/rustc_middle/src/mir/interpret/value.rs
index 88fb14eb3..36dbbe4bf 100644
--- a/compiler/rustc_middle/src/mir/interpret/value.rs
+++ b/compiler/rustc_middle/src/mir/interpret/value.rs
@@ -75,8 +75,8 @@ impl<'tcx> ConstValue<'tcx> {
         self.try_to_scalar_int()?.try_into().ok()
     }
 
-    pub fn try_to_machine_usize(&self, tcx: TyCtxt<'tcx>) -> Option<u64> {
-        self.try_to_scalar_int()?.try_to_machine_usize(tcx).ok()
+    pub fn try_to_target_usize(&self, tcx: TyCtxt<'tcx>) -> Option<u64> {
+        self.try_to_scalar_int()?.try_to_target_usize(tcx).ok()
     }
 
     pub fn try_to_bits_for_ty(
@@ -97,8 +97,8 @@ impl<'tcx> ConstValue<'tcx> {
         ConstValue::Scalar(Scalar::from_u64(i))
     }
 
-    pub fn from_machine_usize(i: u64, cx: &impl HasDataLayout) -> Self {
-        ConstValue::Scalar(Scalar::from_machine_usize(i, cx))
+    pub fn from_target_usize(i: u64, cx: &impl HasDataLayout) -> Self {
+        ConstValue::Scalar(Scalar::from_target_usize(i, cx))
     }
 }
 
@@ -241,7 +241,7 @@ impl<Prov> Scalar<Prov> {
     }
 
     #[inline]
-    pub fn from_machine_usize(i: u64, cx: &impl HasDataLayout) -> Self {
+    pub fn from_target_usize(i: u64, cx: &impl HasDataLayout) -> Self {
         Self::from_uint(i, cx.data_layout().pointer_size)
     }
 
@@ -268,7 +268,7 @@ impl<Prov> Scalar<Prov> {
     }
 
     #[inline]
-    pub fn from_machine_isize(i: i64, cx: &impl HasDataLayout) -> Self {
+    pub fn from_target_isize(i: i64, cx: &impl HasDataLayout) -> Self {
         Self::from_int(i, cx.data_layout().pointer_size)
     }
 
@@ -322,7 +322,7 @@ impl<'tcx, Prov: Provenance> Scalar<Prov> {
             Right(ptr) => Ok(ptr.into()),
             Left(bits) => {
                 let addr = u64::try_from(bits).unwrap();
-                Ok(Pointer::from_addr(addr))
+                Ok(Pointer::from_addr_invalid(addr))
             }
         }
     }
@@ -429,7 +429,7 @@ impl<'tcx, Prov: Provenance> Scalar<Prov> {
 
     /// Converts the scalar to produce a machine-pointer-sized unsigned integer.
     /// Fails if the scalar is a pointer.
-    pub fn to_machine_usize(self, cx: &impl HasDataLayout) -> InterpResult<'tcx, u64> {
+    pub fn to_target_usize(self, cx: &impl HasDataLayout) -> InterpResult<'tcx, u64> {
         let b = self.to_uint(cx.data_layout().pointer_size)?;
         Ok(u64::try_from(b).unwrap())
     }
@@ -469,7 +469,7 @@ impl<'tcx, Prov: Provenance> Scalar<Prov> {
 
     /// Converts the scalar to produce a machine-pointer-sized signed integer.
     /// Fails if the scalar is a pointer.
-    pub fn to_machine_isize(self, cx: &impl HasDataLayout) -> InterpResult<'tcx, i64> {
+    pub fn to_target_isize(self, cx: &impl HasDataLayout) -> InterpResult<'tcx, i64> {
         let b = self.to_int(cx.data_layout().pointer_size)?;
         Ok(i64::try_from(b).unwrap())
     }
diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs
index e52b243ec..99cdb769d 100644
--- a/compiler/rustc_middle/src/mir/mod.rs
+++ b/compiler/rustc_middle/src/mir/mod.rs
@@ -9,7 +9,7 @@ use crate::mir::visit::MirVisitable;
 use crate::ty::codec::{TyDecoder, TyEncoder};
 use crate::ty::fold::{FallibleTypeFolder, TypeFoldable};
 use crate::ty::print::{FmtPrinter, Printer};
-use crate::ty::visit::{TypeVisitable, TypeVisitor};
+use crate::ty::visit::{TypeVisitable, TypeVisitableExt, TypeVisitor};
 use crate::ty::{self, DefIdTree, List, Ty, TyCtxt};
 use crate::ty::{AdtDef, InstanceDef, ScalarInt, UserTypeAnnotationIndex};
 use crate::ty::{GenericArg, InternalSubsts, SubstsRef};
@@ -27,7 +27,6 @@ use polonius_engine::Atom;
 pub use rustc_ast::Mutability;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::graph::dominators::Dominators;
-use rustc_index::bit_set::BitMatrix;
 use rustc_index::vec::{Idx, IndexVec};
 use rustc_serialize::{Decodable, Encodable};
 use rustc_span::symbol::Symbol;
@@ -47,25 +46,21 @@ mod basic_blocks;
 pub mod coverage;
 mod generic_graph;
 pub mod generic_graphviz;
-mod graph_cyclic_cache;
 pub mod graphviz;
 pub mod interpret;
 pub mod mono;
 pub mod patch;
-mod predecessors;
 pub mod pretty;
 mod query;
 pub mod spanview;
 mod syntax;
 pub use syntax::*;
-mod switch_sources;
 pub mod tcx;
 pub mod terminator;
 pub use terminator::*;
 
 pub mod traversal;
 mod type_foldable;
-mod type_visitable;
 pub mod visit;
 
 pub use self::generic_graph::graphviz_safe_def_name;
@@ -419,11 +414,7 @@ impl<'tcx> Body<'tcx> {
         (self.arg_count + 1..self.local_decls.len()).filter_map(move |index| {
             let local = Local::new(index);
             let decl = &self.local_decls[local];
-            if decl.is_user_variable() && decl.mutability == Mutability::Mut {
-                Some(local)
-            } else {
-                None
-            }
+            (decl.is_user_variable() && decl.mutability.is_mut()).then_some(local)
         })
     }
 
@@ -712,7 +703,11 @@ pub enum BindingForm<'tcx> {
     RefForGuard,
 }
 
-TrivialTypeTraversalAndLiftImpls! { BindingForm<'tcx>, }
+TrivialTypeTraversalAndLiftImpls! {
+    for<'tcx> {
+        BindingForm<'tcx>,
+    }
+}
 
 mod binding_form_impl {
     use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -905,6 +900,8 @@ pub enum LocalInfo<'tcx> {
     AggregateTemp,
     /// A temporary created during the pass `Derefer` to avoid it's retagging
     DerefTemp,
+    /// A temporary created for borrow checking.
+    FakeBorrow,
 }
 
 impl<'tcx> LocalDecl<'tcx> {
@@ -1464,6 +1461,7 @@ impl Debug for Statement<'_> {
             }
             Coverage(box ref coverage) => write!(fmt, "Coverage::{:?}", coverage.kind),
             Intrinsic(box ref intrinsic) => write!(fmt, "{intrinsic}"),
+            ConstEvalCounter => write!(fmt, "ConstEvalCounter"),
             Nop => write!(fmt, "nop"),
         }
     }
@@ -1622,7 +1620,7 @@ impl<'tcx> Place<'tcx> {
             &v
         };
 
-        Place { local: self.local, projection: tcx.intern_place_elems(new_projections) }
+        Place { local: self.local, projection: tcx.mk_place_elems(new_projections) }
     }
 }
 
@@ -1644,6 +1642,14 @@ impl<'tcx> PlaceRef<'tcx> {
         }
     }
 
+    /// Returns `true` if this `Place` contains a `Deref` projection.
+    ///
+    /// If `Place::is_indirect` returns false, the caller knows that the `Place` refers to the
+    /// same region of memory as its base.
+    pub fn is_indirect(&self) -> bool {
+        self.projection.iter().any(|elem| elem.is_indirect())
+    }
+
     /// If MirPhase >= Derefered and if projection contains Deref,
     /// It's guaranteed to be in the first place
     pub fn has_deref(&self) -> bool {
@@ -2102,10 +2108,7 @@ impl<'tcx> Debug for Rvalue<'tcx> {
                     AggregateKind::Closure(def_id, substs) => ty::tls::with(|tcx| {
                         let name = if tcx.sess.opts.unstable_opts.span_free_formats {
                             let substs = tcx.lift(substs).unwrap();
-                            format!(
-                                "[closure@{}]",
-                                tcx.def_path_str_with_substs(def_id.to_def_id(), substs),
-                            )
+                            format!("[closure@{}]", tcx.def_path_str_with_substs(def_id, substs),)
                         } else {
                             let span = tcx.def_span(def_id);
                             format!(
@@ -2116,11 +2119,17 @@ impl<'tcx> Debug for Rvalue<'tcx> {
                         let mut struct_fmt = fmt.debug_struct(&name);
 
                         // FIXME(project-rfc-2229#48): This should be a list of capture names/places
-                        if let Some(upvars) = tcx.upvars_mentioned(def_id) {
+                        if let Some(def_id) = def_id.as_local()
+                            && let Some(upvars) = tcx.upvars_mentioned(def_id)
+                        {
                             for (&var_id, place) in iter::zip(upvars.keys(), places) {
                                 let var_name = tcx.hir().name(var_id);
                                 struct_fmt.field(var_name.as_str(), place);
                             }
+                        } else {
+                            for (index, place) in places.iter().enumerate() {
+                                struct_fmt.field(&format!("{index}"), place);
+                            }
                         }
 
                         struct_fmt.finish()
@@ -2131,11 +2140,17 @@ impl<'tcx> Debug for Rvalue<'tcx> {
                         let mut struct_fmt = fmt.debug_struct(&name);
 
                         // FIXME(project-rfc-2229#48): This should be a list of capture names/places
-                        if let Some(upvars) = tcx.upvars_mentioned(def_id) {
+                        if let Some(def_id) = def_id.as_local()
+                            && let Some(upvars) = tcx.upvars_mentioned(def_id)
+                        {
                             for (&var_id, place) in iter::zip(upvars.keys(), places) {
                                 let var_name = tcx.hir().name(var_id);
                                 struct_fmt.field(var_name.as_str(), place);
                             }
+                        } else {
+                            for (index, place) in places.iter().enumerate() {
+                                struct_fmt.field(&format!("{index}"), place);
+                            }
                         }
 
                         struct_fmt.finish()
@@ -2335,13 +2350,17 @@ impl<'tcx> ConstantKind<'tcx> {
     }
 
     #[inline]
-    pub fn try_eval_usize(&self, tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>) -> Option<u64> {
+    pub fn try_eval_target_usize(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        param_env: ty::ParamEnv<'tcx>,
+    ) -> Option<u64> {
         match self {
-            Self::Ty(ct) => ct.try_eval_usize(tcx, param_env),
-            Self::Val(val, _) => val.try_to_machine_usize(tcx),
+            Self::Ty(ct) => ct.try_eval_target_usize(tcx, param_env),
+            Self::Val(val, _) => val.try_to_target_usize(tcx),
             Self::Unevaluated(uneval, _) => {
                 match tcx.const_eval_resolve(param_env, *uneval, None) {
-                    Ok(val) => val.try_to_machine_usize(tcx),
+                    Ok(val) => val.try_to_target_usize(tcx),
                     Err(_) => None,
                 }
             }
@@ -2478,7 +2497,7 @@ impl<'tcx> ConstantKind<'tcx> {
         };
         debug!("expr.kind: {:?}", expr.kind);
 
-        let ty = tcx.type_of(def.def_id_for_type_of());
+        let ty = tcx.type_of(def.def_id_for_type_of()).subst_identity();
         debug!(?ty);
 
         // FIXME(const_generics): We currently have to special case parameters because `min_const_generics`
@@ -2506,20 +2525,19 @@ impl<'tcx> ConstantKind<'tcx> {
         }
 
         let hir_id = tcx.hir().local_def_id_to_hir_id(def.did);
-        let parent_substs = if let Some(parent_hir_id) = tcx.hir().opt_parent_id(hir_id) {
-            if let Some(parent_did) = tcx.hir().opt_local_def_id(parent_hir_id) {
-                InternalSubsts::identity_for_item(tcx, parent_did.to_def_id())
-            } else {
-                tcx.mk_substs(Vec::<GenericArg<'tcx>>::new().into_iter())
-            }
+        let parent_substs = if let Some(parent_hir_id) = tcx.hir().opt_parent_id(hir_id)
+            && let Some(parent_did) = parent_hir_id.as_owner()
+        {
+            InternalSubsts::identity_for_item(tcx, parent_did.to_def_id())
         } else {
-            tcx.mk_substs(Vec::<GenericArg<'tcx>>::new().into_iter())
+            List::empty()
         };
         debug!(?parent_substs);
 
         let did = def.did.to_def_id();
         let child_substs = InternalSubsts::identity_for_item(tcx, did);
-        let substs = tcx.mk_substs(parent_substs.into_iter().chain(child_substs.into_iter()));
+        let substs =
+            tcx.mk_substs_from_iter(parent_substs.into_iter().chain(child_substs.into_iter()));
         debug!(?substs);
 
         let hir_id = tcx.hir().local_def_id_to_hir_id(def.did);
@@ -2737,8 +2755,11 @@ impl UserTypeProjection {
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for UserTypeProjection {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for UserTypeProjection {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self, F::Error> {
         Ok(UserTypeProjection {
             base: self.base.try_fold_with(folder)?,
             projs: self.projs.try_fold_with(folder)?,
@@ -2746,8 +2767,11 @@ impl<'tcx> TypeFoldable<'tcx> for UserTypeProjection {
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx> for UserTypeProjection {
-    fn visit_with<Vs: TypeVisitor<'tcx>>(&self, visitor: &mut Vs) -> ControlFlow<Vs::BreakTy> {
+impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for UserTypeProjection {
+    fn visit_with<Vs: TypeVisitor<TyCtxt<'tcx>>>(
+        &self,
+        visitor: &mut Vs,
+    ) -> ControlFlow<Vs::BreakTy> {
         self.base.visit_with(visitor)
         // Note: there's nothing in `self.proj` to visit.
     }
@@ -2884,7 +2908,7 @@ fn pretty_print_const_value<'tcx>(
             // the `destructure_const` query with an empty `ty::ParamEnv` without
             // introducing ICEs (e.g. via `layout_of`) from missing bounds.
             // E.g. `transmute([0usize; 2]): (u8, *mut T)` needs to know `T: Sized`
-            // to be able to destructure the tuple into `(0u8, *mut T)
+            // to be able to destructure the tuple into `(0u8, *mut T)`
             //
             // FIXME(eddyb) for `--emit=mir`/`-Z dump-mir`, we should provide the
             // correct `ty::ParamEnv` to allow printing *all* constant values.
@@ -3049,7 +3073,7 @@ impl Location {
         if self.block == other.block {
             self.statement_index <= other.statement_index
         } else {
-            dominators.is_dominated_by(other.block, self.block)
+            dominators.dominates(self.block, other.block)
         }
     }
 }
diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs
index 1e8d5f7ea..7a05ee2ff 100644
--- a/compiler/rustc_middle/src/mir/mono.rs
+++ b/compiler/rustc_middle/src/mir/mono.rs
@@ -318,16 +318,19 @@ impl<'tcx> CodegenUnit<'tcx> {
         base_n::encode(hash, base_n::CASE_INSENSITIVE)
     }
 
-    pub fn estimate_size(&mut self, tcx: TyCtxt<'tcx>) {
+    pub fn create_size_estimate(&mut self, tcx: TyCtxt<'tcx>) {
         // Estimate the size of a codegen unit as (approximately) the number of MIR
         // statements it corresponds to.
         self.size_estimate = Some(self.items.keys().map(|mi| mi.size_estimate(tcx)).sum());
     }
 
     #[inline]
+    /// Should only be called if [`create_size_estimate`] has previously been called.
+    ///
+    /// [`create_size_estimate`]: Self::create_size_estimate
     pub fn size_estimate(&self) -> usize {
-        // Should only be called if `estimate_size` has previously been called.
-        self.size_estimate.expect("estimate_size must be called before getting a size_estimate")
+        self.size_estimate
+            .expect("create_size_estimate must be called before getting a size_estimate")
     }
 
     pub fn modify_size_estimate(&mut self, delta: usize) {
diff --git a/compiler/rustc_middle/src/mir/predecessors.rs b/compiler/rustc_middle/src/mir/predecessors.rs
deleted file mode 100644
index 5f1fadaf3..000000000
--- a/compiler/rustc_middle/src/mir/predecessors.rs
+++ /dev/null
@@ -1,78 +0,0 @@
-//! Lazily compute the reverse control-flow graph for the MIR.
-
-use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_data_structures::sync::OnceCell;
-use rustc_index::vec::IndexVec;
-use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
-use smallvec::SmallVec;
-
-use crate::mir::{BasicBlock, BasicBlockData};
-
-// Typically 95%+ of basic blocks have 4 or fewer predecessors.
-pub type Predecessors = IndexVec<BasicBlock, SmallVec<[BasicBlock; 4]>>;
-
-#[derive(Clone, Debug)]
-pub(super) struct PredecessorCache {
-    cache: OnceCell<Predecessors>,
-}
-
-impl PredecessorCache {
-    #[inline]
-    pub(super) fn new() -> Self {
-        PredecessorCache { cache: OnceCell::new() }
-    }
-
-    /// Invalidates the predecessor cache.
-    #[inline]
-    pub(super) fn invalidate(&mut self) {
-        // Invalidating the predecessor cache requires mutating the MIR, which in turn requires a
-        // unique reference (`&mut`) to the `mir::Body`. Because of this, we can assume that all
-        // callers of `invalidate` have a unique reference to the MIR and thus to the predecessor
-        // cache. This means we never need to do synchronization when `invalidate` is called, we can
-        // simply reinitialize the `OnceCell`.
-        self.cache = OnceCell::new();
-    }
-
-    /// Returns the predecessor graph for this MIR.
-    #[inline]
-    pub(super) fn compute(
-        &self,
-        basic_blocks: &IndexVec<BasicBlock, BasicBlockData<'_>>,
-    ) -> &Predecessors {
-        self.cache.get_or_init(|| {
-            let mut preds = IndexVec::from_elem(SmallVec::new(), basic_blocks);
-            for (bb, data) in basic_blocks.iter_enumerated() {
-                if let Some(term) = &data.terminator {
-                    for succ in term.successors() {
-                        preds[succ].push(bb);
-                    }
-                }
-            }
-
-            preds
-        })
-    }
-}
-
-impl<S: Encoder> Encodable<S> for PredecessorCache {
-    #[inline]
-    fn encode(&self, _s: &mut S) {}
-}
-
-impl<D: Decoder> Decodable<D> for PredecessorCache {
-    #[inline]
-    fn decode(_: &mut D) -> Self {
-        Self::new()
-    }
-}
-
-impl<CTX> HashStable<CTX> for PredecessorCache {
-    #[inline]
-    fn hash_stable(&self, _: &mut CTX, _: &mut StableHasher) {
-        // do nothing
-    }
-}
-
-TrivialTypeTraversalAndLiftImpls! {
-    PredecessorCache,
-}
diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs
index 40289af25..d8829e3e7 100644
--- a/compiler/rustc_middle/src/mir/pretty.rs
+++ b/compiler/rustc_middle/src/mir/pretty.rs
@@ -12,8 +12,8 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::def_id::DefId;
 use rustc_index::vec::Idx;
 use rustc_middle::mir::interpret::{
-    alloc_range, read_target_uint, AllocId, Allocation, ConstAllocation, ConstValue, GlobalAlloc,
-    Pointer, Provenance,
+    alloc_range, read_target_uint, AllocBytes, AllocId, Allocation, ConstAllocation, ConstValue,
+    GlobalAlloc, Pointer, Provenance,
 };
 use rustc_middle::mir::visit::Visitor;
 use rustc_middle::mir::*;
@@ -580,7 +580,7 @@ fn write_scope_tree(
             continue;
         }
 
-        let mut_str = if local_decl.mutability == Mutability::Mut { "mut " } else { "" };
+        let mut_str = local_decl.mutability.prefix_str();
 
         let mut indented_decl =
             format!("{0:1$}let {2}{3:?}: {4:?}", INDENT, indent, mut_str, local, local_decl.ty);
@@ -787,21 +787,21 @@ pub fn write_allocations<'tcx>(
 /// After the hex dump, an ascii dump follows, replacing all unprintable characters (control
 /// characters or characters whose value is larger than 127) with a `.`
 /// This also prints provenance adequately.
-pub fn display_allocation<'a, 'tcx, Prov: Provenance, Extra>(
+pub fn display_allocation<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes>(
     tcx: TyCtxt<'tcx>,
-    alloc: &'a Allocation<Prov, Extra>,
-) -> RenderAllocation<'a, 'tcx, Prov, Extra> {
+    alloc: &'a Allocation<Prov, Extra, Bytes>,
+) -> RenderAllocation<'a, 'tcx, Prov, Extra, Bytes> {
     RenderAllocation { tcx, alloc }
 }
 
 #[doc(hidden)]
-pub struct RenderAllocation<'a, 'tcx, Prov: Provenance, Extra> {
+pub struct RenderAllocation<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes> {
     tcx: TyCtxt<'tcx>,
-    alloc: &'a Allocation<Prov, Extra>,
+    alloc: &'a Allocation<Prov, Extra, Bytes>,
 }
 
-impl<'a, 'tcx, Prov: Provenance, Extra> std::fmt::Display
-    for RenderAllocation<'a, 'tcx, Prov, Extra>
+impl<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes> std::fmt::Display
+    for RenderAllocation<'a, 'tcx, Prov, Extra, Bytes>
 {
     fn fmt(&self, w: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let RenderAllocation { tcx, alloc } = *self;
@@ -845,9 +845,9 @@ fn write_allocation_newline(
 /// The `prefix` argument allows callers to add an arbitrary prefix before each line (even if there
 /// is only one line). Note that your prefix should contain a trailing space as the lines are
 /// printed directly after it.
-fn write_allocation_bytes<'tcx, Prov: Provenance, Extra>(
+fn write_allocation_bytes<'tcx, Prov: Provenance, Extra, Bytes: AllocBytes>(
     tcx: TyCtxt<'tcx>,
-    alloc: &Allocation<Prov, Extra>,
+    alloc: &Allocation<Prov, Extra, Bytes>,
     w: &mut dyn std::fmt::Write,
     prefix: &str,
 ) -> std::fmt::Result {
diff --git a/compiler/rustc_middle/src/mir/query.rs b/compiler/rustc_middle/src/mir/query.rs
index a8a453222..b964c1852 100644
--- a/compiler/rustc_middle/src/mir/query.rs
+++ b/compiler/rustc_middle/src/mir/query.rs
@@ -8,7 +8,7 @@ use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_index::bit_set::BitMatrix;
-use rustc_index::vec::IndexVec;
+use rustc_index::vec::{Idx, IndexVec};
 use rustc_span::Span;
 use rustc_target::abi::VariantIdx;
 use smallvec::SmallVec;
@@ -135,11 +135,20 @@ rustc_index::newtype_index! {
     pub struct GeneratorSavedLocal {}
 }
 
+#[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)]
+pub struct GeneratorSavedTy<'tcx> {
+    pub ty: Ty<'tcx>,
+    /// Source info corresponding to the local in the original MIR body.
+    pub source_info: SourceInfo,
+    /// Whether the local should be ignored for trait bound computations.
+    pub ignore_for_traits: bool,
+}
+
 /// The layout of generator state.
 #[derive(Clone, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)]
 pub struct GeneratorLayout<'tcx> {
     /// The type of every local stored inside the generator.
-    pub field_tys: IndexVec<GeneratorSavedLocal, Ty<'tcx>>,
+    pub field_tys: IndexVec<GeneratorSavedLocal, GeneratorSavedTy<'tcx>>,
 
     /// Which of the above fields are in each variant. Note that one field may
     /// be stored in multiple variants.
@@ -152,6 +161,8 @@ pub struct GeneratorLayout<'tcx> {
     /// Which saved locals are storage-live at the same time. Locals that do not
     /// have conflicts with each other are allowed to overlap in the computed
     /// layout.
+    #[type_foldable(identity)]
+    #[type_visitable(ignore)]
     pub storage_conflicts: BitMatrix<GeneratorSavedLocal, GeneratorSavedLocal>,
 }
 
@@ -278,13 +289,6 @@ pub struct ConstQualifs {
 /// instance of the closure is created, the corresponding free regions
 /// can be extracted from its type and constrained to have the given
 /// outlives relationship.
-///
-/// In some cases, we have to record outlives requirements between types and
-/// regions as well. In that case, if those types include any regions, those
-/// regions are recorded using their external names (`ReStatic`,
-/// `ReEarlyBound`, `ReFree`). We use these because in a query response we
-/// cannot use `ReVar` (which is what we use internally within the rest of the
-/// NLL code).
 #[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable)]
 pub struct ClosureRegionRequirements<'tcx> {
     /// The number of external regions defined on the closure. In our
@@ -381,16 +385,59 @@ pub enum ClosureOutlivesSubject<'tcx> {
     /// Subject is a type, typically a type parameter, but could also
     /// be a projection. Indicates a requirement like `T: 'a` being
     /// passed to the caller, where the type here is `T`.
-    ///
-    /// The type here is guaranteed not to contain any free regions at
-    /// present.
-    Ty(Ty<'tcx>),
+    Ty(ClosureOutlivesSubjectTy<'tcx>),
 
     /// Subject is a free region from the closure. Indicates a requirement
     /// like `'a: 'b` being passed to the caller; the region here is `'a`.
     Region(ty::RegionVid),
 }
 
+/// Represents a `ty::Ty` for use in [`ClosureOutlivesSubject`].
+///
+/// This abstraction is necessary because the type may include `ReVar` regions,
+/// which is what we use internally within NLL code, and they can't be used in
+/// a query response.
+///
+/// DO NOT implement `TypeVisitable` or `TypeFoldable` traits, because this
+/// type is not recognized as a binder for late-bound region.
+#[derive(Copy, Clone, Debug, TyEncodable, TyDecodable, HashStable)]
+pub struct ClosureOutlivesSubjectTy<'tcx> {
+    inner: Ty<'tcx>,
+}
+
+impl<'tcx> ClosureOutlivesSubjectTy<'tcx> {
+    /// All regions of `ty` must be of kind `ReVar` and must represent
+    /// universal regions *external* to the closure.
+    pub fn bind(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Self {
+        let inner = tcx.fold_regions(ty, |r, depth| match r.kind() {
+            ty::ReVar(vid) => {
+                let br = ty::BoundRegion {
+                    var: ty::BoundVar::new(vid.index()),
+                    kind: ty::BrAnon(vid.as_u32(), None),
+                };
+                tcx.mk_re_late_bound(depth, br)
+            }
+            _ => bug!("unexpected region in ClosureOutlivesSubjectTy: {r:?}"),
+        });
+
+        Self { inner }
+    }
+
+    pub fn instantiate(
+        self,
+        tcx: TyCtxt<'tcx>,
+        mut map: impl FnMut(ty::RegionVid) -> ty::Region<'tcx>,
+    ) -> Ty<'tcx> {
+        tcx.fold_regions(self.inner, |r, depth| match r.kind() {
+            ty::ReLateBound(debruijn, br) => {
+                debug_assert_eq!(debruijn, depth);
+                map(ty::RegionVid::new(br.var.index()))
+            }
+            _ => bug!("unexpected region {r:?}"),
+        })
+    }
+}
+
 /// The constituent parts of a mir constant of kind ADT or array.
 #[derive(Copy, Clone, Debug, HashStable)]
 pub struct DestructuredConstant<'tcx> {
diff --git a/compiler/rustc_middle/src/mir/spanview.rs b/compiler/rustc_middle/src/mir/spanview.rs
index 887ee5715..28a3b51b7 100644
--- a/compiler/rustc_middle/src/mir/spanview.rs
+++ b/compiler/rustc_middle/src/mir/spanview.rs
@@ -11,7 +11,7 @@ use std::io::{self, Write};
 pub const TOOLTIP_INDENT: &str = "    ";
 
 const CARET: char = '\u{2038}'; // Unicode `CARET`
-const ANNOTATION_LEFT_BRACKET: char = '\u{298a}'; // Unicode `Z NOTATION RIGHT BINDING BRACKET
+const ANNOTATION_LEFT_BRACKET: char = '\u{298a}'; // Unicode `Z NOTATION RIGHT BINDING BRACKET`
 const ANNOTATION_RIGHT_BRACKET: char = '\u{2989}'; // Unicode `Z NOTATION LEFT BINDING BRACKET`
 const NEW_LINE_SPAN: &str = "</span>\n<span class=\"line\">";
 const HEADER: &str = r#"<!DOCTYPE html>
@@ -250,6 +250,7 @@ pub fn statement_kind_name(statement: &Statement<'_>) -> &'static str {
         AscribeUserType(..) => "AscribeUserType",
         Coverage(..) => "Coverage",
         Intrinsic(..) => "Intrinsic",
+        ConstEvalCounter => "ConstEvalCounter",
         Nop => "Nop",
     }
 }
@@ -670,7 +671,7 @@ fn fn_span(tcx: TyCtxt<'_>, def_id: DefId) -> Span {
 
 fn hir_body(tcx: TyCtxt<'_>, def_id: DefId) -> Option<&rustc_hir::Body<'_>> {
     let hir_node = tcx.hir().get_if_local(def_id).expect("expected DefId is local");
-    hir::map::associated_body(hir_node).map(|fn_body_id| tcx.hir().body(fn_body_id))
+    hir::map::associated_body(hir_node).map(|(_, fn_body_id)| tcx.hir().body(fn_body_id))
 }
 
 fn escape_html(s: &str) -> String {
diff --git a/compiler/rustc_middle/src/mir/switch_sources.rs b/compiler/rustc_middle/src/mir/switch_sources.rs
deleted file mode 100644
index b91c0c257..000000000
--- a/compiler/rustc_middle/src/mir/switch_sources.rs
+++ /dev/null
@@ -1,78 +0,0 @@
-//! Lazily compute the inverse of each `SwitchInt`'s switch targets. Modeled after
-//! `Predecessors`/`PredecessorCache`.
-
-use rustc_data_structures::fx::FxHashMap;
-use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_data_structures::sync::OnceCell;
-use rustc_index::vec::IndexVec;
-use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
-use smallvec::SmallVec;
-
-use crate::mir::{BasicBlock, BasicBlockData, Terminator, TerminatorKind};
-
-pub type SwitchSources = FxHashMap<(BasicBlock, BasicBlock), SmallVec<[Option<u128>; 1]>>;
-
-#[derive(Clone, Debug)]
-pub(super) struct SwitchSourceCache {
-    cache: OnceCell<SwitchSources>,
-}
-
-impl SwitchSourceCache {
-    #[inline]
-    pub(super) fn new() -> Self {
-        SwitchSourceCache { cache: OnceCell::new() }
-    }
-
-    /// Invalidates the switch source cache.
-    #[inline]
-    pub(super) fn invalidate(&mut self) {
-        self.cache = OnceCell::new();
-    }
-
-    /// Returns the switch sources for this MIR.
-    #[inline]
-    pub(super) fn compute(
-        &self,
-        basic_blocks: &IndexVec<BasicBlock, BasicBlockData<'_>>,
-    ) -> &SwitchSources {
-        self.cache.get_or_init(|| {
-            let mut switch_sources: SwitchSources = FxHashMap::default();
-            for (bb, data) in basic_blocks.iter_enumerated() {
-                if let Some(Terminator {
-                    kind: TerminatorKind::SwitchInt { targets, .. }, ..
-                }) = &data.terminator
-                {
-                    for (value, target) in targets.iter() {
-                        switch_sources.entry((target, bb)).or_default().push(Some(value));
-                    }
-                    switch_sources.entry((targets.otherwise(), bb)).or_default().push(None);
-                }
-            }
-
-            switch_sources
-        })
-    }
-}
-
-impl<S: Encoder> Encodable<S> for SwitchSourceCache {
-    #[inline]
-    fn encode(&self, _s: &mut S) {}
-}
-
-impl<D: Decoder> Decodable<D> for SwitchSourceCache {
-    #[inline]
-    fn decode(_: &mut D) -> Self {
-        Self::new()
-    }
-}
-
-impl<CTX> HashStable<CTX> for SwitchSourceCache {
-    #[inline]
-    fn hash_stable(&self, _: &mut CTX, _: &mut StableHasher) {
-        // do nothing
-    }
-}
-
-TrivialTypeTraversalAndLiftImpls! {
-    SwitchSourceCache,
-}
diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs
index 52c2b10cb..ae09562a8 100644
--- a/compiler/rustc_middle/src/mir/syntax.rs
+++ b/compiler/rustc_middle/src/mir/syntax.rs
@@ -355,6 +355,12 @@ pub enum StatementKind<'tcx> {
     /// This avoids adding a new block and a terminator for simple intrinsics.
     Intrinsic(Box<NonDivergingIntrinsic<'tcx>>),
 
+    /// Instructs the const eval interpreter to increment a counter; this counter is used to track
+    /// how many steps the interpreter has taken. It is used to prevent the user from writing const
+    /// code that runs for too long or infinitely. Other than in the const eval interpreter, this
+    /// is a no-op.
+    ConstEvalCounter,
+
     /// No-op. Useful for deleting instructions without affecting statement indices.
     Nop,
 }
@@ -665,6 +671,12 @@ pub enum TerminatorKind<'tcx> {
     /// as parameters, and `None` for the destination. Keep in mind that the `cleanup` path is not
     /// necessarily executed even in the case of a panic, for example in `-C panic=abort`. If the
     /// assertion does not fail, execution continues at the specified basic block.
+    ///
+    /// When overflow checking is disabled and this is run-time MIR (as opposed to compile-time MIR
+    /// that is used for CTFE), the following variants of this terminator behave as `goto target`:
+    /// - `OverflowNeg(..)`,
+    /// - `Overflow(op, ..)` if op is a "checkable" operation (add, sub, mul, shl, shr, but NOT
+    /// div or rem).
     Assert {
         cond: Operand<'tcx>,
         expected: bool,
@@ -1097,10 +1109,6 @@ pub enum Rvalue<'tcx> {
 
     /// Same as `BinaryOp`, but yields `(T, bool)` with a `bool` indicating an error condition.
     ///
-    /// When overflow checking is disabled and we are generating run-time code, the error condition
-    /// is false. Otherwise, and always during CTFE, the error condition is determined as described
-    /// below.
-    ///
     /// For addition, subtraction, and multiplication on integers the error condition is set when
     /// the infinite precision result would be unequal to the actual result.
     ///
@@ -1197,10 +1205,8 @@ pub enum AggregateKind<'tcx> {
     /// active field index would identity the field `c`
     Adt(DefId, VariantIdx, SubstsRef<'tcx>, Option<UserTypeAnnotationIndex>, Option<usize>),
 
-    // Note: We can use LocalDefId since closures and generators a deaggregated
-    // before codegen.
-    Closure(LocalDefId, SubstsRef<'tcx>),
-    Generator(LocalDefId, SubstsRef<'tcx>, hir::Movability),
+    Closure(DefId, SubstsRef<'tcx>),
+    Generator(DefId, SubstsRef<'tcx>, hir::Movability),
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, TyEncodable, TyDecodable, Hash, HashStable)]
diff --git a/compiler/rustc_middle/src/mir/tcx.rs b/compiler/rustc_middle/src/mir/tcx.rs
index 599f0b9d3..0aa2c500f 100644
--- a/compiler/rustc_middle/src/mir/tcx.rs
+++ b/compiler/rustc_middle/src/mir/tcx.rs
@@ -97,7 +97,7 @@ impl<'tcx> PlaceTy<'tcx> {
                     ty::Slice(..) => self.ty,
                     ty::Array(inner, _) if !from_end => tcx.mk_array(*inner, (to - from) as u64),
                     ty::Array(inner, size) if from_end => {
-                        let size = size.eval_usize(tcx, param_env);
+                        let size = size.eval_target_usize(tcx, param_env);
                         let len = size - (from as u64) - (to as u64);
                         tcx.mk_array(*inner, len)
                     }
@@ -162,10 +162,10 @@ impl<'tcx> Rvalue<'tcx> {
         match *self {
             Rvalue::Use(ref operand) => operand.ty(local_decls, tcx),
             Rvalue::Repeat(ref operand, count) => {
-                tcx.mk_ty(ty::Array(operand.ty(local_decls, tcx), count))
+                tcx.mk_array_with_const_len(operand.ty(local_decls, tcx), count)
             }
             Rvalue::ThreadLocalRef(did) => {
-                let static_ty = tcx.type_of(did);
+                let static_ty = tcx.type_of(did).subst_identity();
                 if tcx.is_mutable_static(did) {
                     tcx.mk_mut_ptr(static_ty)
                 } else if tcx.is_foreign_item(did) {
@@ -194,20 +194,20 @@ impl<'tcx> Rvalue<'tcx> {
                 let lhs_ty = lhs.ty(local_decls, tcx);
                 let rhs_ty = rhs.ty(local_decls, tcx);
                 let ty = op.ty(tcx, lhs_ty, rhs_ty);
-                tcx.intern_tup(&[ty, tcx.types.bool])
+                tcx.mk_tup(&[ty, tcx.types.bool])
             }
             Rvalue::UnaryOp(UnOp::Not | UnOp::Neg, ref operand) => operand.ty(local_decls, tcx),
             Rvalue::Discriminant(ref place) => place.ty(local_decls, tcx).ty.discriminant_ty(tcx),
             Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf, _) => tcx.types.usize,
             Rvalue::Aggregate(ref ak, ref ops) => match **ak {
                 AggregateKind::Array(ty) => tcx.mk_array(ty, ops.len() as u64),
-                AggregateKind::Tuple => tcx.mk_tup(ops.iter().map(|op| op.ty(local_decls, tcx))),
-                AggregateKind::Adt(did, _, substs, _, _) => {
-                    tcx.bound_type_of(did).subst(tcx, substs)
+                AggregateKind::Tuple => {
+                    tcx.mk_tup_from_iter(ops.iter().map(|op| op.ty(local_decls, tcx)))
                 }
-                AggregateKind::Closure(did, substs) => tcx.mk_closure(did.to_def_id(), substs),
+                AggregateKind::Adt(did, _, substs, _, _) => tcx.type_of(did).subst(tcx, substs),
+                AggregateKind::Closure(did, substs) => tcx.mk_closure(did, substs),
                 AggregateKind::Generator(did, substs, movability) => {
-                    tcx.mk_generator(did.to_def_id(), substs, movability)
+                    tcx.mk_generator(did, substs, movability)
                 }
             },
             Rvalue::ShallowInitBox(_, ty) => tcx.mk_box(ty),
diff --git a/compiler/rustc_middle/src/mir/traversal.rs b/compiler/rustc_middle/src/mir/traversal.rs
index 0b461d1ce..f37222cb2 100644
--- a/compiler/rustc_middle/src/mir/traversal.rs
+++ b/compiler/rustc_middle/src/mir/traversal.rs
@@ -1,7 +1,4 @@
-use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_data_structures::sync::OnceCell;
 use rustc_index::bit_set::BitSet;
-use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
 
 use super::*;
 
@@ -339,50 +336,3 @@ pub fn reverse_postorder<'a, 'tcx>(body: &'a Body<'tcx>) -> ReversePostorderIter
     let len = blocks.len();
     ReversePostorderIter { body, blocks, idx: len }
 }
-
-#[derive(Clone, Debug)]
-pub(super) struct PostorderCache {
-    cache: OnceCell<Vec<BasicBlock>>,
-}
-
-impl PostorderCache {
-    #[inline]
-    pub(super) fn new() -> Self {
-        PostorderCache { cache: OnceCell::new() }
-    }
-
-    /// Invalidates the postorder cache.
-    #[inline]
-    pub(super) fn invalidate(&mut self) {
-        self.cache = OnceCell::new();
-    }
-
-    /// Returns the `&[BasicBlocks]` represents the postorder graph for this MIR.
-    #[inline]
-    pub(super) fn compute(&self, body: &IndexVec<BasicBlock, BasicBlockData<'_>>) -> &[BasicBlock] {
-        self.cache.get_or_init(|| Postorder::new(body, START_BLOCK).map(|(bb, _)| bb).collect())
-    }
-}
-
-impl<S: Encoder> Encodable<S> for PostorderCache {
-    #[inline]
-    fn encode(&self, _s: &mut S) {}
-}
-
-impl<D: Decoder> Decodable<D> for PostorderCache {
-    #[inline]
-    fn decode(_: &mut D) -> Self {
-        Self::new()
-    }
-}
-
-impl<CTX> HashStable<CTX> for PostorderCache {
-    #[inline]
-    fn hash_stable(&self, _: &mut CTX, _: &mut StableHasher) {
-        // do nothing
-    }
-}
-
-TrivialTypeTraversalAndLiftImpls! {
-    PostorderCache,
-}
diff --git a/compiler/rustc_middle/src/mir/type_foldable.rs b/compiler/rustc_middle/src/mir/type_foldable.rs
index 0705b4cff..988158321 100644
--- a/compiler/rustc_middle/src/mir/type_foldable.rs
+++ b/compiler/rustc_middle/src/mir/type_foldable.rs
@@ -30,26 +30,29 @@ TrivialTypeTraversalImpls! {
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for &'tcx [InlineAsmTemplatePiece] {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, _folder: &mut F) -> Result<Self, F::Error> {
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx [InlineAsmTemplatePiece] {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
+        _folder: &mut F,
+    ) -> Result<Self, F::Error> {
         Ok(self)
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for &'tcx [Span] {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, _folder: &mut F) -> Result<Self, F::Error> {
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx [Span] {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
+        _folder: &mut F,
+    ) -> Result<Self, F::Error> {
         Ok(self)
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<PlaceElem<'tcx>> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        ty::util::fold_list(self, folder, |tcx, v| tcx.intern_place_elems(v))
-    }
-}
-
-impl<'tcx, R: Idx, C: Idx> TypeFoldable<'tcx> for BitMatrix<R, C> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, _: &mut F) -> Result<Self, F::Error> {
-        Ok(self)
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx ty::List<PlaceElem<'tcx>> {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self, F::Error> {
+        ty::util::fold_list(self, folder, |tcx, v| tcx.mk_place_elems(v))
     }
 }
diff --git a/compiler/rustc_middle/src/mir/type_visitable.rs b/compiler/rustc_middle/src/mir/type_visitable.rs
deleted file mode 100644
index d44c6809b..000000000
--- a/compiler/rustc_middle/src/mir/type_visitable.rs
+++ /dev/null
@@ -1,9 +0,0 @@
-//! `TypeVisitable` implementations for MIR types
-
-use super::*;
-
-impl<'tcx, R: Idx, C: Idx> TypeVisitable<'tcx> for BitMatrix<R, C> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, _: &mut V) -> ControlFlow<V::BreakTy> {
-        ControlFlow::Continue(())
-    }
-}
diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs
index 1a264d2d5..5c056b299 100644
--- a/compiler/rustc_middle/src/mir/visit.rs
+++ b/compiler/rustc_middle/src/mir/visit.rs
@@ -323,7 +323,7 @@ macro_rules! make_mir_visitor {
                     self.visit_source_scope($(& $mutability)? *parent_scope);
                 }
                 if let Some((callee, callsite_span)) = inlined {
-                    let location = START_BLOCK.start_location();
+                    let location = Location::START;
 
                     self.visit_span($(& $mutability)? *callsite_span);
 
@@ -427,6 +427,7 @@ macro_rules! make_mir_visitor {
                             }
                         }
                     }
+                    StatementKind::ConstEvalCounter => {}
                     StatementKind::Nop => {}
                 }
             }
@@ -836,7 +837,7 @@ macro_rules! make_mir_visitor {
                 } = var_debug_info;
 
                 self.visit_source_info(source_info);
-                let location = START_BLOCK.start_location();
+                let location = Location::START;
                 match value {
                     VarDebugInfoContents::Const(c) => self.visit_constant(c, location),
                     VarDebugInfoContents::Place(place) =>
@@ -1025,7 +1026,7 @@ macro_rules! super_body {
         $self.visit_span($(& $mutability)? $body.span);
 
         for const_ in &$($mutability)? $body.required_consts {
-            let location = START_BLOCK.start_location();
+            let location = Location::START;
             $self.visit_constant(const_, location);
         }
     }
@@ -1044,7 +1045,7 @@ macro_rules! visit_place_fns {
             self.visit_local(&mut place.local, context, location);
 
             if let Some(new_projection) = self.process_projection(&place.projection, location) {
-                place.projection = self.tcx().intern_place_elems(&new_projection);
+                place.projection = self.tcx().mk_place_elems(&new_projection);
             }
         }
 
@@ -1213,7 +1214,7 @@ impl<'tcx> MirVisitable<'tcx> for Option<Terminator<'tcx>> {
 
 /// Extra information passed to `visit_ty` and friends to give context
 /// about where the type etc appears.
-#[derive(Debug)]
+#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq)]
 pub enum TyContext {
     LocalDecl {
         /// The index of the local variable we are visiting.
diff --git a/compiler/rustc_middle/src/query/keys.rs b/compiler/rustc_middle/src/query/keys.rs
index e4bb3ce3d..78ee8a6a8 100644
--- a/compiler/rustc_middle/src/query/keys.rs
+++ b/compiler/rustc_middle/src/query/keys.rs
@@ -4,11 +4,12 @@ use crate::infer::canonical::Canonical;
 use crate::mir;
 use crate::traits;
 use crate::ty::fast_reject::SimplifiedType;
+use crate::ty::layout::{TyAndLayout, ValidityRequirement};
 use crate::ty::subst::{GenericArg, SubstsRef};
-use crate::ty::{self, layout::TyAndLayout, Ty, TyCtxt};
+use crate::ty::{self, Ty, TyCtxt};
 use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, LOCAL_CRATE};
 use rustc_hir::hir_id::{HirId, OwnerId};
-use rustc_query_system::query::{DefaultCacheSelector, VecCacheSelector};
+use rustc_query_system::query::{DefaultCacheSelector, SingleCacheSelector, VecCacheSelector};
 use rustc_span::symbol::{Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
 
@@ -45,7 +46,7 @@ pub trait Key: Sized {
 }
 
 impl Key for () {
-    type CacheSelector = DefaultCacheSelector<Self>;
+    type CacheSelector = SingleCacheSelector;
 
     #[inline(always)]
     fn query_crate_is_local(&self) -> bool {
@@ -696,3 +697,24 @@ impl Key for HirId {
         None
     }
 }
+
+impl<'tcx> Key for (ValidityRequirement, ty::ParamEnvAnd<'tcx, Ty<'tcx>>) {
+    type CacheSelector = DefaultCacheSelector<Self>;
+
+    // Just forward to `Ty<'tcx>`
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+
+    fn ty_adt_id(&self) -> Option<DefId> {
+        match self.1.value.kind() {
+            ty::Adt(adt, _) => Some(adt.did()),
+            _ => None,
+        }
+    }
+}
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index 6bbf7fa39..5133da342 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -33,7 +33,7 @@ rustc_queries! {
     }
 
     query resolver_for_lowering(_: ()) -> &'tcx Steal<(ty::ResolverAstLowering, Lrc<ast::Crate>)> {
-        feedable
+        eval_always
         no_hash
         desc { "getting the resolver for lowering" }
     }
@@ -54,14 +54,14 @@ rustc_queries! {
     /// This is because the `hir_crate` query gives you access to all other items.
     /// To avoid this fate, do not call `tcx.hir().krate()`; instead,
     /// prefer wrappers like `tcx.visit_all_items_in_krate()`.
-    query hir_crate(key: ()) -> Crate<'tcx> {
+    query hir_crate(key: ()) -> &'tcx Crate<'tcx> {
         arena_cache
         eval_always
         desc { "getting the crate HIR" }
     }
 
     /// All items in the crate.
-    query hir_crate_items(_: ()) -> rustc_middle::hir::ModuleItems {
+    query hir_crate_items(_: ()) -> &'tcx rustc_middle::hir::ModuleItems {
         arena_cache
         eval_always
         desc { "getting HIR crate items" }
@@ -71,7 +71,7 @@ rustc_queries! {
     ///
     /// This can be conveniently accessed by `tcx.hir().visit_item_likes_in_module`.
     /// Avoid calling this query directly.
-    query hir_module_items(key: LocalDefId) -> rustc_middle::hir::ModuleItems {
+    query hir_module_items(key: LocalDefId) -> &'tcx rustc_middle::hir::ModuleItems {
         arena_cache
         desc { |tcx| "getting HIR module items in `{}`", tcx.def_path_str(key.to_def_id()) }
         cache_on_disk_if { true }
@@ -85,12 +85,12 @@ rustc_queries! {
         desc { |tcx| "getting HIR owner of `{}`", tcx.def_path_str(key.to_def_id()) }
     }
 
-    /// Gives access to the HIR ID for the given `LocalDefId` owner `key`.
+    /// Gives access to the HIR ID for the given `LocalDefId` owner `key` if any.
     ///
-    /// This can be conveniently accessed by methods on `tcx.hir()`.
-    /// Avoid calling this query directly.
-    query local_def_id_to_hir_id(key: LocalDefId) -> hir::HirId {
+    /// Definitions that were generated with no HIR, would be feeded to return `None`.
+    query opt_local_def_id_to_hir_id(key: LocalDefId) -> Option<hir::HirId>{
         desc { |tcx| "getting HIR ID of `{}`", tcx.def_path_str(key.to_def_id()) }
+        feedable
     }
 
     /// Gives access to the HIR node's parent for the HIR owner `key`.
@@ -152,7 +152,7 @@ rustc_queries! {
     /// to an alias, it will "skip" this alias to return the aliased type.
     ///
     /// [`DefId`]: rustc_hir::def_id::DefId
-    query type_of(key: DefId) -> Ty<'tcx> {
+    query type_of(key: DefId) -> ty::EarlyBinder<Ty<'tcx>> {
         desc { |tcx|
             "{action} `{path}`",
             action = {
@@ -167,6 +167,7 @@ rustc_queries! {
         }
         cache_on_disk_if { key.is_local() }
         separate_provide_extern
+        feedable
     }
 
     query collect_return_position_impl_trait_in_trait_tys(key: DefId)
@@ -183,6 +184,15 @@ rustc_queries! {
         separate_provide_extern
     }
 
+    query unsizing_params_for_adt(key: DefId) -> &'tcx rustc_index::bit_set::BitSet<u32>
+    {
+        arena_cache
+        desc { |tcx|
+            "determining what parameters of `{}` can participate in unsizing",
+            tcx.def_path_str(key),
+        }
+    }
+
     query analysis(key: ()) -> Result<(), ErrorGuaranteed> {
         eval_always
         desc { "running analysis passes on this crate" }
@@ -209,11 +219,12 @@ rustc_queries! {
 
     /// Maps from the `DefId` of an item (trait/struct/enum/fn) to its
     /// associated generics.
-    query generics_of(key: DefId) -> ty::Generics {
+    query generics_of(key: DefId) -> &'tcx ty::Generics {
         desc { |tcx| "computing generics of `{}`", tcx.def_path_str(key) }
         arena_cache
         cache_on_disk_if { key.is_local() }
         separate_provide_extern
+        feedable
     }
 
     /// Maps from the `DefId` of an item (trait/struct/enum/fn) to the
@@ -256,6 +267,7 @@ rustc_queries! {
         desc { |tcx| "finding item bounds for `{}`", tcx.def_path_str(key) }
         cache_on_disk_if { key.is_local() }
         separate_provide_extern
+        feedable
     }
 
     /// Elaborated version of the predicates from `explicit_item_bounds`.
@@ -286,19 +298,19 @@ rustc_queries! {
     /// These are assembled from the following places:
     /// - `extern` blocks (depending on their `link` attributes)
     /// - the `libs` (`-l`) option
-    query native_libraries(_: CrateNum) -> Vec<NativeLib> {
+    query native_libraries(_: CrateNum) -> &'tcx Vec<NativeLib> {
         arena_cache
         desc { "looking up the native libraries of a linked crate" }
         separate_provide_extern
     }
 
-    query shallow_lint_levels_on(key: hir::OwnerId) -> rustc_middle::lint::ShallowLintLevelMap {
+    query shallow_lint_levels_on(key: hir::OwnerId) -> &'tcx rustc_middle::lint::ShallowLintLevelMap {
         eval_always // fetches `resolutions`
         arena_cache
         desc { |tcx| "looking up lint levels for `{}`", tcx.def_path_str(key.to_def_id()) }
     }
 
-    query lint_expectations(_: ()) -> Vec<(LintExpectationId, LintExpectation)> {
+    query lint_expectations(_: ()) -> &'tcx Vec<(LintExpectationId, LintExpectation)> {
         arena_cache
         desc { "computing `#[expect]`ed lints in this crate" }
     }
@@ -338,7 +350,7 @@ rustc_queries! {
     }
 
     /// Set of param indexes for type params that are in the type's representation
-    query params_in_repr(key: DefId) -> rustc_index::bit_set::BitSet<u32> {
+    query params_in_repr(key: DefId) -> &'tcx rustc_index::bit_set::BitSet<u32> {
         desc { "finding type parameters in the representation" }
         arena_cache
         no_hash
@@ -355,16 +367,23 @@ rustc_queries! {
     }
 
     /// Create a THIR tree for debugging.
-    query thir_tree(key: ty::WithOptConstParam<LocalDefId>) -> String {
+    query thir_tree(key: ty::WithOptConstParam<LocalDefId>) -> &'tcx String {
         no_hash
         arena_cache
         desc { |tcx| "constructing THIR tree for `{}`", tcx.def_path_str(key.did.to_def_id()) }
     }
 
+    /// Create a list-like THIR representation for debugging.
+    query thir_flat(key: ty::WithOptConstParam<LocalDefId>) -> &'tcx String {
+        no_hash
+        arena_cache
+        desc { |tcx| "constructing flat THIR representation for `{}`", tcx.def_path_str(key.did.to_def_id()) }
+    }
+
     /// Set of all the `DefId`s in this crate that have MIR associated with
     /// them. This includes all the body owners, but also things like struct
     /// constructors.
-    query mir_keys(_: ()) -> rustc_data_structures::fx::FxIndexSet<LocalDefId> {
+    query mir_keys(_: ()) -> &'tcx rustc_data_structures::fx::FxIndexSet<LocalDefId> {
         arena_cache
         desc { "getting a list of all mir_keys" }
     }
@@ -460,17 +479,24 @@ rustc_queries! {
         }
     }
 
-    query symbols_for_closure_captures(
-        key: (LocalDefId, LocalDefId)
-    ) -> Vec<rustc_span::Symbol> {
-        arena_cache
+    query closure_typeinfo(key: LocalDefId) -> ty::ClosureTypeInfo<'tcx> {
         desc {
-            |tcx| "finding symbols for captures of closure `{}` in `{}`",
-            tcx.def_path_str(key.1.to_def_id()),
-            tcx.def_path_str(key.0.to_def_id())
+            |tcx| "finding symbols for captures of closure `{}`",
+            tcx.def_path_str(key.to_def_id())
         }
     }
 
+    query mir_generator_witnesses(key: DefId) -> &'tcx mir::GeneratorLayout<'tcx> {
+        arena_cache
+        desc { |tcx| "generator witness types for `{}`", tcx.def_path_str(key) }
+        cache_on_disk_if { key.is_local() }
+        separate_provide_extern
+    }
+
+    query check_generator_obligations(key: LocalDefId) {
+        desc { |tcx| "verify auto trait bounds for generator interior type `{}`", tcx.def_path_str(key.to_def_id()) }
+    }
+
     /// MIR after our optimization passes have run. This is MIR that is ready
     /// for codegen. This is also the only query that can fetch non-local MIR, at present.
     query optimized_mir(key: DefId) -> &'tcx mir::Body<'tcx> {
@@ -481,14 +507,14 @@ rustc_queries! {
 
     /// Returns coverage summary info for a function, after executing the `InstrumentCoverage`
     /// MIR pass (assuming the -Cinstrument-coverage option is enabled).
-    query coverageinfo(key: ty::InstanceDef<'tcx>) -> mir::CoverageInfo {
+    query coverageinfo(key: ty::InstanceDef<'tcx>) -> &'tcx mir::CoverageInfo {
         desc { |tcx| "retrieving coverage info from MIR for `{}`", tcx.def_path_str(key.def_id()) }
         arena_cache
     }
 
     /// Returns the `CodeRegions` for a function that has instrumented coverage, in case the
     /// function was optimized out before codegen, and before being added to the Coverage Map.
-    query covered_code_regions(key: DefId) -> Vec<&'tcx mir::coverage::CodeRegion> {
+    query covered_code_regions(key: DefId) -> &'tcx Vec<&'tcx mir::coverage::CodeRegion> {
         desc {
             |tcx| "retrieving the covered `CodeRegion`s, if instrumented, for `{}`",
             tcx.def_path_str(key)
@@ -530,7 +556,7 @@ rustc_queries! {
         desc { "erasing regions from `{}`", ty }
     }
 
-    query wasm_import_module_map(_: CrateNum) -> FxHashMap<DefId, String> {
+    query wasm_import_module_map(_: CrateNum) -> &'tcx FxHashMap<DefId, String> {
         arena_cache
         desc { "getting wasm import module map" }
     }
@@ -566,6 +592,7 @@ rustc_queries! {
         desc { |tcx| "computing explicit predicates of `{}`", tcx.def_path_str(key) }
         cache_on_disk_if { key.is_local() }
         separate_provide_extern
+        feedable
     }
 
     /// Returns the inferred outlives predicates (e.g., for `struct
@@ -574,6 +601,7 @@ rustc_queries! {
         desc { |tcx| "computing inferred outlives predicates of `{}`", tcx.def_path_str(key) }
         cache_on_disk_if { key.is_local() }
         separate_provide_extern
+        feedable
     }
 
     /// Maps from the `DefId` of a trait to the list of
@@ -605,7 +633,7 @@ rustc_queries! {
         desc { |tcx| "computing the bounds for type parameter `{}`", tcx.hir().ty_param_name(key.1) }
     }
 
-    query trait_def(key: DefId) -> ty::TraitDef {
+    query trait_def(key: DefId) -> &'tcx ty::TraitDef {
         desc { |tcx| "computing trait definition for `{}`", tcx.def_path_str(key) }
         arena_cache
         cache_on_disk_if { key.is_local() }
@@ -676,7 +704,7 @@ rustc_queries! {
     }
 
     /// Gets a map with the variance of every item; use `item_variance` instead.
-    query crate_variances(_: ()) -> ty::CrateVariancesMap<'tcx> {
+    query crate_variances(_: ()) -> &'tcx ty::CrateVariancesMap<'tcx> {
         arena_cache
         desc { "computing the variances for items in this crate" }
     }
@@ -689,7 +717,7 @@ rustc_queries! {
     }
 
     /// Maps from thee `DefId` of a type to its (inferred) outlives.
-    query inferred_outlives_crate(_: ()) -> ty::CratePredicatesMap<'tcx> {
+    query inferred_outlives_crate(_: ()) -> &'tcx ty::CratePredicatesMap<'tcx> {
         arena_cache
         desc { "computing the inferred outlives predicates for items in this crate" }
     }
@@ -704,13 +732,13 @@ rustc_queries! {
     /// Maps from a trait item to the trait item "descriptor".
     query associated_item(key: DefId) -> ty::AssocItem {
         desc { |tcx| "computing associated item data for `{}`", tcx.def_path_str(key) }
-        arena_cache
         cache_on_disk_if { key.is_local() }
         separate_provide_extern
+        feedable
     }
 
     /// Collects the associated items defined on a trait or impl.
-    query associated_items(key: DefId) -> ty::AssocItems<'tcx> {
+    query associated_items(key: DefId) -> &'tcx ty::AssocItems {
         arena_cache
         desc { |tcx| "collecting associated items of `{}`", tcx.def_path_str(key) }
     }
@@ -736,11 +764,31 @@ rustc_queries! {
     ///
     /// The map returned for `tcx.impl_item_implementor_ids(impl_id)` would be
     ///`{ trait_f: impl_f, trait_g: impl_g }`
-    query impl_item_implementor_ids(impl_id: DefId) -> FxHashMap<DefId, DefId> {
+    query impl_item_implementor_ids(impl_id: DefId) -> &'tcx FxHashMap<DefId, DefId> {
         arena_cache
         desc { |tcx| "comparing impl items against trait for `{}`", tcx.def_path_str(impl_id) }
     }
 
+    /// Given `fn_def_id` of a trait or of an impl that implements a given trait:
+    /// if `fn_def_id` is the def id of a function defined inside a trait, then it creates and returns
+    /// the associated items that correspond to each impl trait in return position for that trait.
+    /// if `fn_def_id` is the def id of a function defined inside an impl that implements a trait, then it
+    /// creates and returns the associated items that correspond to each impl trait in return position
+    /// of the implemented trait.
+    query associated_items_for_impl_trait_in_trait(fn_def_id: DefId) -> &'tcx [DefId] {
+        desc { |tcx| "creating associated items for impl trait in trait returned by `{}`", tcx.def_path_str(fn_def_id) }
+        cache_on_disk_if { fn_def_id.is_local() }
+        separate_provide_extern
+    }
+
+    /// Given an impl trait in trait `opaque_ty_def_id`, create and return the corresponding
+    /// associated item.
+    query associated_item_for_impl_trait_in_trait(opaque_ty_def_id: LocalDefId) -> LocalDefId {
+        desc { |tcx| "creates the associated item corresponding to the opaque type `{}`", tcx.def_path_str(opaque_ty_def_id.to_def_id()) }
+        cache_on_disk_if { true }
+        separate_provide_extern
+    }
+
     /// Given an `impl_id`, return the trait it implements.
     /// Return `None` if this is an inherent impl.
     query impl_trait_ref(impl_id: DefId) -> Option<ty::EarlyBinder<ty::TraitRef<'tcx>>> {
@@ -754,7 +802,7 @@ rustc_queries! {
         separate_provide_extern
     }
 
-    query issue33140_self_ty(key: DefId) -> Option<ty::Ty<'tcx>> {
+    query issue33140_self_ty(key: DefId) -> Option<ty::EarlyBinder<ty::Ty<'tcx>>> {
         desc { |tcx| "computing Self type wrt issue #33140 `{}`", tcx.def_path_str(key) }
     }
 
@@ -796,15 +844,6 @@ rustc_queries! {
         }
     }
 
-    /// HACK: when evaluated, this reports an "unsafe derive on repr(packed)" error.
-    ///
-    /// Unsafety checking is executed for each method separately, but we only want
-    /// to emit this error once per derive. As there are some impls with multiple
-    /// methods, we use a query for deduplication.
-    query unsafe_derive_on_repr_packed(key: LocalDefId) -> () {
-        desc { |tcx| "processing `{}`", tcx.def_path_str(key.to_def_id()) }
-    }
-
     /// Returns the types assumed to be well formed while "inside" of the given item.
     ///
     /// Note that we've liberated the late bound regions of function signatures, so
@@ -814,7 +853,7 @@ rustc_queries! {
     }
 
     /// Computes the signature of the function.
-    query fn_sig(key: DefId) -> ty::PolyFnSig<'tcx> {
+    query fn_sig(key: DefId) -> ty::EarlyBinder<ty::PolyFnSig<'tcx>> {
         desc { |tcx| "computing function signature of `{}`", tcx.def_path_str(key) }
         cache_on_disk_if { key.is_local() }
         separate_provide_extern
@@ -866,7 +905,7 @@ rustc_queries! {
     ///
     /// The second return value maps from ADTs to ignored derived traits (e.g. Debug and Clone) and
     /// their respective impl (i.e., part of the derive macro)
-    query live_symbols_and_ignored_derived_traits(_: ()) -> (
+    query live_symbols_and_ignored_derived_traits(_: ()) -> &'tcx (
         FxHashSet<LocalDefId>,
         FxHashMap<LocalDefId, Vec<(DefId, DefId)>>
     ) {
@@ -946,7 +985,7 @@ rustc_queries! {
 
     /// Gets a complete map from all types to their inherent impls.
     /// Not meant to be used directly outside of coherence.
-    query crate_inherent_impls(k: ()) -> CrateInherentImpls {
+    query crate_inherent_impls(k: ()) -> &'tcx CrateInherentImpls {
         arena_cache
         desc { "finding all inherent impls defined in crate" }
     }
@@ -1081,7 +1120,7 @@ rustc_queries! {
         desc { "checking for private elements in public interfaces" }
     }
 
-    query reachable_set(_: ()) -> FxHashSet<LocalDefId> {
+    query reachable_set(_: ()) -> &'tcx FxHashSet<LocalDefId> {
         arena_cache
         desc { "reachability" }
     }
@@ -1093,7 +1132,7 @@ rustc_queries! {
     }
 
     /// Generates a MIR body for the shim.
-    query mir_shims(key: ty::InstanceDef<'tcx>) -> mir::Body<'tcx> {
+    query mir_shims(key: ty::InstanceDef<'tcx>) -> &'tcx mir::Body<'tcx> {
         arena_cache
         desc { |tcx| "generating MIR shim for `{}`", tcx.def_path_str(key.def_id()) }
     }
@@ -1110,6 +1149,15 @@ rustc_queries! {
         desc { |tcx| "looking up definition kind of `{}`", tcx.def_path_str(def_id) }
         cache_on_disk_if { def_id.is_local() }
         separate_provide_extern
+        feedable
+    }
+
+    /// The `opt_rpitit_info` query returns the pair of the def id of the function where the RPIT
+    /// is defined and the opaque def id if any.
+    query opt_rpitit_info(def_id: DefId) -> Option<ty::ImplTraitInTraitData> {
+        desc { |tcx| "opt_rpitit_info `{}`", tcx.def_path_str(def_id) }
+        cache_on_disk_if { def_id.is_local() }
+        feedable
     }
 
     /// Gets the span for the definition.
@@ -1125,6 +1173,7 @@ rustc_queries! {
         desc { |tcx| "looking up span for `{}`'s identifier", tcx.def_path_str(def_id) }
         cache_on_disk_if { def_id.is_local() }
         separate_provide_extern
+        feedable
     }
 
     query lookup_stability(def_id: DefId) -> Option<attr::Stability> {
@@ -1157,6 +1206,7 @@ rustc_queries! {
     /// Determines whether an item is annotated with `doc(hidden)`.
     query is_doc_hidden(def_id: DefId) -> bool {
         desc { |tcx| "checking whether `{}` is `doc(hidden)`", tcx.def_path_str(def_id) }
+        separate_provide_extern
     }
 
     /// Determines whether an item is annotated with `doc(notable_trait)`.
@@ -1172,7 +1222,7 @@ rustc_queries! {
         separate_provide_extern
     }
 
-    query codegen_fn_attrs(def_id: DefId) -> CodegenFnAttrs {
+    query codegen_fn_attrs(def_id: DefId) -> &'tcx CodegenFnAttrs {
         desc { |tcx| "computing codegen attributes of `{}`", tcx.def_path_str(def_id) }
         arena_cache
         cache_on_disk_if { def_id.is_local() }
@@ -1190,7 +1240,7 @@ rustc_queries! {
     }
     /// Gets the rendered value of the specified constant or associated constant.
     /// Used by rustdoc.
-    query rendered_const(def_id: DefId) -> String {
+    query rendered_const(def_id: DefId) -> &'tcx String {
         arena_cache
         desc { |tcx| "rendering constant initializer of `{}`", tcx.def_path_str(def_id) }
         cache_on_disk_if { def_id.is_local() }
@@ -1249,12 +1299,12 @@ rustc_queries! {
     }
 
     /// Given a trait `trait_id`, return all known `impl` blocks.
-    query trait_impls_of(trait_id: DefId) -> ty::trait_def::TraitImpls {
+    query trait_impls_of(trait_id: DefId) -> &'tcx ty::trait_def::TraitImpls {
         arena_cache
         desc { |tcx| "finding trait impls of `{}`", tcx.def_path_str(trait_id) }
     }
 
-    query specialization_graph_of(trait_id: DefId) -> specialization_graph::Graph {
+    query specialization_graph_of(trait_id: DefId) -> &'tcx specialization_graph::Graph {
         arena_cache
         desc { |tcx| "building specialization graph of trait `{}`", tcx.def_path_str(trait_id) }
         cache_on_disk_if { true }
@@ -1262,6 +1312,9 @@ rustc_queries! {
     query object_safety_violations(trait_id: DefId) -> &'tcx [traits::ObjectSafetyViolation] {
         desc { |tcx| "determining object safety of trait `{}`", tcx.def_path_str(trait_id) }
     }
+    query check_is_object_safe(trait_id: DefId) -> bool {
+        desc { |tcx| "checking if trait `{}` is object safe", tcx.def_path_str(trait_id) }
+    }
 
     /// Gets the ParameterEnvironment for a given item; this environment
     /// will be in "user-facing" mode, meaning that it is suitable for
@@ -1381,7 +1434,7 @@ rustc_queries! {
         separate_provide_extern
     }
 
-    query dependency_formats(_: ()) -> Lrc<crate::middle::dependency_format::Dependencies> {
+    query dependency_formats(_: ()) -> &'tcx Lrc<crate::middle::dependency_format::Dependencies> {
         arena_cache
         desc { "getting the linkage format of all dependencies" }
     }
@@ -1462,6 +1515,7 @@ rustc_queries! {
         desc { |tcx| "looking up whether `{}` is a default impl", tcx.def_path_str(def_id) }
         cache_on_disk_if { def_id.is_local() }
         separate_provide_extern
+        feedable
     }
 
     query check_well_formed(key: hir::OwnerId) -> () {
@@ -1481,7 +1535,7 @@ rustc_queries! {
     // Does not include external symbols that don't have a corresponding DefId,
     // like the compiler-generated `main` function and so on.
     query reachable_non_generics(_: CrateNum)
-        -> DefIdMap<SymbolExportInfo> {
+        -> &'tcx DefIdMap<SymbolExportInfo> {
         arena_cache
         desc { "looking up the exported symbols of a crate" }
         separate_provide_extern
@@ -1504,7 +1558,7 @@ rustc_queries! {
     /// added or removed in any upstream crate. Instead use the narrower
     /// `upstream_monomorphizations_for`, `upstream_drop_glue_for`, or, even
     /// better, `Instance::upstream_monomorphization()`.
-    query upstream_monomorphizations(_: ()) -> DefIdMap<FxHashMap<SubstsRef<'tcx>, CrateNum>> {
+    query upstream_monomorphizations(_: ()) -> &'tcx DefIdMap<FxHashMap<SubstsRef<'tcx>, CrateNum>> {
         arena_cache
         desc { "collecting available upstream monomorphizations" }
     }
@@ -1519,7 +1573,6 @@ rustc_queries! {
     query upstream_monomorphizations_for(def_id: DefId)
         -> Option<&'tcx FxHashMap<SubstsRef<'tcx>, CrateNum>>
     {
-        arena_cache
         desc { |tcx|
             "collecting available upstream monomorphizations for `{}`",
             tcx.def_path_str(def_id),
@@ -1547,7 +1600,7 @@ rustc_queries! {
     }
 
     /// Returns a list of all `extern` blocks of a crate.
-    query foreign_modules(_: CrateNum) -> FxHashMap<DefId, ForeignModule> {
+    query foreign_modules(_: CrateNum) -> &'tcx FxHashMap<DefId, ForeignModule> {
         arena_cache
         desc { "looking up the foreign modules of a linked crate" }
         separate_provide_extern
@@ -1581,7 +1634,7 @@ rustc_queries! {
 
     /// Gets the extra data to put in each output filename for a crate.
     /// For example, compiling the `foo` crate with `extra-filename=-a` creates a `libfoo-b.rlib` file.
-    query extra_filename(_: CrateNum) -> String {
+    query extra_filename(_: CrateNum) -> &'tcx String {
         arena_cache
         eval_always
         desc { "looking up the extra filename for a crate" }
@@ -1589,7 +1642,7 @@ rustc_queries! {
     }
 
     /// Gets the paths where the crate came from in the file system.
-    query crate_extern_paths(_: CrateNum) -> Vec<PathBuf> {
+    query crate_extern_paths(_: CrateNum) -> &'tcx Vec<PathBuf> {
         arena_cache
         eval_always
         desc { "looking up the paths for extern crates" }
@@ -1619,16 +1672,16 @@ rustc_queries! {
 
     /// Does lifetime resolution on items. Importantly, we can't resolve
     /// lifetimes directly on things like trait methods, because of trait params.
-    /// See `rustc_resolve::late::lifetimes for details.
-    query resolve_lifetimes(_: hir::OwnerId) -> ResolveLifetimes {
+    /// See `rustc_resolve::late::lifetimes` for details.
+    query resolve_bound_vars(_: hir::OwnerId) -> &'tcx ResolveBoundVars {
         arena_cache
         desc { "resolving lifetimes" }
     }
-    query named_region_map(_: hir::OwnerId) ->
-        Option<&'tcx FxHashMap<ItemLocalId, Region>> {
+    query named_variable_map(_: hir::OwnerId) ->
+        Option<&'tcx FxHashMap<ItemLocalId, ResolvedArg>> {
         desc { "looking up a named region" }
     }
-    query is_late_bound_map(_: LocalDefId) -> Option<&'tcx FxIndexSet<LocalDefId>> {
+    query is_late_bound_map(_: hir::OwnerId) -> Option<&'tcx FxIndexSet<ItemLocalId>> {
         desc { "testing if a region is late bound" }
     }
     /// For a given item's generic parameter, gets the default lifetimes to be used
@@ -1660,6 +1713,7 @@ rustc_queries! {
     query visibility(def_id: DefId) -> ty::Visibility<DefId> {
         desc { |tcx| "computing visibility of `{}`", tcx.def_path_str(def_id) }
         separate_provide_extern
+        feedable
     }
 
     query inhabited_predicate_adt(key: DefId) -> ty::inhabitedness::InhabitedPredicate<'tcx> {
@@ -1691,7 +1745,7 @@ rustc_queries! {
         desc { |tcx| "computing crate imported by `{}`", tcx.def_path_str(def_id.to_def_id()) }
     }
 
-    query lib_features(_: ()) -> LibFeatures {
+    query lib_features(_: ()) -> &'tcx LibFeatures {
         arena_cache
         desc { "calculating the lib features map" }
     }
@@ -1699,7 +1753,7 @@ rustc_queries! {
         desc { "calculating the lib features defined in a crate" }
         separate_provide_extern
     }
-    query stability_implications(_: CrateNum) -> FxHashMap<Symbol, Symbol> {
+    query stability_implications(_: CrateNum) -> &'tcx FxHashMap<Symbol, Symbol> {
         arena_cache
         desc { "calculating the implications between `#[unstable]` features defined in a crate" }
         separate_provide_extern
@@ -1710,14 +1764,14 @@ rustc_queries! {
         separate_provide_extern
     }
     /// Returns the lang items defined in another crate by loading it from metadata.
-    query get_lang_items(_: ()) -> LanguageItems {
+    query get_lang_items(_: ()) -> &'tcx LanguageItems {
         arena_cache
         eval_always
         desc { "calculating the lang items map" }
     }
 
     /// Returns all diagnostic items defined in all crates.
-    query all_diagnostic_items(_: ()) -> rustc_hir::diagnostic_items::DiagnosticItems {
+    query all_diagnostic_items(_: ()) -> &'tcx rustc_hir::diagnostic_items::DiagnosticItems {
         arena_cache
         eval_always
         desc { "calculating the diagnostic items map" }
@@ -1730,7 +1784,7 @@ rustc_queries! {
     }
 
     /// Returns the diagnostic items defined in a crate.
-    query diagnostic_items(_: CrateNum) -> rustc_hir::diagnostic_items::DiagnosticItems {
+    query diagnostic_items(_: CrateNum) -> &'tcx rustc_hir::diagnostic_items::DiagnosticItems {
         arena_cache
         desc { "calculating the diagnostic items map in a crate" }
         separate_provide_extern
@@ -1740,11 +1794,11 @@ rustc_queries! {
         desc { "calculating the missing lang items in a crate" }
         separate_provide_extern
     }
-    query visible_parent_map(_: ()) -> DefIdMap<DefId> {
+    query visible_parent_map(_: ()) -> &'tcx DefIdMap<DefId> {
         arena_cache
         desc { "calculating the visible parent map" }
     }
-    query trimmed_def_paths(_: ()) -> FxHashMap<DefId, Symbol> {
+    query trimmed_def_paths(_: ()) -> &'tcx FxHashMap<DefId, Symbol> {
         arena_cache
         desc { "calculating trimmed def paths" }
     }
@@ -1753,14 +1807,14 @@ rustc_queries! {
         desc { "seeing if we're missing an `extern crate` item for this crate" }
         separate_provide_extern
     }
-    query used_crate_source(_: CrateNum) -> Lrc<CrateSource> {
+    query used_crate_source(_: CrateNum) -> &'tcx Lrc<CrateSource> {
         arena_cache
         eval_always
         desc { "looking at the source for a crate" }
         separate_provide_extern
     }
     /// Returns the debugger visualizers defined for this crate.
-    query debugger_visualizers(_: CrateNum) -> Vec<rustc_span::DebuggerVisualizerFile> {
+    query debugger_visualizers(_: CrateNum) -> &'tcx Vec<rustc_span::DebuggerVisualizerFile> {
         arena_cache
         desc { "looking up the debugger visualizers for this crate" }
         separate_provide_extern
@@ -1791,14 +1845,11 @@ rustc_queries! {
     query maybe_unused_trait_imports(_: ()) -> &'tcx FxIndexSet<LocalDefId> {
         desc { "fetching potentially unused trait imports" }
     }
-    query maybe_unused_extern_crates(_: ()) -> &'tcx [(LocalDefId, Span)] {
-        desc { "looking up all possibly unused extern crates" }
-    }
     query names_imported_by_glob_use(def_id: LocalDefId) -> &'tcx FxHashSet<Symbol> {
         desc { |tcx| "finding names imported by glob use for `{}`", tcx.def_path_str(def_id.to_def_id()) }
     }
 
-    query stability_index(_: ()) -> stability::Index {
+    query stability_index(_: ()) -> &'tcx stability::Index {
         arena_cache
         eval_always
         desc { "calculating the stability index for the local crate" }
@@ -1815,6 +1866,11 @@ rustc_queries! {
         separate_provide_extern
     }
 
+    query trait_impls_in_crate(_: CrateNum) -> &'tcx [DefId] {
+        desc { "fetching all trait impls in a crate" }
+        separate_provide_extern
+    }
+
     /// The list of symbols exported from the given crate.
     ///
     /// - All names contained in `exported_symbols(cnum)` are guaranteed to
@@ -1865,6 +1921,7 @@ rustc_queries! {
     query output_filenames(_: ()) -> &'tcx Arc<OutputFilenames> {
         feedable
         desc { "getting output filenames" }
+        arena_cache
     }
 
     /// Do not call this query directly: invoke `normalize` instead.
@@ -2034,7 +2091,7 @@ rustc_queries! {
         remap_env_constness
     }
 
-    query supported_target_features(_: CrateNum) -> FxHashMap<String, Option<Symbol>> {
+    query supported_target_features(_: CrateNum) -> &'tcx FxHashMap<String, Option<Symbol>> {
         arena_cache
         eval_always
         desc { "looking up supported target features" }
@@ -2051,6 +2108,18 @@ rustc_queries! {
         desc { "looking up enabled feature gates" }
     }
 
+    query metadata_loader((): ()) -> &'tcx Steal<Box<rustc_session::cstore::MetadataLoaderDyn>> {
+        feedable
+        no_hash
+        desc { "raw operations for metadata file access" }
+    }
+
+    query crate_for_resolver((): ()) -> &'tcx Steal<rustc_ast::ast::Crate> {
+        feedable
+        no_hash
+        desc { "the ast before macro expansion and name resolution" }
+    }
+
     /// Attempt to resolve the given `DefId` to an `Instance`, for the
     /// given generics args (`SubstsRef`), returning one of:
     ///  * `Ok(Some(instance))` on success
@@ -2093,34 +2162,31 @@ rustc_queries! {
     /// span) for an *existing* error. Therefore, it is best-effort, and may never handle
     /// all of the cases that the normal `ty::Ty`-based wfcheck does. This is fine,
     /// because the `ty::Ty`-based wfcheck is always run.
-    query diagnostic_hir_wf_check(key: (ty::Predicate<'tcx>, traits::WellFormedLoc)) -> Option<traits::ObligationCause<'tcx>> {
+    query diagnostic_hir_wf_check(
+        key: (ty::Predicate<'tcx>, traits::WellFormedLoc)
+    ) -> &'tcx Option<traits::ObligationCause<'tcx>> {
         arena_cache
         eval_always
         no_hash
         desc { "performing HIR wf-checking for predicate `{:?}` at item `{:?}`", key.0, key.1 }
     }
 
-
     /// The list of backend features computed from CLI flags (`-Ctarget-cpu`, `-Ctarget-feature`,
     /// `--target` and similar).
-    query global_backend_features(_: ()) -> Vec<String> {
+    query global_backend_features(_: ()) -> &'tcx Vec<String> {
         arena_cache
         eval_always
         desc { "computing the backend features for CLI flags" }
     }
 
-    query generator_diagnostic_data(key: DefId) -> Option<GeneratorDiagnosticData<'tcx>> {
+    query generator_diagnostic_data(key: DefId) -> &'tcx Option<GeneratorDiagnosticData<'tcx>> {
         arena_cache
         desc { |tcx| "looking up generator diagnostic data of `{}`", tcx.def_path_str(key) }
         separate_provide_extern
     }
 
-    query permits_uninit_init(key: TyAndLayout<'tcx>) -> bool {
-        desc { "checking to see if `{}` permits being left uninit", key.ty }
-    }
-
-    query permits_zero_init(key: TyAndLayout<'tcx>) -> bool {
-        desc { "checking to see if `{}` permits being left zeroed", key.ty }
+    query check_validity_requirement(key: (ValidityRequirement, ty::ParamEnvAnd<'tcx, Ty<'tcx>>)) -> Result<bool, ty::layout::LayoutError<'tcx>> {
+        desc { "checking validity requirement for `{}`: {}", key.1.value, key.0 }
     }
 
     query compare_impl_const(
@@ -2133,4 +2199,23 @@ rustc_queries! {
         desc { |tcx| "deducing parameter attributes for {}", tcx.def_path_str(def_id) }
         separate_provide_extern
     }
+
+    query doc_link_resolutions(def_id: DefId) -> &'tcx DocLinkResMap {
+        eval_always
+        desc { "resolutions for documentation links for a module" }
+        separate_provide_extern
+    }
+
+    query doc_link_traits_in_scope(def_id: DefId) -> &'tcx [DefId] {
+        eval_always
+        desc { "traits in scope for documentation links for a module" }
+        separate_provide_extern
+    }
+
+    /// Used in `super_combine_consts` to ICE if the type of the two consts are definitely not going to end up being
+    /// equal to eachother. This might return `Ok` even if the types are unequal, but will never return `Err` if
+    /// the types might be equal.
+    query check_tys_might_be_eq(arg: Canonical<'tcx, (ty::ParamEnv<'tcx>, Ty<'tcx>, Ty<'tcx>)>) -> Result<(), NoSolution> {
+        desc { "check whether two const param are definitely not equal to eachother"}
+    }
 }
diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs
index 5f320708c..3b11fab8c 100644
--- a/compiler/rustc_middle/src/thir.rs
+++ b/compiler/rustc_middle/src/thir.rs
@@ -20,7 +20,7 @@ use rustc_middle::mir::interpret::AllocId;
 use rustc_middle::mir::{self, BinOp, BorrowKind, FakeReadCause, Field, Mutability, UnOp};
 use rustc_middle::ty::adjustment::PointerCast;
 use rustc_middle::ty::subst::SubstsRef;
-use rustc_middle::ty::{self, AdtDef, Ty, UpvarSubsts};
+use rustc_middle::ty::{self, AdtDef, FnSig, Ty, UpvarSubsts};
 use rustc_middle::ty::{CanonicalUserType, CanonicalUserTypeAnnotation};
 use rustc_span::def_id::LocalDefId;
 use rustc_span::{sym, Span, Symbol, DUMMY_SP};
@@ -32,7 +32,12 @@ use std::ops::Index;
 pub mod visit;
 
 macro_rules! thir_with_elements {
-    ($($name:ident: $id:ty => $value:ty => $format:literal,)*) => {
+    (
+        $($field_name:ident: $field_ty:ty,)*
+
+    @elements:
+        $($name:ident: $id:ty => $value:ty => $format:literal,)*
+    ) => {
         $(
             newtype_index! {
                 #[derive(HashStable)]
@@ -47,14 +52,20 @@ macro_rules! thir_with_elements {
         #[derive(Debug, HashStable, Clone)]
         pub struct Thir<'tcx> {
             $(
+                pub $field_name: $field_ty,
+            )*
+            $(
                 pub $name: IndexVec<$id, $value>,
             )*
         }
 
         impl<'tcx> Thir<'tcx> {
-            pub fn new() -> Thir<'tcx> {
+            pub fn new($($field_name: $field_ty,)*) -> Thir<'tcx> {
                 Thir {
                     $(
+                        $field_name,
+                    )*
+                    $(
                         $name: IndexVec::new(),
                     )*
                 }
@@ -75,6 +86,9 @@ macro_rules! thir_with_elements {
 pub const UPVAR_ENV_PARAM: ParamId = ParamId::from_u32(0);
 
 thir_with_elements! {
+    body_type: BodyTy<'tcx>,
+
+@elements:
     arms: ArmId => Arm<'tcx> => "a{}",
     blocks: BlockId => Block => "b{}",
     exprs: ExprId => Expr<'tcx> => "e{}",
@@ -82,6 +96,12 @@ thir_with_elements! {
     params: ParamId => Param<'tcx> => "p{}",
 }
 
+#[derive(Debug, HashStable, Clone)]
+pub enum BodyTy<'tcx> {
+    Const(Ty<'tcx>),
+    Fn(FnSig<'tcx>),
+}
+
 /// Description of a type-checked function parameter.
 #[derive(Clone, Debug, HashStable)]
 pub struct Param<'tcx> {
@@ -217,6 +237,9 @@ pub struct LocalVarId(pub hir::HirId);
 /// A THIR expression.
 #[derive(Clone, Debug, HashStable)]
 pub struct Expr<'tcx> {
+    /// kind of expression
+    pub kind: ExprKind<'tcx>,
+
     /// The type of this expression
     pub ty: Ty<'tcx>,
 
@@ -226,9 +249,6 @@ pub struct Expr<'tcx> {
 
     /// span of the expression in the source
     pub span: Span,
-
-    /// kind of expression
-    pub kind: ExprKind<'tcx>,
 }
 
 #[derive(Clone, Debug, HashStable)]
diff --git a/compiler/rustc_middle/src/traits/mod.rs b/compiler/rustc_middle/src/traits/mod.rs
index d00b26a5a..6231dd9b6 100644
--- a/compiler/rustc_middle/src/traits/mod.rs
+++ b/compiler/rustc_middle/src/traits/mod.rs
@@ -5,6 +5,7 @@
 mod chalk;
 pub mod query;
 pub mod select;
+pub mod solve;
 pub mod specialization_graph;
 mod structural_impls;
 pub mod util;
@@ -18,7 +19,8 @@ use crate::ty::{self, AdtKind, Ty, TyCtxt};
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::{Applicability, Diagnostic};
 use rustc_hir as hir;
-use rustc_hir::def_id::{DefId, LocalDefId};
+use rustc_hir::def_id::DefId;
+use rustc_span::def_id::{LocalDefId, CRATE_DEF_ID};
 use rustc_span::symbol::Symbol;
 use rustc_span::{Span, DUMMY_SP};
 use smallvec::SmallVec;
@@ -36,7 +38,7 @@ pub use self::chalk::{ChalkEnvironmentAndGoal, RustInterner as ChalkRustInterner
 
 /// Depending on the stage of compilation, we want projection to be
 /// more or less conservative.
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, HashStable)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, HashStable, Encodable, Decodable)]
 pub enum Reveal {
     /// At type-checking time, we refuse to project any associated
     /// type that is marked `default`. Non-`default` ("final") types
@@ -89,7 +91,8 @@ pub enum Reveal {
 ///
 /// We do not want to intern this as there are a lot of obligation causes which
 /// only live for a short period of time.
-#[derive(Clone, Debug, PartialEq, Eq, Lift)]
+#[derive(Clone, Debug, PartialEq, Eq, Lift, HashStable, TyEncodable, TyDecodable)]
+#[derive(TypeVisitable, TypeFoldable)]
 pub struct ObligationCause<'tcx> {
     pub span: Span,
 
@@ -99,7 +102,7 @@ pub struct ObligationCause<'tcx> {
     /// (in particular, closures can add new assumptions). See the
     /// field `region_obligations` of the `FulfillmentContext` for more
     /// information.
-    pub body_id: hir::HirId,
+    pub body_id: LocalDefId,
 
     code: InternedObligationCauseCode<'tcx>,
 }
@@ -120,13 +123,13 @@ impl<'tcx> ObligationCause<'tcx> {
     #[inline]
     pub fn new(
         span: Span,
-        body_id: hir::HirId,
+        body_id: LocalDefId,
         code: ObligationCauseCode<'tcx>,
     ) -> ObligationCause<'tcx> {
         ObligationCause { span, body_id, code: code.into() }
     }
 
-    pub fn misc(span: Span, body_id: hir::HirId) -> ObligationCause<'tcx> {
+    pub fn misc(span: Span, body_id: LocalDefId) -> ObligationCause<'tcx> {
         ObligationCause::new(span, body_id, MiscObligation)
     }
 
@@ -137,7 +140,7 @@ impl<'tcx> ObligationCause<'tcx> {
 
     #[inline(always)]
     pub fn dummy_with_span(span: Span) -> ObligationCause<'tcx> {
-        ObligationCause { span, body_id: hir::CRATE_HIR_ID, code: Default::default() }
+        ObligationCause { span, body_id: CRATE_DEF_ID, code: Default::default() }
     }
 
     pub fn span(&self) -> Span {
@@ -196,14 +199,16 @@ impl<'tcx> ObligationCause<'tcx> {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Hash, Lift)]
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Lift, HashStable, TyEncodable, TyDecodable)]
+#[derive(TypeVisitable, TypeFoldable)]
 pub struct UnifyReceiverContext<'tcx> {
     pub assoc_item: ty::AssocItem,
     pub param_env: ty::ParamEnv<'tcx>,
     pub substs: SubstsRef<'tcx>,
 }
 
-#[derive(Clone, PartialEq, Eq, Hash, Lift, Default)]
+#[derive(Clone, PartialEq, Eq, Hash, Lift, Default, HashStable)]
+#[derive(TypeVisitable, TypeFoldable, TyEncodable, TyDecodable)]
 pub struct InternedObligationCauseCode<'tcx> {
     /// `None` for `ObligationCauseCode::MiscObligation` (a common case, occurs ~60% of
     /// the time). `Some` otherwise.
@@ -238,7 +243,8 @@ impl<'tcx> std::ops::Deref for InternedObligationCauseCode<'tcx> {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Hash, Lift)]
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Lift, HashStable, TyEncodable, TyDecodable)]
+#[derive(TypeVisitable, TypeFoldable)]
 pub enum ObligationCauseCode<'tcx> {
     /// Not well classified or should be obvious from the span.
     MiscObligation,
@@ -446,7 +452,8 @@ pub enum ObligationCauseCode<'tcx> {
 /// This information is used to obtain an `hir::Ty`, which
 /// we can walk in order to obtain precise spans for any
 /// 'nested' types (e.g. `Foo` in `Option<Foo>`).
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, HashStable)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, HashStable, Encodable, Decodable)]
+#[derive(TypeVisitable, TypeFoldable)]
 pub enum WellFormedLoc {
     /// Use the type of the provided definition.
     Ty(LocalDefId),
@@ -463,10 +470,17 @@ pub enum WellFormedLoc {
     },
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Hash, Lift)]
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Lift, HashStable, TyEncodable, TyDecodable)]
+#[derive(TypeVisitable, TypeFoldable)]
 pub struct ImplDerivedObligationCause<'tcx> {
     pub derived: DerivedObligationCause<'tcx>,
-    pub impl_def_id: DefId,
+    /// The `DefId` of the `impl` that gave rise to the `derived` obligation.
+    /// If the `derived` obligation arose from a trait alias, which conceptually has a synthetic impl,
+    /// then this will be the `DefId` of that trait alias. Care should therefore be taken to handle
+    /// that exceptional case where appropriate.
+    pub impl_or_alias_def_id: DefId,
+    /// The index of the derived predicate in the parent impl's predicates.
+    pub impl_def_predicate_index: Option<usize>,
     pub span: Span,
 }
 
@@ -517,7 +531,8 @@ impl<'tcx> ty::Lift<'tcx> for StatementAsExpression {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Hash, Lift)]
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Lift, HashStable, TyEncodable, TyDecodable)]
+#[derive(TypeVisitable, TypeFoldable)]
 pub struct MatchExpressionArmCause<'tcx> {
     pub arm_block_id: Option<hir::HirId>,
     pub arm_ty: Ty<'tcx>,
@@ -533,7 +548,7 @@ pub struct MatchExpressionArmCause<'tcx> {
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
-#[derive(Lift, TypeFoldable, TypeVisitable)]
+#[derive(Lift, TypeFoldable, TypeVisitable, HashStable, TyEncodable, TyDecodable)]
 pub struct IfExpressionCause<'tcx> {
     pub then_id: hir::HirId,
     pub else_id: hir::HirId,
@@ -543,7 +558,8 @@ pub struct IfExpressionCause<'tcx> {
     pub opt_suggest_box_span: Option<Span>,
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Hash, Lift)]
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Lift, HashStable, TyEncodable, TyDecodable)]
+#[derive(TypeVisitable, TypeFoldable)]
 pub struct DerivedObligationCause<'tcx> {
     /// The trait predicate of the parent obligation that led to the
     /// current obligation. Note that only trait obligations lead to
diff --git a/compiler/rustc_middle/src/traits/query.rs b/compiler/rustc_middle/src/traits/query.rs
index 615154a55..c4f871875 100644
--- a/compiler/rustc_middle/src/traits/query.rs
+++ b/compiler/rustc_middle/src/traits/query.rs
@@ -14,7 +14,7 @@ use rustc_span::source_map::Span;
 
 pub mod type_op {
     use crate::ty::fold::TypeFoldable;
-    use crate::ty::{Predicate, Ty, UserType};
+    use crate::ty::{Predicate, Ty, TyCtxt, UserType};
     use std::fmt;
 
     #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, HashStable, Lift)]
@@ -64,7 +64,7 @@ pub mod type_op {
 
     impl<'tcx, T> Normalize<T>
     where
-        T: fmt::Debug + TypeFoldable<'tcx>,
+        T: fmt::Debug + TypeFoldable<TyCtxt<'tcx>>,
     {
         pub fn new(value: T) -> Self {
             Self { value }
diff --git a/compiler/rustc_middle/src/traits/solve.rs b/compiler/rustc_middle/src/traits/solve.rs
new file mode 100644
index 000000000..bd43867a3
--- /dev/null
+++ b/compiler/rustc_middle/src/traits/solve.rs
@@ -0,0 +1,66 @@
+use std::ops::ControlFlow;
+
+use rustc_data_structures::intern::Interned;
+
+use crate::infer::canonical::QueryRegionConstraints;
+use crate::ty::{
+    FallibleTypeFolder, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeVisitable, TypeVisitor,
+};
+
+#[derive(Debug, PartialEq, Eq, Copy, Clone, Hash)]
+pub struct ExternalConstraints<'tcx>(pub(crate) Interned<'tcx, ExternalConstraintsData<'tcx>>);
+
+impl<'tcx> std::ops::Deref for ExternalConstraints<'tcx> {
+    type Target = ExternalConstraintsData<'tcx>;
+
+    fn deref(&self) -> &Self::Target {
+        &*self.0
+    }
+}
+
+/// Additional constraints returned on success.
+#[derive(Debug, PartialEq, Eq, Clone, Hash, Default, TypeFoldable, TypeVisitable)]
+pub struct ExternalConstraintsData<'tcx> {
+    // FIXME: implement this.
+    pub region_constraints: QueryRegionConstraints<'tcx>,
+    pub opaque_types: Vec<(Ty<'tcx>, Ty<'tcx>)>,
+}
+
+// FIXME: Having to clone `region_constraints` for folding feels bad and
+// probably isn't great wrt performance.
+//
+// Not sure how to fix this, maybe we should also intern `opaque_types` and
+// `region_constraints` here or something.
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for ExternalConstraints<'tcx> {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self, F::Error> {
+        Ok(FallibleTypeFolder::interner(folder).mk_external_constraints(ExternalConstraintsData {
+            region_constraints: self.region_constraints.clone().try_fold_with(folder)?,
+            opaque_types: self
+                .opaque_types
+                .iter()
+                .map(|opaque| opaque.try_fold_with(folder))
+                .collect::<Result<_, F::Error>>()?,
+        }))
+    }
+
+    fn fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
+        TypeFolder::interner(folder).mk_external_constraints(ExternalConstraintsData {
+            region_constraints: self.region_constraints.clone().fold_with(folder),
+            opaque_types: self.opaque_types.iter().map(|opaque| opaque.fold_with(folder)).collect(),
+        })
+    }
+}
+
+impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for ExternalConstraints<'tcx> {
+    fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(
+        &self,
+        visitor: &mut V,
+    ) -> std::ops::ControlFlow<V::BreakTy> {
+        self.region_constraints.visit_with(visitor)?;
+        self.opaque_types.visit_with(visitor)?;
+        ControlFlow::Continue(())
+    }
+}
diff --git a/compiler/rustc_middle/src/traits/specialization_graph.rs b/compiler/rustc_middle/src/traits/specialization_graph.rs
index aad5b2fbe..c016f7227 100644
--- a/compiler/rustc_middle/src/traits/specialization_graph.rs
+++ b/compiler/rustc_middle/src/traits/specialization_graph.rs
@@ -1,6 +1,6 @@
 use crate::error::StrictCoherenceNeedsNegativeCoherence;
 use crate::ty::fast_reject::SimplifiedType;
-use crate::ty::visit::TypeVisitable;
+use crate::ty::visit::TypeVisitableExt;
 use crate::ty::{self, TyCtxt};
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_errors::ErrorGuaranteed;
@@ -133,11 +133,7 @@ impl Node {
     ///
     /// If this returns `None`, the item can potentially still be found in
     /// parents of this node.
-    pub fn item<'tcx>(
-        &self,
-        tcx: TyCtxt<'tcx>,
-        trait_item_def_id: DefId,
-    ) -> Option<&'tcx ty::AssocItem> {
+    pub fn item<'tcx>(&self, tcx: TyCtxt<'tcx>, trait_item_def_id: DefId) -> Option<ty::AssocItem> {
         match *self {
             Node::Trait(_) => Some(tcx.associated_item(trait_item_def_id)),
             Node::Impl(impl_def_id) => {
@@ -239,7 +235,7 @@ impl<'tcx> Ancestors<'tcx> {
                     }
                 }
 
-                Some(LeafDef { item: *item, defining_node: node, finalizing_node })
+                Some(LeafDef { item, defining_node: node, finalizing_node })
             } else {
                 // Item not mentioned. This "finalizes" any defaulted item provided by an ancestor.
                 finalizing_node = Some(node);
@@ -263,7 +259,7 @@ pub fn ancestors(
 
     if let Some(reported) = specialization_graph.has_errored {
         Err(reported)
-    } else if let Err(reported) = tcx.type_of(start_from_impl).error_reported() {
+    } else if let Err(reported) = tcx.type_of(start_from_impl).subst_identity().error_reported() {
         Err(reported)
     } else {
         Ok(Ancestors {
diff --git a/compiler/rustc_middle/src/ty/_match.rs b/compiler/rustc_middle/src/ty/_match.rs
index b9c5a4e0d..df9aa765d 100644
--- a/compiler/rustc_middle/src/ty/_match.rs
+++ b/compiler/rustc_middle/src/ty/_match.rs
@@ -89,9 +89,7 @@ impl<'tcx> TypeRelation<'tcx> for Match<'tcx> {
                 Err(TypeError::Sorts(relate::expected_found(self, a, b)))
             }
 
-            (&ty::Error(guar), _) | (_, &ty::Error(guar)) => {
-                Ok(self.tcx().ty_error_with_guaranteed(guar))
-            }
+            (&ty::Error(guar), _) | (_, &ty::Error(guar)) => Ok(self.tcx().ty_error(guar)),
 
             _ => relate::super_relate_tys(self, a, b),
         }
diff --git a/compiler/rustc_middle/src/ty/abstract_const.rs b/compiler/rustc_middle/src/ty/abstract_const.rs
index 5de758ad9..f889ce827 100644
--- a/compiler/rustc_middle/src/ty/abstract_const.rs
+++ b/compiler/rustc_middle/src/ty/abstract_const.rs
@@ -1,7 +1,7 @@
 //! A subset of a mir body used for const evaluatability checking.
 use crate::ty::{
     self, Const, EarlyBinder, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeSuperFoldable,
-    TypeVisitable,
+    TypeVisitableExt,
 };
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def_id::DefId;
@@ -48,13 +48,13 @@ impl<'tcx> TyCtxt<'tcx> {
         Ok(ac?.map(|ac| EarlyBinder(ac)))
     }
 
-    pub fn expand_abstract_consts<T: TypeFoldable<'tcx>>(self, ac: T) -> T {
+    pub fn expand_abstract_consts<T: TypeFoldable<TyCtxt<'tcx>>>(self, ac: T) -> T {
         struct Expander<'tcx> {
             tcx: TyCtxt<'tcx>,
         }
 
-        impl<'tcx> TypeFolder<'tcx> for Expander<'tcx> {
-            fn tcx(&self) -> TyCtxt<'tcx> {
+        impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Expander<'tcx> {
+            fn interner(&self) -> TyCtxt<'tcx> {
                 self.tcx
             }
             fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
diff --git a/compiler/rustc_middle/src/ty/adt.rs b/compiler/rustc_middle/src/ty/adt.rs
index d3d667f68..ec21030b3 100644
--- a/compiler/rustc_middle/src/ty/adt.rs
+++ b/compiler/rustc_middle/src/ty/adt.rs
@@ -54,7 +54,7 @@ bitflags! {
 
 /// The definition of a user-defined type, e.g., a `struct`, `enum`, or `union`.
 ///
-/// These are all interned (by `alloc_adt_def`) into the global arena.
+/// These are all interned (by `mk_adt_def`) into the global arena.
 ///
 /// The initialism *ADT* stands for an [*algebraic data type (ADT)*][adt].
 /// This is slightly wrong because `union`s are not ADTs.
@@ -188,7 +188,7 @@ impl<'tcx> AdtDef<'tcx> {
     }
 }
 
-#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, TyEncodable, TyDecodable)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, HashStable, TyEncodable, TyDecodable)]
 pub enum AdtKind {
     Struct,
     Union,
@@ -406,6 +406,7 @@ impl<'tcx> AdtDef<'tcx> {
     }
 
     /// Return the index of `VariantDef` given a variant id.
+    #[inline]
     pub fn variant_index_with_id(self, vid: DefId) -> VariantIdx {
         self.variants()
             .iter_enumerated()
diff --git a/compiler/rustc_middle/src/ty/assoc.rs b/compiler/rustc_middle/src/ty/assoc.rs
index bb7fba3ee..f1a9e50a4 100644
--- a/compiler/rustc_middle/src/ty/assoc.rs
+++ b/compiler/rustc_middle/src/ty/assoc.rs
@@ -79,11 +79,11 @@ impl AssocItem {
                 // late-bound regions, and we don't want method signatures to show up
                 // `as for<'r> fn(&'r MyType)`. Pretty-printing handles late-bound
                 // regions just fine, showing `fn(&MyType)`.
-                tcx.fn_sig(self.def_id).skip_binder().to_string()
+                tcx.fn_sig(self.def_id).subst_identity().skip_binder().to_string()
             }
             ty::AssocKind::Type => format!("type {};", self.name),
             ty::AssocKind::Const => {
-                format!("const {}: {:?};", self.name, tcx.type_of(self.def_id))
+                format!("const {}: {:?};", self.name, tcx.type_of(self.def_id).subst_identity())
             }
         }
     }
@@ -129,13 +129,13 @@ impl std::fmt::Display for AssocKind {
 /// it is relatively expensive. Instead, items are indexed by `Symbol` and hygienic comparison is
 /// done only on items with the same name.
 #[derive(Debug, Clone, PartialEq, HashStable)]
-pub struct AssocItems<'tcx> {
-    pub(super) items: SortedIndexMultiMap<u32, Symbol, &'tcx ty::AssocItem>,
+pub struct AssocItems {
+    items: SortedIndexMultiMap<u32, Symbol, ty::AssocItem>,
 }
 
-impl<'tcx> AssocItems<'tcx> {
+impl AssocItems {
     /// Constructs an `AssociatedItems` map from a series of `ty::AssocItem`s in definition order.
-    pub fn new(items_in_def_order: impl IntoIterator<Item = &'tcx ty::AssocItem>) -> Self {
+    pub fn new(items_in_def_order: impl IntoIterator<Item = ty::AssocItem>) -> Self {
         let items = items_in_def_order.into_iter().map(|item| (item.name, item)).collect();
         AssocItems { items }
     }
@@ -145,7 +145,7 @@ impl<'tcx> AssocItems<'tcx> {
     /// New code should avoid relying on definition order. If you need a particular associated item
     /// for a known trait, make that trait a lang item instead of indexing this array.
     pub fn in_definition_order(&self) -> impl '_ + Iterator<Item = &ty::AssocItem> {
-        self.items.iter().map(|(_, v)| *v)
+        self.items.iter().map(|(_, v)| v)
     }
 
     pub fn len(&self) -> usize {
@@ -157,7 +157,7 @@ impl<'tcx> AssocItems<'tcx> {
         &self,
         name: Symbol,
     ) -> impl '_ + Iterator<Item = &ty::AssocItem> {
-        self.items.get_by_key(name).copied()
+        self.items.get_by_key(name)
     }
 
     /// Returns the associated item with the given name and `AssocKind`, if one exists.
diff --git a/compiler/rustc_middle/src/ty/closure.rs b/compiler/rustc_middle/src/ty/closure.rs
index 6ade8935f..dc2bd54b7 100644
--- a/compiler/rustc_middle/src/ty/closure.rs
+++ b/compiler/rustc_middle/src/ty/closure.rs
@@ -5,10 +5,10 @@ use crate::{mir, ty};
 
 use std::fmt::Write;
 
-use hir::LangItem;
 use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
-use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
+use rustc_hir::{self as hir, LangItem};
+use rustc_span::symbol::Ident;
 use rustc_span::{Span, Symbol};
 
 use super::{Ty, TyCtxt};
@@ -129,6 +129,9 @@ impl<'tcx> ClosureKind {
 #[derive(PartialEq, Clone, Debug, TyEncodable, TyDecodable, HashStable)]
 #[derive(TypeFoldable, TypeVisitable)]
 pub struct CapturedPlace<'tcx> {
+    /// Name and span where the binding happens.
+    pub var_ident: Ident,
+
     /// The `Place` that is captured.
     pub place: HirPlace<'tcx>,
 
@@ -148,12 +151,8 @@ impl<'tcx> CapturedPlace<'tcx> {
     }
 
     /// Returns a symbol of the captured upvar, which looks like `name__field1__field2`.
-    fn to_symbol(&self, tcx: TyCtxt<'tcx>) -> Symbol {
-        let hir_id = match self.place.base {
-            HirPlaceBase::Upvar(upvar_id) => upvar_id.var_path.hir_id,
-            base => bug!("Expected an upvar, found {:?}", base),
-        };
-        let mut symbol = tcx.hir().name(hir_id).as_str().to_string();
+    pub fn to_symbol(&self) -> Symbol {
+        let mut symbol = self.var_ident.to_string();
 
         let mut ty = self.place.base_ty;
         for proj in self.place.projections.iter() {
@@ -169,11 +168,7 @@ impl<'tcx> CapturedPlace<'tcx> {
                         .unwrap();
                     }
                     ty => {
-                        span_bug!(
-                            self.get_capture_kind_span(tcx),
-                            "Unexpected type {:?} for `Field` projection",
-                            ty
-                        )
+                        bug!("Unexpected type {:?} for `Field` projection", ty)
                     }
                 },
 
@@ -238,10 +233,39 @@ impl<'tcx> CapturedPlace<'tcx> {
     }
 }
 
-fn symbols_for_closure_captures(tcx: TyCtxt<'_>, def_id: (LocalDefId, LocalDefId)) -> Vec<Symbol> {
-    let typeck_results = tcx.typeck(def_id.0);
-    let captures = typeck_results.closure_min_captures_flattened(def_id.1);
-    captures.into_iter().map(|captured_place| captured_place.to_symbol(tcx)).collect()
+#[derive(Copy, Clone, Debug, HashStable)]
+pub struct ClosureTypeInfo<'tcx> {
+    user_provided_sig: ty::CanonicalPolyFnSig<'tcx>,
+    captures: &'tcx [&'tcx ty::CapturedPlace<'tcx>],
+    kind_origin: Option<&'tcx (Span, HirPlace<'tcx>)>,
+}
+
+fn closure_typeinfo<'tcx>(tcx: TyCtxt<'tcx>, def: LocalDefId) -> ClosureTypeInfo<'tcx> {
+    debug_assert!(tcx.is_closure(def.to_def_id()));
+    let typeck_results = tcx.typeck(def);
+    let user_provided_sig = typeck_results.user_provided_sigs[&def];
+    let captures = typeck_results.closure_min_captures_flattened(def);
+    let captures = tcx.arena.alloc_from_iter(captures);
+    let hir_id = tcx.hir().local_def_id_to_hir_id(def);
+    let kind_origin = typeck_results.closure_kind_origins().get(hir_id);
+    ClosureTypeInfo { user_provided_sig, captures, kind_origin }
+}
+
+impl<'tcx> TyCtxt<'tcx> {
+    pub fn closure_kind_origin(self, def_id: LocalDefId) -> Option<&'tcx (Span, HirPlace<'tcx>)> {
+        self.closure_typeinfo(def_id).kind_origin
+    }
+
+    pub fn closure_user_provided_sig(self, def_id: LocalDefId) -> ty::CanonicalPolyFnSig<'tcx> {
+        self.closure_typeinfo(def_id).user_provided_sig
+    }
+
+    pub fn closure_captures(self, def_id: LocalDefId) -> &'tcx [&'tcx ty::CapturedPlace<'tcx>] {
+        if !self.is_closure(def_id.to_def_id()) {
+            return &[];
+        };
+        self.closure_typeinfo(def_id).captures
+    }
 }
 
 /// Return true if the `proj_possible_ancestor` represents an ancestor path
@@ -434,5 +458,5 @@ impl BorrowKind {
 }
 
 pub fn provide(providers: &mut ty::query::Providers) {
-    *providers = ty::query::Providers { symbols_for_closure_captures, ..*providers }
+    *providers = ty::query::Providers { closure_typeinfo, ..*providers }
 }
diff --git a/compiler/rustc_middle/src/ty/codec.rs b/compiler/rustc_middle/src/ty/codec.rs
index 8cc8286c1..3ce80e06a 100644
--- a/compiler/rustc_middle/src/ty/codec.rs
+++ b/compiler/rustc_middle/src/ty/codec.rs
@@ -157,6 +157,14 @@ impl<'tcx, E: TyEncoder<I = TyCtxt<'tcx>>> Encodable<E> for AllocId {
     }
 }
 
+impl<'tcx, E: TyEncoder<I = TyCtxt<'tcx>>> Encodable<E> for ty::ParamEnv<'tcx> {
+    fn encode(&self, e: &mut E) {
+        self.caller_bounds().encode(e);
+        self.reveal().encode(e);
+        self.constness().encode(e);
+    }
+}
+
 #[inline]
 fn decode_arena_allocable<
     'tcx,
@@ -199,7 +207,7 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for Ty<'tcx> {
             })
         } else {
             let tcx = decoder.interner();
-            tcx.mk_ty(rustc_type_ir::TyKind::decode(decoder))
+            tcx.mk_ty_from_kind(rustc_type_ir::TyKind::decode(decoder))
         }
     }
 }
@@ -236,7 +244,7 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for SubstsRef<'tcx> {
     fn decode(decoder: &mut D) -> Self {
         let len = decoder.read_usize();
         let tcx = decoder.interner();
-        tcx.mk_substs(
+        tcx.mk_substs_from_iter(
             (0..len).map::<ty::subst::GenericArg<'tcx>, _>(|_| Decodable::decode(decoder)),
         )
     }
@@ -246,7 +254,7 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for mir::Place<'tcx> {
     fn decode(decoder: &mut D) -> Self {
         let local: mir::Local = Decodable::decode(decoder);
         let len = decoder.read_usize();
-        let projection = decoder.interner().mk_place_elems(
+        let projection = decoder.interner().mk_place_elems_from_iter(
             (0..len).map::<mir::PlaceElem<'tcx>, _>(|_| Decodable::decode(decoder)),
         );
         mir::Place { local, projection }
@@ -255,16 +263,16 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for mir::Place<'tcx> {
 
 impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for ty::Region<'tcx> {
     fn decode(decoder: &mut D) -> Self {
-        decoder.interner().mk_region(Decodable::decode(decoder))
+        decoder.interner().mk_region_from_kind(Decodable::decode(decoder))
     }
 }
 
 impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for CanonicalVarInfos<'tcx> {
     fn decode(decoder: &mut D) -> Self {
         let len = decoder.read_usize();
-        let interned: Vec<CanonicalVarInfo<'tcx>> =
-            (0..len).map(|_| Decodable::decode(decoder)).collect();
-        decoder.interner().intern_canonical_var_infos(interned.as_slice())
+        decoder.interner().mk_canonical_var_infos_from_iter(
+            (0..len).map::<CanonicalVarInfo<'tcx>, _>(|_| Decodable::decode(decoder)),
+        )
     }
 }
 
@@ -280,8 +288,17 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for ty::SymbolName<'tcx>
     }
 }
 
+impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for ty::ParamEnv<'tcx> {
+    fn decode(d: &mut D) -> Self {
+        let caller_bounds = Decodable::decode(d);
+        let reveal = Decodable::decode(d);
+        let constness = Decodable::decode(d);
+        ty::ParamEnv::new(caller_bounds, reveal, constness)
+    }
+}
+
 macro_rules! impl_decodable_via_ref {
-    ($($t:ty),+) => {
+    ($($t:ty,)+) => {
         $(impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for $t {
             fn decode(decoder: &mut D) -> Self {
                 RefDecodable::decode(decoder)
@@ -293,7 +310,9 @@ macro_rules! impl_decodable_via_ref {
 impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for ty::List<Ty<'tcx>> {
     fn decode(decoder: &mut D) -> &'tcx Self {
         let len = decoder.read_usize();
-        decoder.interner().mk_type_list((0..len).map::<Ty<'tcx>, _>(|_| Decodable::decode(decoder)))
+        decoder
+            .interner()
+            .mk_type_list_from_iter((0..len).map::<Ty<'tcx>, _>(|_| Decodable::decode(decoder)))
     }
 }
 
@@ -302,7 +321,7 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D>
 {
     fn decode(decoder: &mut D) -> &'tcx Self {
         let len = decoder.read_usize();
-        decoder.interner().mk_poly_existential_predicates(
+        decoder.interner().mk_poly_existential_predicates_from_iter(
             (0..len).map::<ty::Binder<'tcx, _>, _>(|_| Decodable::decode(decoder)),
         )
     }
@@ -325,13 +344,13 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for [ty::ValTre
 
 impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for ConstAllocation<'tcx> {
     fn decode(decoder: &mut D) -> Self {
-        decoder.interner().intern_const_alloc(Decodable::decode(decoder))
+        decoder.interner().mk_const_alloc(Decodable::decode(decoder))
     }
 }
 
 impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for AdtDef<'tcx> {
     fn decode(decoder: &mut D) -> Self {
-        decoder.interner().intern_adt_def(Decodable::decode(decoder))
+        decoder.interner().mk_adt_def_from_data(Decodable::decode(decoder))
     }
 }
 
@@ -358,7 +377,7 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D>
 {
     fn decode(decoder: &mut D) -> &'tcx Self {
         let len = decoder.read_usize();
-        decoder.interner().mk_bound_variable_kinds(
+        decoder.interner().mk_bound_variable_kinds_from_iter(
             (0..len).map::<ty::BoundVariableKind, _>(|_| Decodable::decode(decoder)),
         )
     }
@@ -367,9 +386,18 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D>
 impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for ty::List<ty::Const<'tcx>> {
     fn decode(decoder: &mut D) -> &'tcx Self {
         let len = decoder.read_usize();
-        decoder
-            .interner()
-            .mk_const_list((0..len).map::<ty::Const<'tcx>, _>(|_| Decodable::decode(decoder)))
+        decoder.interner().mk_const_list_from_iter(
+            (0..len).map::<ty::Const<'tcx>, _>(|_| Decodable::decode(decoder)),
+        )
+    }
+}
+
+impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for ty::List<ty::Predicate<'tcx>> {
+    fn decode(decoder: &mut D) -> &'tcx Self {
+        let len = decoder.read_usize();
+        decoder.interner().mk_predicates_from_iter(
+            (0..len).map::<ty::Predicate<'tcx>, _>(|_| Decodable::decode(decoder)),
+        )
     }
 }
 
@@ -382,7 +410,8 @@ impl_decodable_via_ref! {
     &'tcx mir::UnsafetyCheckResult,
     &'tcx mir::BorrowCheckResult<'tcx>,
     &'tcx mir::coverage::CodeRegion,
-    &'tcx ty::List<ty::BoundVariableKind>
+    &'tcx ty::List<ty::BoundVariableKind>,
+    &'tcx ty::List<ty::Predicate<'tcx>>,
 }
 
 #[macro_export]
@@ -519,6 +548,8 @@ macro_rules! impl_binder_encode_decode {
 impl_binder_encode_decode! {
     &'tcx ty::List<Ty<'tcx>>,
     ty::FnSig<'tcx>,
+    ty::Predicate<'tcx>,
+    ty::TraitPredicate<'tcx>,
     ty::ExistentialPredicate<'tcx>,
     ty::TraitRef<'tcx>,
     Vec<ty::GeneratorInteriorTypeCause<'tcx>>,
diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs
index 65cbac3e8..527ec9f6e 100644
--- a/compiler/rustc_middle/src/ty/consts.rs
+++ b/compiler/rustc_middle/src/ty/consts.rs
@@ -1,7 +1,9 @@
+use crate::middle::resolve_bound_vars as rbv;
 use crate::mir::interpret::LitToConstInput;
 use crate::ty::{self, DefIdTree, InternalSubsts, ParamEnv, ParamEnvAnd, Ty, TyCtxt};
 use rustc_data_structures::intern::Interned;
 use rustc_hir as hir;
+use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_macros::HashStable;
 use std::fmt;
@@ -14,7 +16,7 @@ pub use int::*;
 pub use kind::*;
 pub use valtree::*;
 
-/// Use this rather than `ConstData, whenever possible.
+/// Use this rather than `ConstData`, whenever possible.
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, HashStable)]
 #[rustc_pass_by_value]
 pub struct Const<'tcx>(pub(super) Interned<'tcx, ConstData<'tcx>>);
@@ -71,7 +73,10 @@ impl<'tcx> Const<'tcx> {
         let expr = &tcx.hir().body(body_id).value;
         debug!(?expr);
 
-        let ty = tcx.type_of(def.def_id_for_type_of());
+        let ty = tcx
+            .type_of(def.def_id_for_type_of())
+            .no_bound_vars()
+            .expect("const parameter types cannot be generic");
 
         match Self::try_eval_lit_or_param(tcx, ty, expr) {
             Some(v) => v,
@@ -125,16 +130,30 @@ impl<'tcx> Const<'tcx> {
             }
         }
 
-        use hir::{def::DefKind::ConstParam, def::Res, ExprKind, Path, QPath};
         match expr.kind {
-            ExprKind::Path(QPath::Resolved(_, &Path { res: Res::Def(ConstParam, def_id), .. })) => {
-                // Find the name and index of the const parameter by indexing the generics of
-                // the parent item and construct a `ParamConst`.
-                let item_def_id = tcx.parent(def_id);
-                let generics = tcx.generics_of(item_def_id);
-                let index = generics.param_def_id_to_index[&def_id];
-                let name = tcx.item_name(def_id);
-                Some(tcx.mk_const(ty::ParamConst::new(index, name), ty))
+            hir::ExprKind::Path(hir::QPath::Resolved(
+                _,
+                &hir::Path { res: Res::Def(DefKind::ConstParam, def_id), .. },
+            )) => {
+                match tcx.named_bound_var(expr.hir_id) {
+                    Some(rbv::ResolvedArg::EarlyBound(_)) => {
+                        // Find the name and index of the const parameter by indexing the generics of
+                        // the parent item and construct a `ParamConst`.
+                        let item_def_id = tcx.parent(def_id);
+                        let generics = tcx.generics_of(item_def_id);
+                        let index = generics.param_def_id_to_index[&def_id];
+                        let name = tcx.item_name(def_id);
+                        Some(tcx.mk_const(ty::ParamConst::new(index, name), ty))
+                    }
+                    Some(rbv::ResolvedArg::LateBound(debruijn, index, _)) => Some(tcx.mk_const(
+                        ty::ConstKind::Bound(debruijn, ty::BoundVar::from_u32(index)),
+                        ty,
+                    )),
+                    Some(rbv::ResolvedArg::Error(guar)) => {
+                        Some(tcx.const_error_with_guaranteed(ty, guar))
+                    }
+                    arg => bug!("unexpected bound var resolution for {:?}: {arg:?}", expr.hir_id),
+                }
             }
             _ => None,
         }
@@ -175,7 +194,7 @@ impl<'tcx> Const<'tcx> {
 
     #[inline]
     /// Creates an interned usize constant.
-    pub fn from_usize(tcx: TyCtxt<'tcx>, n: u64) -> Self {
+    pub fn from_target_usize(tcx: TyCtxt<'tcx>, n: u64) -> Self {
         Self::from_bits(tcx, n as u128, ParamEnv::empty().and(tcx.types.usize))
     }
 
@@ -201,8 +220,12 @@ impl<'tcx> Const<'tcx> {
     }
 
     #[inline]
-    pub fn try_eval_usize(self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>) -> Option<u64> {
-        self.kind().eval(tcx, param_env).try_to_machine_usize(tcx)
+    pub fn try_eval_target_usize(
+        self,
+        tcx: TyCtxt<'tcx>,
+        param_env: ParamEnv<'tcx>,
+    ) -> Option<u64> {
+        self.kind().eval(tcx, param_env).try_to_target_usize(tcx)
     }
 
     #[inline]
@@ -229,8 +252,8 @@ impl<'tcx> Const<'tcx> {
 
     #[inline]
     /// Panics if the value cannot be evaluated or doesn't contain a valid `usize`.
-    pub fn eval_usize(self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>) -> u64 {
-        self.try_eval_usize(tcx, param_env)
+    pub fn eval_target_usize(self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>) -> u64 {
+        self.try_eval_target_usize(tcx, param_env)
             .unwrap_or_else(|| bug!("expected usize, got {:#?}", self))
     }
 
diff --git a/compiler/rustc_middle/src/ty/consts/int.rs b/compiler/rustc_middle/src/ty/consts/int.rs
index 48958e0d9..eecd78ab6 100644
--- a/compiler/rustc_middle/src/ty/consts/int.rs
+++ b/compiler/rustc_middle/src/ty/consts/int.rs
@@ -232,7 +232,7 @@ impl ScalarInt {
     }
 
     #[inline]
-    pub fn try_to_machine_usize(&self, tcx: TyCtxt<'_>) -> Result<u64, Size> {
+    pub fn try_to_target_usize(&self, tcx: TyCtxt<'_>) -> Result<u64, Size> {
         Ok(self.to_bits(tcx.data_layout.pointer_size)? as u64)
     }
 
diff --git a/compiler/rustc_middle/src/ty/consts/kind.rs b/compiler/rustc_middle/src/ty/consts/kind.rs
index d9721863a..560caa041 100644
--- a/compiler/rustc_middle/src/ty/consts/kind.rs
+++ b/compiler/rustc_middle/src/ty/consts/kind.rs
@@ -4,7 +4,7 @@ use crate::mir::interpret::{AllocId, ConstValue, Scalar};
 use crate::ty::abstract_const::CastKind;
 use crate::ty::subst::{InternalSubsts, SubstsRef};
 use crate::ty::ParamEnv;
-use crate::ty::{self, List, Ty, TyCtxt, TypeVisitable};
+use crate::ty::{self, List, Ty, TyCtxt, TypeVisitableExt};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def_id::DefId;
@@ -125,8 +125,8 @@ impl<'tcx> ConstKind<'tcx> {
     }
 
     #[inline]
-    pub fn try_to_machine_usize(self, tcx: TyCtxt<'tcx>) -> Option<u64> {
-        self.try_to_value()?.try_to_machine_usize(tcx)
+    pub fn try_to_target_usize(self, tcx: TyCtxt<'tcx>) -> Option<u64> {
+        self.try_to_value()?.try_to_target_usize(tcx)
     }
 }
 
@@ -217,23 +217,21 @@ impl<'tcx> ConstKind<'tcx> {
             // Note that we erase regions *before* calling `with_reveal_all_normalized`,
             // so that we don't try to invoke this query with
             // any region variables.
-            let param_env_and = tcx
-                .erase_regions(param_env)
-                .with_reveal_all_normalized(tcx)
-                .and(tcx.erase_regions(unevaluated));
 
             // HACK(eddyb) when the query key would contain inference variables,
             // attempt using identity substs and `ParamEnv` instead, that will succeed
             // when the expression doesn't depend on any parameters.
             // FIXME(eddyb, skinny121) pass `InferCtxt` into here when it's available, so that
             // we can call `infcx.const_eval_resolve` which handles inference variables.
-            let param_env_and = if param_env_and.needs_infer() {
+            let param_env_and = if (param_env, unevaluated).has_non_region_infer() {
                 tcx.param_env(unevaluated.def.did).and(ty::UnevaluatedConst {
                     def: unevaluated.def,
                     substs: InternalSubsts::identity_for_item(tcx, unevaluated.def.did),
                 })
             } else {
-                param_env_and
+                tcx.erase_regions(param_env)
+                    .with_reveal_all_normalized(tcx)
+                    .and(tcx.erase_regions(unevaluated))
             };
 
             // FIXME(eddyb) maybe the `const_eval_*` methods should take
diff --git a/compiler/rustc_middle/src/ty/consts/valtree.rs b/compiler/rustc_middle/src/ty/consts/valtree.rs
index a803fca0d..5ed4af2e9 100644
--- a/compiler/rustc_middle/src/ty/consts/valtree.rs
+++ b/compiler/rustc_middle/src/ty/consts/valtree.rs
@@ -78,8 +78,8 @@ impl<'tcx> ValTree<'tcx> {
         }
     }
 
-    pub fn try_to_machine_usize(self, tcx: TyCtxt<'tcx>) -> Option<u64> {
-        self.try_to_scalar_int().map(|s| s.try_to_machine_usize(tcx).ok()).flatten()
+    pub fn try_to_target_usize(self, tcx: TyCtxt<'tcx>) -> Option<u64> {
+        self.try_to_scalar_int().map(|s| s.try_to_target_usize(tcx).ok()).flatten()
     }
 
     /// Get the values inside the ValTree as a slice of bytes. This only works for
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index ce04d8d21..d9af2fd74 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -2,12 +2,14 @@
 
 #![allow(rustc::usage_of_ty_tykind)]
 
+pub mod tls;
+
 use crate::arena::Arena;
 use crate::dep_graph::{DepGraph, DepKindStruct};
-use crate::infer::canonical::{CanonicalVarInfo, CanonicalVarInfos};
+use crate::infer::canonical::CanonicalVarInfo;
 use crate::lint::struct_lint_level;
 use crate::middle::codegen_fn_attrs::CodegenFnAttrs;
-use crate::middle::resolve_lifetime;
+use crate::middle::resolve_bound_vars;
 use crate::middle::stability;
 use crate::mir::interpret::{self, Allocation, ConstAllocation};
 use crate::mir::{
@@ -15,6 +17,7 @@ use crate::mir::{
 };
 use crate::thir::Thir;
 use crate::traits;
+use crate::traits::solve::{ExternalConstraints, ExternalConstraintsData};
 use crate::ty::query::{self, TyCtxtAt};
 use crate::ty::{
     self, AdtDef, AdtDefData, AdtKind, Binder, Const, ConstData, DefIdTree, FloatTy, FloatVar,
@@ -33,7 +36,7 @@ use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::sharded::{IntoPointer, ShardedHashMap};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::steal::Steal;
-use rustc_data_structures::sync::{self, Lock, Lrc, ReadGuard, WorkerLocal};
+use rustc_data_structures::sync::{self, Lock, Lrc, MappedReadGuard, ReadGuard, WorkerLocal};
 use rustc_errors::{
     DecorateLint, DiagnosticBuilder, DiagnosticMessage, ErrorGuaranteed, MultiSpan,
 };
@@ -64,7 +67,7 @@ use rustc_target::abi::{Layout, LayoutS, TargetDataLayout, VariantIdx};
 use rustc_target::spec::abi;
 use rustc_type_ir::sty::TyKind::*;
 use rustc_type_ir::WithCachedTypeInfo;
-use rustc_type_ir::{DynKind, InternAs, InternIteratorElement, Interner, TypeFlags};
+use rustc_type_ir::{CollectAndApply, DynKind, Interner, TypeFlags};
 
 use std::any::Any;
 use std::borrow::Borrow;
@@ -75,6 +78,8 @@ use std::iter;
 use std::mem;
 use std::ops::{Bound, Deref};
 
+const TINY_CONST_EVAL_LIMIT: Limit = Limit(20);
+
 pub trait OnDiskCache<'tcx>: rustc_data_structures::sync::Sync {
     /// Creates a new `OnDiskCache` instance from the serialized data in `data`.
     fn new(sess: &'tcx Session, data: Mmap, start_pos: usize) -> Self
@@ -95,9 +100,11 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
     type AdtDef = ty::AdtDef<'tcx>;
     type SubstsRef = ty::SubstsRef<'tcx>;
     type DefId = DefId;
+    type Binder<T> = Binder<'tcx, T>;
     type Ty = Ty<'tcx>;
     type Const = ty::Const<'tcx>;
     type Region = Region<'tcx>;
+    type Predicate = Predicate<'tcx>;
     type TypeAndMut = TypeAndMut<'tcx>;
     type Mutability = hir::Mutability;
     type Movability = hir::Movability;
@@ -142,8 +149,9 @@ pub struct CtxtInterners<'tcx> {
     const_: InternedSet<'tcx, ConstData<'tcx>>,
     const_allocation: InternedSet<'tcx, Allocation>,
     bound_variable_kinds: InternedSet<'tcx, List<ty::BoundVariableKind>>,
-    layout: InternedSet<'tcx, LayoutS<VariantIdx>>,
+    layout: InternedSet<'tcx, LayoutS>,
     adt_def: InternedSet<'tcx, AdtDefData>,
+    external_constraints: InternedSet<'tcx, ExternalConstraintsData<'tcx>>,
 }
 
 impl<'tcx> CtxtInterners<'tcx> {
@@ -165,10 +173,11 @@ impl<'tcx> CtxtInterners<'tcx> {
             bound_variable_kinds: Default::default(),
             layout: Default::default(),
             adt_def: Default::default(),
+            external_constraints: Default::default(),
         }
     }
 
-    /// Interns a type.
+    /// Interns a type. (Use `mk_*` functions instead, where possible.)
     #[allow(rustc::usage_of_ty_tykind)]
     #[inline(never)]
     fn intern_ty(&self, kind: TyKind<'tcx>, sess: &Session, untracked: &Untracked) -> Ty<'tcx> {
@@ -208,6 +217,7 @@ impl<'tcx> CtxtInterners<'tcx> {
         }
     }
 
+    /// Interns a predicate. (Use `mk_predicate` instead, where possible.)
     #[inline(never)]
     fn intern_predicate(
         &self,
@@ -234,6 +244,20 @@ impl<'tcx> CtxtInterners<'tcx> {
     }
 }
 
+// For these preinterned values, an alternative would be to have
+// variable-length vectors that grow as needed. But that turned out to be
+// slightly more complex and no faster.
+
+const NUM_PREINTERNED_TY_VARS: u32 = 100;
+const NUM_PREINTERNED_FRESH_TYS: u32 = 20;
+const NUM_PREINTERNED_FRESH_INT_TYS: u32 = 3;
+const NUM_PREINTERNED_FRESH_FLOAT_TYS: u32 = 3;
+
+// This number may seem high, but it is reached in all but the smallest crates.
+const NUM_PREINTERNED_RE_VARS: u32 = 500;
+const NUM_PREINTERNED_RE_LATE_BOUNDS_I: u32 = 2;
+const NUM_PREINTERNED_RE_LATE_BOUNDS_V: u32 = 20;
+
 pub struct CommonTypes<'tcx> {
     pub unit: Ty<'tcx>,
     pub bool: Ty<'tcx>,
@@ -259,7 +283,20 @@ pub struct CommonTypes<'tcx> {
     /// Dummy type used for the `Self` of a `TraitRef` created for converting
     /// a trait object, and which gets removed in `ExistentialTraitRef`.
     /// This type must not appear anywhere in other converted types.
+    /// `Infer(ty::FreshTy(0))` does the job.
     pub trait_object_dummy_self: Ty<'tcx>,
+
+    /// Pre-interned `Infer(ty::TyVar(n))` for small values of `n`.
+    pub ty_vars: Vec<Ty<'tcx>>,
+
+    /// Pre-interned `Infer(ty::FreshTy(n))` for small values of `n`.
+    pub fresh_tys: Vec<Ty<'tcx>>,
+
+    /// Pre-interned `Infer(ty::FreshIntTy(n))` for small values of `n`.
+    pub fresh_int_tys: Vec<Ty<'tcx>>,
+
+    /// Pre-interned `Infer(ty::FreshFloatTy(n))` for small values of `n`.
+    pub fresh_float_tys: Vec<Ty<'tcx>>,
 }
 
 pub struct CommonLifetimes<'tcx> {
@@ -268,6 +305,14 @@ pub struct CommonLifetimes<'tcx> {
 
     /// Erased region, used outside of type inference.
     pub re_erased: Region<'tcx>,
+
+    /// Pre-interned `ReVar(ty::RegionVar(n))` for small values of `n`.
+    pub re_vars: Vec<Region<'tcx>>,
+
+    /// Pre-interned values of the form:
+    /// `ReLateBound(DebruijnIndex(i), BoundRegion { var: v, kind: BrAnon(v, None) })`
+    /// for small values of `i` and `v`.
+    pub re_late_bounds: Vec<Vec<Region<'tcx>>>,
 }
 
 pub struct CommonConsts<'tcx> {
@@ -282,6 +327,15 @@ impl<'tcx> CommonTypes<'tcx> {
     ) -> CommonTypes<'tcx> {
         let mk = |ty| interners.intern_ty(ty, sess, untracked);
 
+        let ty_vars =
+            (0..NUM_PREINTERNED_TY_VARS).map(|n| mk(Infer(ty::TyVar(TyVid::from(n))))).collect();
+        let fresh_tys: Vec<_> =
+            (0..NUM_PREINTERNED_FRESH_TYS).map(|n| mk(Infer(ty::FreshTy(n)))).collect();
+        let fresh_int_tys: Vec<_> =
+            (0..NUM_PREINTERNED_FRESH_INT_TYS).map(|n| mk(Infer(ty::FreshIntTy(n)))).collect();
+        let fresh_float_tys: Vec<_> =
+            (0..NUM_PREINTERNED_FRESH_FLOAT_TYS).map(|n| mk(Infer(ty::FreshFloatTy(n)))).collect();
+
         CommonTypes {
             unit: mk(Tuple(List::empty())),
             bool: mk(Bool),
@@ -304,7 +358,12 @@ impl<'tcx> CommonTypes<'tcx> {
             str_: mk(Str),
             self_param: mk(ty::Param(ty::ParamTy { index: 0, name: kw::SelfUpper })),
 
-            trait_object_dummy_self: mk(Infer(ty::FreshTy(0))),
+            trait_object_dummy_self: fresh_tys[0],
+
+            ty_vars,
+            fresh_tys,
+            fresh_int_tys,
+            fresh_float_tys,
         }
     }
 }
@@ -317,7 +376,31 @@ impl<'tcx> CommonLifetimes<'tcx> {
             ))
         };
 
-        CommonLifetimes { re_static: mk(ty::ReStatic), re_erased: mk(ty::ReErased) }
+        let re_vars =
+            (0..NUM_PREINTERNED_RE_VARS).map(|n| mk(ty::ReVar(ty::RegionVid::from(n)))).collect();
+
+        let re_late_bounds = (0..NUM_PREINTERNED_RE_LATE_BOUNDS_I)
+            .map(|i| {
+                (0..NUM_PREINTERNED_RE_LATE_BOUNDS_V)
+                    .map(|v| {
+                        mk(ty::ReLateBound(
+                            ty::DebruijnIndex::from(i),
+                            ty::BoundRegion {
+                                var: ty::BoundVar::from(v),
+                                kind: ty::BrAnon(v, None),
+                            },
+                        ))
+                    })
+                    .collect()
+            })
+            .collect();
+
+        CommonLifetimes {
+            re_static: mk(ty::ReStatic),
+            re_erased: mk(ty::ReErased),
+            re_vars,
+            re_late_bounds,
+        }
     }
 }
 
@@ -438,7 +521,7 @@ pub struct GlobalCtxt<'tcx> {
     pub on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>,
 
     pub queries: &'tcx dyn query::QueryEngine<'tcx>,
-    pub query_caches: query::QueryCaches<'tcx>,
+    pub query_system: query::QuerySystem<'tcx>,
     pub(crate) query_kinds: &'tcx [DepKindStruct<'tcx>],
 
     // Internal caches for metadata decoding. No need to track deps on this.
@@ -461,6 +544,18 @@ pub struct GlobalCtxt<'tcx> {
     pub(crate) alloc_map: Lock<interpret::AllocMap<'tcx>>,
 }
 
+impl<'tcx> GlobalCtxt<'tcx> {
+    /// Installs `self` in a `TyCtxt` and `ImplicitCtxt` for the duration of
+    /// `f`.
+    pub fn enter<'a: 'tcx, F, R>(&'a self, f: F) -> R
+    where
+        F: FnOnce(TyCtxt<'tcx>) -> R,
+    {
+        let icx = tls::ImplicitCtxt::new(self);
+        tls::enter_context(&icx, || f(icx.tcx))
+    }
+}
+
 impl<'tcx> TyCtxt<'tcx> {
     /// Expects a body and returns its codegen attributes.
     ///
@@ -521,21 +616,21 @@ impl<'tcx> TyCtxt<'tcx> {
         self.arena.alloc(Steal::new(promoted))
     }
 
-    pub fn alloc_adt_def(
+    pub fn mk_adt_def(
         self,
         did: DefId,
         kind: AdtKind,
         variants: IndexVec<VariantIdx, ty::VariantDef>,
         repr: ReprOptions,
     ) -> ty::AdtDef<'tcx> {
-        self.intern_adt_def(ty::AdtDefData::new(self, did, kind, variants, repr))
+        self.mk_adt_def_from_data(ty::AdtDefData::new(self, did, kind, variants, repr))
     }
 
     /// Allocates a read-only byte or string literal for `mir::interpret`.
     pub fn allocate_bytes(self, bytes: &[u8]) -> interpret::AllocId {
         // Create an allocation that just contains these bytes.
         let alloc = interpret::Allocation::from_bytes_byte_aligned_immutable(bytes);
-        let alloc = self.intern_const_alloc(alloc);
+        let alloc = self.mk_const_alloc(alloc);
         self.create_memory_alloc(alloc)
     }
 
@@ -611,7 +706,7 @@ impl<'tcx> TyCtxt<'tcx> {
             untracked,
             on_disk_cache,
             queries,
-            query_caches: query::QueryCaches::default(),
+            query_system: Default::default(),
             query_kinds,
             ty_rcache: Default::default(),
             pred_rcache: Default::default(),
@@ -624,13 +719,13 @@ impl<'tcx> TyCtxt<'tcx> {
 
     /// Constructs a `TyKind::Error` type with current `ErrorGuaranteed`
     #[track_caller]
-    pub fn ty_error_with_guaranteed(self, reported: ErrorGuaranteed) -> Ty<'tcx> {
-        self.mk_ty(Error(reported))
+    pub fn ty_error(self, reported: ErrorGuaranteed) -> Ty<'tcx> {
+        self.mk_ty_from_kind(Error(reported))
     }
 
     /// Constructs a `TyKind::Error` type and registers a `delay_span_bug` to ensure it gets used.
     #[track_caller]
-    pub fn ty_error(self) -> Ty<'tcx> {
+    pub fn ty_error_misc(self) -> Ty<'tcx> {
         self.ty_error_with_message(DUMMY_SP, "TyKind::Error constructed but no error reported")
     }
 
@@ -639,7 +734,31 @@ impl<'tcx> TyCtxt<'tcx> {
     #[track_caller]
     pub fn ty_error_with_message<S: Into<MultiSpan>>(self, span: S, msg: &str) -> Ty<'tcx> {
         let reported = self.sess.delay_span_bug(span, msg);
-        self.mk_ty(Error(reported))
+        self.mk_ty_from_kind(Error(reported))
+    }
+
+    /// Constructs a `RegionKind::ReError` lifetime.
+    #[track_caller]
+    pub fn mk_re_error(self, reported: ErrorGuaranteed) -> Region<'tcx> {
+        self.intern_region(ty::ReError(reported))
+    }
+
+    /// Constructs a `RegionKind::ReError` lifetime and registers a `delay_span_bug` to ensure it
+    /// gets used.
+    #[track_caller]
+    pub fn mk_re_error_misc(self) -> Region<'tcx> {
+        self.mk_re_error_with_message(
+            DUMMY_SP,
+            "RegionKind::ReError constructed but no error reported",
+        )
+    }
+
+    /// Constructs a `RegionKind::ReError` lifetime and registers a `delay_span_bug` with the given
+    /// `msg` to ensure it gets used.
+    #[track_caller]
+    pub fn mk_re_error_with_message<S: Into<MultiSpan>>(self, span: S, msg: &str) -> Region<'tcx> {
+        let reported = self.sess.delay_span_bug(span, msg);
+        self.mk_re_error(reported)
     }
 
     /// Like [TyCtxt::ty_error] but for constants, with current `ErrorGuaranteed`
@@ -675,8 +794,7 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn consider_optimizing<T: Fn() -> String>(self, msg: T) -> bool {
-        let cname = self.crate_name(LOCAL_CRATE);
-        self.sess.consider_optimizing(cname.as_str(), msg)
+        self.sess.consider_optimizing(|| self.crate_name(LOCAL_CRATE), msg)
     }
 
     /// Obtain all lang items of this crate and all dependencies (recursively)
@@ -718,7 +836,7 @@ impl<'tcx> TyCtxt<'tcx> {
         if let Some(id) = id.as_local() {
             self.definitions_untracked().def_key(id)
         } else {
-            self.untracked.cstore.def_key(id)
+            self.cstore_untracked().def_key(id)
         }
     }
 
@@ -732,7 +850,7 @@ impl<'tcx> TyCtxt<'tcx> {
         if let Some(id) = id.as_local() {
             self.definitions_untracked().def_path(id)
         } else {
-            self.untracked.cstore.def_path(id)
+            self.cstore_untracked().def_path(id)
         }
     }
 
@@ -742,7 +860,7 @@ impl<'tcx> TyCtxt<'tcx> {
         if let Some(def_id) = def_id.as_local() {
             self.definitions_untracked().def_path_hash(def_id)
         } else {
-            self.untracked.cstore.def_path_hash(def_id)
+            self.cstore_untracked().def_path_hash(def_id)
         }
     }
 
@@ -751,7 +869,7 @@ impl<'tcx> TyCtxt<'tcx> {
         if crate_num == LOCAL_CRATE {
             self.sess.local_stable_crate_id()
         } else {
-            self.untracked.cstore.stable_crate_id(crate_num)
+            self.cstore_untracked().stable_crate_id(crate_num)
         }
     }
 
@@ -762,7 +880,7 @@ impl<'tcx> TyCtxt<'tcx> {
         if stable_crate_id == self.sess.local_stable_crate_id() {
             LOCAL_CRATE
         } else {
-            self.untracked.cstore.stable_crate_id_to_crate_num(stable_crate_id)
+            self.cstore_untracked().stable_crate_id_to_crate_num(stable_crate_id)
         }
     }
 
@@ -781,7 +899,7 @@ impl<'tcx> TyCtxt<'tcx> {
         } else {
             // If this is a DefPathHash from an upstream crate, let the CrateStore map
             // it to a DefId.
-            let cstore = &*self.untracked.cstore;
+            let cstore = &*self.cstore_untracked();
             let cnum = cstore.stable_crate_id_to_crate_num(stable_crate_id);
             cstore.def_path_hash_to_def_id(cnum, hash)
         }
@@ -795,7 +913,7 @@ impl<'tcx> TyCtxt<'tcx> {
         let (crate_name, stable_crate_id) = if def_id.is_local() {
             (self.crate_name(LOCAL_CRATE), self.sess.local_stable_crate_id())
         } else {
-            let cstore = &*self.untracked.cstore;
+            let cstore = &*self.cstore_untracked();
             (cstore.crate_name(def_id.krate), cstore.stable_crate_id(def_id.krate))
         };
 
@@ -893,10 +1011,15 @@ impl<'tcx> TyCtxt<'tcx> {
 
     /// Note that this is *untracked* and should only be used within the query
     /// system if the result is otherwise tracked through queries
-    pub fn cstore_untracked(self) -> &'tcx CrateStoreDyn {
-        &*self.untracked.cstore
+    #[inline]
+    pub fn cstore_untracked(self) -> MappedReadGuard<'tcx, CrateStoreDyn> {
+        ReadGuard::map(self.untracked.cstore.read(), |c| &**c)
     }
 
+    /// Give out access to the untracked data without any sanity checks.
+    pub fn untracked(self) -> &'tcx Untracked {
+        &self.untracked
+    }
     /// Note that this is *untracked* and should only be used within the query
     /// system if the result is otherwise tracked through queries
     #[inline]
@@ -908,7 +1031,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// system if the result is otherwise tracked through queries
     #[inline]
     pub fn source_span_untracked(self, def_id: LocalDefId) -> Span {
-        self.untracked.source_span.get(def_id).copied().unwrap_or(DUMMY_SP)
+        self.untracked.source_span.get(def_id).unwrap_or(DUMMY_SP)
     }
 
     #[inline(always)]
@@ -997,6 +1120,30 @@ impl<'tcx> TyCtxt<'tcx> {
         v.0
     }
 
+    /// Given a `DefId` for an `fn`, return all the `dyn` and `impl` traits in its return type and associated alias span when type alias is used
+    pub fn return_type_impl_or_dyn_traits_with_type_alias(
+        self,
+        scope_def_id: LocalDefId,
+    ) -> Option<(Vec<&'tcx hir::Ty<'tcx>>, Span)> {
+        let hir_id = self.hir().local_def_id_to_hir_id(scope_def_id);
+        let mut v = TraitObjectVisitor(vec![], self.hir());
+        // when the return type is a type alias
+        if let Some(hir::FnDecl { output: hir::FnRetTy::Return(hir_output), .. }) = self.hir().fn_decl_by_hir_id(hir_id)
+            && let hir::TyKind::Path(hir::QPath::Resolved(
+                None,
+                hir::Path { res: hir::def::Res::Def(DefKind::TyAlias, def_id), .. }, )) = hir_output.kind
+            && let Some(local_id) = def_id.as_local()
+            && let Some(alias_ty) = self.hir().get_by_def_id(local_id).alias_ty() // it is type alias
+            && let Some(alias_generics) = self.hir().get_by_def_id(local_id).generics()
+        {
+            v.visit_ty(alias_ty);
+            if !v.0.is_empty() {
+                return Some((v.0, alias_generics.span));
+            }
+        }
+        return None;
+    }
+
     pub fn return_type_impl_trait(self, scope_def_id: LocalDefId) -> Option<(Ty<'tcx>, Span)> {
         // `type_of()` will fail on these (#55796, #86483), so only allow `fn`s or closures.
         match self.hir().get_by_def_id(scope_def_id) {
@@ -1007,18 +1154,16 @@ impl<'tcx> TyCtxt<'tcx> {
             _ => return None,
         }
 
-        let ret_ty = self.type_of(scope_def_id);
+        let ret_ty = self.type_of(scope_def_id).subst_identity();
         match ret_ty.kind() {
             ty::FnDef(_, _) => {
                 let sig = ret_ty.fn_sig(self);
                 let output = self.erase_late_bound_regions(sig.output());
-                if output.is_impl_trait() {
+                output.is_impl_trait().then(|| {
                     let hir_id = self.hir().local_def_id_to_hir_id(scope_def_id);
                     let fn_decl = self.hir().fn_decl_by_hir_id(hir_id).unwrap();
-                    Some((output, fn_decl.output.span()))
-                } else {
-                    None
-                }
+                    (output, fn_decl.output.span())
+                })
             }
             _ => None,
         }
@@ -1049,20 +1194,15 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn caller_location_ty(self) -> Ty<'tcx> {
         self.mk_imm_ref(
             self.lifetimes.re_static,
-            self.bound_type_of(self.require_lang_item(LangItem::PanicLocation, None))
-                .subst(self, self.mk_substs([self.lifetimes.re_static.into()].iter())),
+            self.type_of(self.require_lang_item(LangItem::PanicLocation, None))
+                .subst(self, self.mk_substs(&[self.lifetimes.re_static.into()])),
         )
     }
 
     /// Returns a displayable description and article for the given `def_id` (e.g. `("a", "struct")`).
     pub fn article_and_description(self, def_id: DefId) -> (&'static str, &'static str) {
-        match self.def_kind(def_id) {
-            DefKind::Generator => match self.generator_kind(def_id).unwrap() {
-                rustc_hir::GeneratorKind::Async(..) => ("an", "async closure"),
-                rustc_hir::GeneratorKind::Gen => ("a", "generator"),
-            },
-            def_kind => (def_kind.article(), def_kind.descr(def_id)),
-        }
+        let kind = self.def_kind(def_id);
+        (self.def_kind_descr_article(kind, def_id), self.def_kind_descr(kind, def_id))
     }
 
     pub fn type_length_limit(self) -> Limit {
@@ -1078,7 +1218,11 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn const_eval_limit(self) -> Limit {
-        self.limits(()).const_eval_limit
+        if self.sess.opts.unstable_opts.tiny_const_eval_limit {
+            TINY_CONST_EVAL_LIMIT
+        } else {
+            self.limits(()).const_eval_limit
+        }
     }
 
     pub fn all_traits(self) -> impl Iterator<Item = DefId> + 'tcx {
@@ -1120,13 +1264,12 @@ macro_rules! nop_lift {
         impl<'a, 'tcx> Lift<'tcx> for $ty {
             type Lifted = $lifted;
             fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-                if tcx.interners.$set.contains_pointer_to(&InternedInSet(&*self.0.0)) {
+                tcx.interners
+                    .$set
+                    .contains_pointer_to(&InternedInSet(&*self.0.0))
                     // SAFETY: `self` is interned and therefore valid
                     // for the entire lifetime of the `TyCtxt`.
-                    Some(unsafe { mem::transmute(self) })
-                } else {
-                    None
-                }
+                    .then(|| unsafe { mem::transmute(self) })
             }
         }
     };
@@ -1134,20 +1277,20 @@ macro_rules! nop_lift {
 
 // Can't use the macros as we have reuse the `substs` here.
 //
-// See `intern_type_list` for more info.
+// See `mk_type_list` for more info.
 impl<'a, 'tcx> Lift<'tcx> for &'a List<Ty<'a>> {
     type Lifted = &'tcx List<Ty<'tcx>>;
     fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
         if self.is_empty() {
             return Some(List::empty());
         }
-        if tcx.interners.substs.contains_pointer_to(&InternedInSet(self.as_substs())) {
+
+        tcx.interners
+            .substs
+            .contains_pointer_to(&InternedInSet(self.as_substs()))
             // SAFETY: `self` is interned and therefore valid
             // for the entire lifetime of the `TyCtxt`.
-            Some(unsafe { mem::transmute::<&'a List<Ty<'a>>, &'tcx List<Ty<'tcx>>>(self) })
-        } else {
-            None
-        }
+            .then(|| unsafe { mem::transmute::<&'a List<Ty<'a>>, &'tcx List<Ty<'tcx>>>(self) })
     }
 }
 
@@ -1159,11 +1302,10 @@ macro_rules! nop_list_lift {
                 if self.is_empty() {
                     return Some(List::empty());
                 }
-                if tcx.interners.$set.contains_pointer_to(&InternedInSet(self)) {
-                    Some(unsafe { mem::transmute(self) })
-                } else {
-                    None
-                }
+                tcx.interners
+                    .$set
+                    .contains_pointer_to(&InternedInSet(self))
+                    .then(|| unsafe { mem::transmute(self) })
             }
         }
     };
@@ -1188,178 +1330,6 @@ CloneLiftImpls! { for<'tcx> {
     Constness, traits::WellFormedLoc, ImplPolarity, crate::mir::ReturnConstraint,
 } }
 
-pub mod tls {
-    use super::{ptr_eq, GlobalCtxt, TyCtxt};
-
-    use crate::dep_graph::TaskDepsRef;
-    use crate::ty::query;
-    use rustc_data_structures::sync::{self, Lock};
-    use rustc_errors::Diagnostic;
-    use std::mem;
-    use thin_vec::ThinVec;
-
-    #[cfg(not(parallel_compiler))]
-    use std::cell::Cell;
-
-    #[cfg(parallel_compiler)]
-    use rustc_rayon_core as rayon_core;
-
-    /// This is the implicit state of rustc. It contains the current
-    /// `TyCtxt` and query. It is updated when creating a local interner or
-    /// executing a new query. Whenever there's a `TyCtxt` value available
-    /// you should also have access to an `ImplicitCtxt` through the functions
-    /// in this module.
-    #[derive(Clone)]
-    pub struct ImplicitCtxt<'a, 'tcx> {
-        /// The current `TyCtxt`.
-        pub tcx: TyCtxt<'tcx>,
-
-        /// The current query job, if any. This is updated by `JobOwner::start` in
-        /// `ty::query::plumbing` when executing a query.
-        pub query: Option<query::QueryJobId>,
-
-        /// Where to store diagnostics for the current query job, if any.
-        /// This is updated by `JobOwner::start` in `ty::query::plumbing` when executing a query.
-        pub diagnostics: Option<&'a Lock<ThinVec<Diagnostic>>>,
-
-        /// Used to prevent queries from calling too deeply.
-        pub query_depth: usize,
-
-        /// The current dep graph task. This is used to add dependencies to queries
-        /// when executing them.
-        pub task_deps: TaskDepsRef<'a>,
-    }
-
-    impl<'a, 'tcx> ImplicitCtxt<'a, 'tcx> {
-        pub fn new(gcx: &'tcx GlobalCtxt<'tcx>) -> Self {
-            let tcx = TyCtxt { gcx };
-            ImplicitCtxt {
-                tcx,
-                query: None,
-                diagnostics: None,
-                query_depth: 0,
-                task_deps: TaskDepsRef::Ignore,
-            }
-        }
-    }
-
-    /// Sets Rayon's thread-local variable, which is preserved for Rayon jobs
-    /// to `value` during the call to `f`. It is restored to its previous value after.
-    /// This is used to set the pointer to the new `ImplicitCtxt`.
-    #[cfg(parallel_compiler)]
-    #[inline]
-    fn set_tlv<F: FnOnce() -> R, R>(value: usize, f: F) -> R {
-        rayon_core::tlv::with(value, f)
-    }
-
-    /// Gets Rayon's thread-local variable, which is preserved for Rayon jobs.
-    /// This is used to get the pointer to the current `ImplicitCtxt`.
-    #[cfg(parallel_compiler)]
-    #[inline]
-    pub fn get_tlv() -> usize {
-        rayon_core::tlv::get()
-    }
-
-    #[cfg(not(parallel_compiler))]
-    thread_local! {
-        /// A thread local variable that stores a pointer to the current `ImplicitCtxt`.
-        static TLV: Cell<usize> = const { Cell::new(0) };
-    }
-
-    /// Sets TLV to `value` during the call to `f`.
-    /// It is restored to its previous value after.
-    /// This is used to set the pointer to the new `ImplicitCtxt`.
-    #[cfg(not(parallel_compiler))]
-    #[inline]
-    fn set_tlv<F: FnOnce() -> R, R>(value: usize, f: F) -> R {
-        let old = get_tlv();
-        let _reset = rustc_data_structures::OnDrop(move || TLV.with(|tlv| tlv.set(old)));
-        TLV.with(|tlv| tlv.set(value));
-        f()
-    }
-
-    /// Gets the pointer to the current `ImplicitCtxt`.
-    #[cfg(not(parallel_compiler))]
-    #[inline]
-    fn get_tlv() -> usize {
-        TLV.with(|tlv| tlv.get())
-    }
-
-    /// Sets `context` as the new current `ImplicitCtxt` for the duration of the function `f`.
-    #[inline]
-    pub fn enter_context<'a, 'tcx, F, R>(context: &ImplicitCtxt<'a, 'tcx>, f: F) -> R
-    where
-        F: FnOnce(&ImplicitCtxt<'a, 'tcx>) -> R,
-    {
-        set_tlv(context as *const _ as usize, || f(&context))
-    }
-
-    /// Allows access to the current `ImplicitCtxt` in a closure if one is available.
-    #[inline]
-    pub fn with_context_opt<F, R>(f: F) -> R
-    where
-        F: for<'a, 'tcx> FnOnce(Option<&ImplicitCtxt<'a, 'tcx>>) -> R,
-    {
-        let context = get_tlv();
-        if context == 0 {
-            f(None)
-        } else {
-            // We could get an `ImplicitCtxt` pointer from another thread.
-            // Ensure that `ImplicitCtxt` is `Sync`.
-            sync::assert_sync::<ImplicitCtxt<'_, '_>>();
-
-            unsafe { f(Some(&*(context as *const ImplicitCtxt<'_, '_>))) }
-        }
-    }
-
-    /// Allows access to the current `ImplicitCtxt`.
-    /// Panics if there is no `ImplicitCtxt` available.
-    #[inline]
-    pub fn with_context<F, R>(f: F) -> R
-    where
-        F: for<'a, 'tcx> FnOnce(&ImplicitCtxt<'a, 'tcx>) -> R,
-    {
-        with_context_opt(|opt_context| f(opt_context.expect("no ImplicitCtxt stored in tls")))
-    }
-
-    /// Allows access to the current `ImplicitCtxt` whose tcx field is the same as the tcx argument
-    /// passed in. This means the closure is given an `ImplicitCtxt` with the same `'tcx` lifetime
-    /// as the `TyCtxt` passed in.
-    /// This will panic if you pass it a `TyCtxt` which is different from the current
-    /// `ImplicitCtxt`'s `tcx` field.
-    #[inline]
-    pub fn with_related_context<'tcx, F, R>(tcx: TyCtxt<'tcx>, f: F) -> R
-    where
-        F: FnOnce(&ImplicitCtxt<'_, 'tcx>) -> R,
-    {
-        with_context(|context| unsafe {
-            assert!(ptr_eq(context.tcx.gcx, tcx.gcx));
-            let context: &ImplicitCtxt<'_, '_> = mem::transmute(context);
-            f(context)
-        })
-    }
-
-    /// Allows access to the `TyCtxt` in the current `ImplicitCtxt`.
-    /// Panics if there is no `ImplicitCtxt` available.
-    #[inline]
-    pub fn with<F, R>(f: F) -> R
-    where
-        F: for<'tcx> FnOnce(TyCtxt<'tcx>) -> R,
-    {
-        with_context(|context| f(context.tcx))
-    }
-
-    /// Allows access to the `TyCtxt` in the current `ImplicitCtxt`.
-    /// The closure is passed None if there is no `ImplicitCtxt` available.
-    #[inline]
-    pub fn with_opt<F, R>(f: F) -> R
-    where
-        F: for<'tcx> FnOnce(Option<TyCtxt<'tcx>>) -> R,
-    {
-        with_context_opt(|opt_context| f(opt_context.map(|context| context.tcx)))
-    }
-}
-
 macro_rules! sty_debug_print {
     ($fmt: expr, $ctxt: expr, $($variant: ident),*) => {{
         // Curious inner module to allow variant names to be used as
@@ -1452,6 +1422,7 @@ impl<'tcx> TyCtxt<'tcx> {
                     Placeholder,
                     Generator,
                     GeneratorWitness,
+                    GeneratorWitnessMIR,
                     Dynamic,
                     Closure,
                     Tuple,
@@ -1547,7 +1518,7 @@ impl<'tcx, T: Hash> Hash for InternedInSet<'tcx, List<T>> {
 }
 
 macro_rules! direct_interners {
-    ($($name:ident: $method:ident($ty:ty): $ret_ctor:ident -> $ret_ty:ty,)+) => {
+    ($($name:ident: $vis:vis $method:ident($ty:ty): $ret_ctor:ident -> $ret_ty:ty,)+) => {
         $(impl<'tcx> Borrow<$ty> for InternedInSet<'tcx, $ty> {
             fn borrow<'a>(&'a self) -> &'a $ty {
                 &self.0
@@ -1573,7 +1544,7 @@ macro_rules! direct_interners {
         }
 
         impl<'tcx> TyCtxt<'tcx> {
-            pub fn $method(self, v: $ty) -> $ret_ty {
+            $vis fn $method(self, v: $ty) -> $ret_ty {
                 $ret_ctor(Interned::new_unchecked(self.interners.$name.intern(v, |v| {
                     InternedInSet(self.interners.arena.alloc(v))
                 }).0))
@@ -1582,36 +1553,47 @@ macro_rules! direct_interners {
     }
 }
 
+// Functions with a `mk_` prefix are intended for use outside this file and
+// crate. Functions with an `intern_` prefix are intended for use within this
+// file only, and have a corresponding `mk_` function.
 direct_interners! {
-    region: mk_region(RegionKind<'tcx>): Region -> Region<'tcx>,
-    const_: mk_const_internal(ConstData<'tcx>): Const -> Const<'tcx>,
-    const_allocation: intern_const_alloc(Allocation): ConstAllocation -> ConstAllocation<'tcx>,
-    layout: intern_layout(LayoutS<VariantIdx>): Layout -> Layout<'tcx>,
-    adt_def: intern_adt_def(AdtDefData): AdtDef -> AdtDef<'tcx>,
+    region: intern_region(RegionKind<'tcx>): Region -> Region<'tcx>,
+    const_: intern_const(ConstData<'tcx>): Const -> Const<'tcx>,
+    const_allocation: pub mk_const_alloc(Allocation): ConstAllocation -> ConstAllocation<'tcx>,
+    layout: pub mk_layout(LayoutS): Layout -> Layout<'tcx>,
+    adt_def: pub mk_adt_def_from_data(AdtDefData): AdtDef -> AdtDef<'tcx>,
+    external_constraints: pub mk_external_constraints(ExternalConstraintsData<'tcx>):
+        ExternalConstraints -> ExternalConstraints<'tcx>,
 }
 
 macro_rules! slice_interners {
-    ($($field:ident: $method:ident($ty:ty)),+ $(,)?) => (
+    ($($field:ident: $vis:vis $method:ident($ty:ty)),+ $(,)?) => (
         impl<'tcx> TyCtxt<'tcx> {
-            $(pub fn $method(self, v: &[$ty]) -> &'tcx List<$ty> {
-                self.interners.$field.intern_ref(v, || {
-                    InternedInSet(List::from_arena(&*self.arena, v))
-                }).0
+            $($vis fn $method(self, v: &[$ty]) -> &'tcx List<$ty> {
+                if v.is_empty() {
+                    List::empty()
+                } else {
+                    self.interners.$field.intern_ref(v, || {
+                        InternedInSet(List::from_arena(&*self.arena, v))
+                    }).0
+                }
             })+
         }
     );
 }
 
+// These functions intern slices. They all have a corresponding
+// `mk_foo_from_iter` function that interns an iterator. The slice version
+// should be used when possible, because it's faster.
 slice_interners!(
-    const_lists: _intern_const_list(Const<'tcx>),
-    substs: _intern_substs(GenericArg<'tcx>),
-    canonical_var_infos: _intern_canonical_var_infos(CanonicalVarInfo<'tcx>),
-    poly_existential_predicates:
-        _intern_poly_existential_predicates(PolyExistentialPredicate<'tcx>),
-    predicates: _intern_predicates(Predicate<'tcx>),
-    projs: _intern_projs(ProjectionKind),
-    place_elems: _intern_place_elems(PlaceElem<'tcx>),
-    bound_variable_kinds: _intern_bound_variable_kinds(ty::BoundVariableKind),
+    const_lists: pub mk_const_list(Const<'tcx>),
+    substs: pub mk_substs(GenericArg<'tcx>),
+    canonical_var_infos: pub mk_canonical_var_infos(CanonicalVarInfo<'tcx>),
+    poly_existential_predicates: intern_poly_existential_predicates(PolyExistentialPredicate<'tcx>),
+    predicates: intern_predicates(Predicate<'tcx>),
+    projs: pub mk_projs(ProjectionKind),
+    place_elems: pub mk_place_elems(PlaceElem<'tcx>),
+    bound_variable_kinds: pub mk_bound_variable_kinds(ty::BoundVariableKind),
 );
 
 impl<'tcx> TyCtxt<'tcx> {
@@ -1688,24 +1670,18 @@ impl<'tcx> TyCtxt<'tcx> {
         unsafety: hir::Unsafety,
     ) -> PolyFnSig<'tcx> {
         sig.map_bound(|s| {
-            let params_iter = match s.inputs()[0].kind() {
-                ty::Tuple(params) => params.into_iter(),
+            let params = match s.inputs()[0].kind() {
+                ty::Tuple(params) => *params,
                 _ => bug!(),
             };
-            self.mk_fn_sig(params_iter, s.output(), s.c_variadic, unsafety, abi::Abi::Rust)
+            self.mk_fn_sig(params, s.output(), s.c_variadic, unsafety, abi::Abi::Rust)
         })
     }
 
-    /// Same a `self.mk_region(kind)`, but avoids accessing the interners if
-    /// `*r == kind`.
-    #[inline]
-    pub fn reuse_or_mk_region(self, r: Region<'tcx>, kind: RegionKind<'tcx>) -> Region<'tcx> {
-        if *r == kind { r } else { self.mk_region(kind) }
-    }
-
+    // Avoid this in favour of more specific `mk_*` methods, where possible.
     #[allow(rustc::usage_of_ty_tykind)]
     #[inline]
-    pub fn mk_ty(self, st: TyKind<'tcx>) -> Ty<'tcx> {
+    pub fn mk_ty_from_kind(self, st: TyKind<'tcx>) -> Ty<'tcx> {
         self.interners.intern_ty(
             st,
             self.sess,
@@ -1770,12 +1746,12 @@ impl<'tcx> TyCtxt<'tcx> {
     #[inline]
     pub fn mk_adt(self, def: AdtDef<'tcx>, substs: SubstsRef<'tcx>) -> Ty<'tcx> {
         // Take a copy of substs so that we own the vectors inside.
-        self.mk_ty(Adt(def, substs))
+        self.mk_ty_from_kind(Adt(def, substs))
     }
 
     #[inline]
     pub fn mk_foreign(self, def_id: DefId) -> Ty<'tcx> {
-        self.mk_ty(Foreign(def_id))
+        self.mk_ty_from_kind(Foreign(def_id))
     }
 
     fn mk_generic_adt(self, wrapper_def_id: DefId, ty_param: Ty<'tcx>) -> Ty<'tcx> {
@@ -1788,11 +1764,11 @@ impl<'tcx> TyCtxt<'tcx> {
                         ty_param.into()
                     } else {
                         assert!(has_default);
-                        self.bound_type_of(param.def_id).subst(self, substs).into()
+                        self.type_of(param.def_id).subst(self, substs).into()
                     }
                 }
             });
-        self.mk_ty(Adt(adt_def, substs))
+        self.mk_ty_from_kind(Adt(adt_def, substs))
     }
 
     #[inline]
@@ -1821,12 +1797,12 @@ impl<'tcx> TyCtxt<'tcx> {
 
     #[inline]
     pub fn mk_ptr(self, tm: TypeAndMut<'tcx>) -> Ty<'tcx> {
-        self.mk_ty(RawPtr(tm))
+        self.mk_ty_from_kind(RawPtr(tm))
     }
 
     #[inline]
     pub fn mk_ref(self, r: Region<'tcx>, tm: TypeAndMut<'tcx>) -> Ty<'tcx> {
-        self.mk_ty(Ref(r, tm.ty, tm.mutbl))
+        self.mk_ty_from_kind(Ref(r, tm.ty, tm.mutbl))
     }
 
     #[inline]
@@ -1851,21 +1827,34 @@ impl<'tcx> TyCtxt<'tcx> {
 
     #[inline]
     pub fn mk_array(self, ty: Ty<'tcx>, n: u64) -> Ty<'tcx> {
-        self.mk_ty(Array(ty, ty::Const::from_usize(self, n)))
+        self.mk_ty_from_kind(Array(ty, ty::Const::from_target_usize(self, n)))
+    }
+
+    #[inline]
+    pub fn mk_array_with_const_len(self, ty: Ty<'tcx>, ct: Const<'tcx>) -> Ty<'tcx> {
+        self.mk_ty_from_kind(Array(ty, ct))
     }
 
     #[inline]
     pub fn mk_slice(self, ty: Ty<'tcx>) -> Ty<'tcx> {
-        self.mk_ty(Slice(ty))
+        self.mk_ty_from_kind(Slice(ty))
     }
 
     #[inline]
-    pub fn intern_tup(self, ts: &[Ty<'tcx>]) -> Ty<'tcx> {
-        self.mk_ty(Tuple(self.intern_type_list(&ts)))
+    pub fn mk_tup(self, ts: &[Ty<'tcx>]) -> Ty<'tcx> {
+        if ts.is_empty() {
+            self.types.unit
+        } else {
+            self.mk_ty_from_kind(Tuple(self.mk_type_list(&ts)))
+        }
     }
 
-    pub fn mk_tup<I: InternAs<Ty<'tcx>, Ty<'tcx>>>(self, iter: I) -> I::Output {
-        iter.intern_with(|ts| self.mk_ty(Tuple(self.intern_type_list(&ts))))
+    pub fn mk_tup_from_iter<I, T>(self, iter: I) -> T::Output
+    where
+        I: Iterator<Item = T>,
+        T: CollectAndApply<Ty<'tcx>, Ty<'tcx>>,
+    {
+        T::collect_and_apply(iter, |ts| self.mk_tup(ts))
     }
 
     #[inline]
@@ -1882,17 +1871,17 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn mk_fn_def(
         self,
         def_id: DefId,
-        substs: impl IntoIterator<Item = impl Into<GenericArg<'tcx>>>,
+        substs: impl IntoIterator<Item: Into<GenericArg<'tcx>>>,
     ) -> Ty<'tcx> {
-        let substs = self.check_substs(def_id, substs);
-        self.mk_ty(FnDef(def_id, substs))
+        let substs = self.check_and_mk_substs(def_id, substs);
+        self.mk_ty_from_kind(FnDef(def_id, substs))
     }
 
     #[inline(always)]
-    fn check_substs(
+    fn check_and_mk_substs(
         self,
         _def_id: DefId,
-        substs: impl IntoIterator<Item = impl Into<GenericArg<'tcx>>>,
+        substs: impl IntoIterator<Item: Into<GenericArg<'tcx>>>,
     ) -> SubstsRef<'tcx> {
         let substs = substs.into_iter().map(Into::into);
         #[cfg(debug_assertions)]
@@ -1905,12 +1894,12 @@ impl<'tcx> TyCtxt<'tcx> {
                 substs.collect::<Vec<_>>(),
             );
         }
-        self.mk_substs(substs)
+        self.mk_substs_from_iter(substs)
     }
 
     #[inline]
     pub fn mk_fn_ptr(self, fty: PolyFnSig<'tcx>) -> Ty<'tcx> {
-        self.mk_ty(FnPtr(fty))
+        self.mk_ty_from_kind(FnPtr(fty))
     }
 
     #[inline]
@@ -1920,21 +1909,21 @@ impl<'tcx> TyCtxt<'tcx> {
         reg: ty::Region<'tcx>,
         repr: DynKind,
     ) -> Ty<'tcx> {
-        self.mk_ty(Dynamic(obj, reg, repr))
+        self.mk_ty_from_kind(Dynamic(obj, reg, repr))
     }
 
     #[inline]
     pub fn mk_projection(
         self,
         item_def_id: DefId,
-        substs: impl IntoIterator<Item = impl Into<GenericArg<'tcx>>>,
+        substs: impl IntoIterator<Item: Into<GenericArg<'tcx>>>,
     ) -> Ty<'tcx> {
-        self.mk_ty(Alias(ty::Projection, self.mk_alias_ty(item_def_id, substs)))
+        self.mk_alias(ty::Projection, self.mk_alias_ty(item_def_id, substs))
     }
 
     #[inline]
     pub fn mk_closure(self, closure_id: DefId, closure_substs: SubstsRef<'tcx>) -> Ty<'tcx> {
-        self.mk_ty(Closure(closure_id, closure_substs))
+        self.mk_ty_from_kind(Closure(closure_id, closure_substs))
     }
 
     #[inline]
@@ -1944,71 +1933,183 @@ impl<'tcx> TyCtxt<'tcx> {
         generator_substs: SubstsRef<'tcx>,
         movability: hir::Movability,
     ) -> Ty<'tcx> {
-        self.mk_ty(Generator(id, generator_substs, movability))
+        self.mk_ty_from_kind(Generator(id, generator_substs, movability))
     }
 
     #[inline]
     pub fn mk_generator_witness(self, types: ty::Binder<'tcx, &'tcx List<Ty<'tcx>>>) -> Ty<'tcx> {
-        self.mk_ty(GeneratorWitness(types))
+        self.mk_ty_from_kind(GeneratorWitness(types))
     }
 
     /// Creates a `&mut Context<'_>` [`Ty`] with erased lifetimes.
     pub fn mk_task_context(self) -> Ty<'tcx> {
         let context_did = self.require_lang_item(LangItem::Context, None);
         let context_adt_ref = self.adt_def(context_did);
-        let context_substs = self.intern_substs(&[self.lifetimes.re_erased.into()]);
+        let context_substs = self.mk_substs(&[self.lifetimes.re_erased.into()]);
         let context_ty = self.mk_adt(context_adt_ref, context_substs);
         self.mk_mut_ref(self.lifetimes.re_erased, context_ty)
     }
 
     #[inline]
-    pub fn mk_ty_var(self, v: TyVid) -> Ty<'tcx> {
-        self.mk_ty_infer(TyVar(v))
+    pub fn mk_generator_witness_mir(self, id: DefId, substs: SubstsRef<'tcx>) -> Ty<'tcx> {
+        self.mk_ty_from_kind(GeneratorWitnessMIR(id, substs))
     }
 
     #[inline]
     pub fn mk_const(self, kind: impl Into<ty::ConstKind<'tcx>>, ty: Ty<'tcx>) -> Const<'tcx> {
-        self.mk_const_internal(ty::ConstData { kind: kind.into(), ty })
+        self.intern_const(ty::ConstData { kind: kind.into(), ty })
+    }
+
+    #[inline]
+    pub fn mk_ty_var(self, v: TyVid) -> Ty<'tcx> {
+        // Use a pre-interned one when possible.
+        self.types
+            .ty_vars
+            .get(v.as_usize())
+            .copied()
+            .unwrap_or_else(|| self.mk_ty_from_kind(Infer(TyVar(v))))
     }
 
     #[inline]
     pub fn mk_int_var(self, v: IntVid) -> Ty<'tcx> {
-        self.mk_ty_infer(IntVar(v))
+        self.mk_ty_from_kind(Infer(IntVar(v)))
     }
 
     #[inline]
     pub fn mk_float_var(self, v: FloatVid) -> Ty<'tcx> {
-        self.mk_ty_infer(FloatVar(v))
+        self.mk_ty_from_kind(Infer(FloatVar(v)))
+    }
+
+    #[inline]
+    pub fn mk_fresh_ty(self, n: u32) -> Ty<'tcx> {
+        // Use a pre-interned one when possible.
+        self.types
+            .fresh_tys
+            .get(n as usize)
+            .copied()
+            .unwrap_or_else(|| self.mk_ty_from_kind(Infer(ty::FreshTy(n))))
     }
 
     #[inline]
-    pub fn mk_ty_infer(self, it: InferTy) -> Ty<'tcx> {
-        self.mk_ty(Infer(it))
+    pub fn mk_fresh_int_ty(self, n: u32) -> Ty<'tcx> {
+        // Use a pre-interned one when possible.
+        self.types
+            .fresh_int_tys
+            .get(n as usize)
+            .copied()
+            .unwrap_or_else(|| self.mk_ty_from_kind(Infer(ty::FreshIntTy(n))))
+    }
+
+    #[inline]
+    pub fn mk_fresh_float_ty(self, n: u32) -> Ty<'tcx> {
+        // Use a pre-interned one when possible.
+        self.types
+            .fresh_float_tys
+            .get(n as usize)
+            .copied()
+            .unwrap_or_else(|| self.mk_ty_from_kind(Infer(ty::FreshFloatTy(n))))
     }
 
     #[inline]
     pub fn mk_ty_param(self, index: u32, name: Symbol) -> Ty<'tcx> {
-        self.mk_ty(Param(ParamTy { index, name }))
+        self.mk_ty_from_kind(Param(ParamTy { index, name }))
     }
 
     pub fn mk_param_from_def(self, param: &ty::GenericParamDef) -> GenericArg<'tcx> {
         match param.kind {
             GenericParamDefKind::Lifetime => {
-                self.mk_region(ty::ReEarlyBound(param.to_early_bound_region_data())).into()
+                self.mk_re_early_bound(param.to_early_bound_region_data()).into()
             }
             GenericParamDefKind::Type { .. } => self.mk_ty_param(param.index, param.name).into(),
             GenericParamDefKind::Const { .. } => self
                 .mk_const(
                     ParamConst { index: param.index, name: param.name },
-                    self.type_of(param.def_id),
+                    self.type_of(param.def_id)
+                        .no_bound_vars()
+                        .expect("const parameter types cannot be generic"),
                 )
                 .into(),
         }
     }
 
     #[inline]
+    pub fn mk_bound(self, index: ty::DebruijnIndex, bound_ty: ty::BoundTy) -> Ty<'tcx> {
+        self.mk_ty_from_kind(Bound(index, bound_ty))
+    }
+
+    #[inline]
+    pub fn mk_placeholder(self, placeholder: ty::PlaceholderType) -> Ty<'tcx> {
+        self.mk_ty_from_kind(Placeholder(placeholder))
+    }
+
+    #[inline]
+    pub fn mk_alias(self, kind: ty::AliasKind, alias_ty: ty::AliasTy<'tcx>) -> Ty<'tcx> {
+        self.mk_ty_from_kind(Alias(kind, alias_ty))
+    }
+
+    #[inline]
     pub fn mk_opaque(self, def_id: DefId, substs: SubstsRef<'tcx>) -> Ty<'tcx> {
-        self.mk_ty(Alias(ty::Opaque, self.mk_alias_ty(def_id, substs)))
+        self.mk_alias(ty::Opaque, self.mk_alias_ty(def_id, substs))
+    }
+
+    #[inline]
+    pub fn mk_re_early_bound(self, early_bound_region: ty::EarlyBoundRegion) -> Region<'tcx> {
+        self.intern_region(ty::ReEarlyBound(early_bound_region))
+    }
+
+    #[inline]
+    pub fn mk_re_late_bound(
+        self,
+        debruijn: ty::DebruijnIndex,
+        bound_region: ty::BoundRegion,
+    ) -> Region<'tcx> {
+        // Use a pre-interned one when possible.
+        if let ty::BoundRegion { var, kind: ty::BrAnon(v, None) } = bound_region
+            && var.as_u32() == v
+            && let Some(inner) = self.lifetimes.re_late_bounds.get(debruijn.as_usize())
+            && let Some(re) = inner.get(v as usize).copied()
+        {
+            re
+        } else {
+            self.intern_region(ty::ReLateBound(debruijn, bound_region))
+        }
+    }
+
+    #[inline]
+    pub fn mk_re_free(self, scope: DefId, bound_region: ty::BoundRegionKind) -> Region<'tcx> {
+        self.intern_region(ty::ReFree(ty::FreeRegion { scope, bound_region }))
+    }
+
+    #[inline]
+    pub fn mk_re_var(self, v: ty::RegionVid) -> Region<'tcx> {
+        // Use a pre-interned one when possible.
+        self.lifetimes
+            .re_vars
+            .get(v.as_usize())
+            .copied()
+            .unwrap_or_else(|| self.intern_region(ty::ReVar(v)))
+    }
+
+    #[inline]
+    pub fn mk_re_placeholder(self, placeholder: ty::PlaceholderRegion) -> Region<'tcx> {
+        self.intern_region(ty::RePlaceholder(placeholder))
+    }
+
+    // Avoid this in favour of more specific `mk_re_*` methods, where possible,
+    // to avoid the cost of the `match`.
+    pub fn mk_region_from_kind(self, kind: ty::RegionKind<'tcx>) -> Region<'tcx> {
+        match kind {
+            ty::ReEarlyBound(region) => self.mk_re_early_bound(region),
+            ty::ReLateBound(debruijn, region) => self.mk_re_late_bound(debruijn, region),
+            ty::ReFree(ty::FreeRegion { scope, bound_region }) => {
+                self.mk_re_free(scope, bound_region)
+            }
+            ty::ReStatic => self.lifetimes.re_static,
+            ty::ReVar(vid) => self.mk_re_var(vid),
+            ty::RePlaceholder(region) => self.mk_re_placeholder(region),
+            ty::ReErased => self.lifetimes.re_erased,
+            ty::ReError(reported) => self.mk_re_error(reported),
+        }
     }
 
     pub fn mk_place_field(self, place: Place<'tcx>, f: Field, ty: Ty<'tcx>) -> Place<'tcx> {
@@ -2050,10 +2151,10 @@ impl<'tcx> TyCtxt<'tcx> {
         let mut projection = place.projection.to_vec();
         projection.push(elem);
 
-        Place { local: place.local, projection: self.intern_place_elems(&projection) }
+        Place { local: place.local, projection: self.mk_place_elems(&projection) }
     }
 
-    pub fn intern_poly_existential_predicates(
+    pub fn mk_poly_existential_predicates(
         self,
         eps: &[PolyExistentialPredicate<'tcx>],
     ) -> &'tcx List<PolyExistentialPredicate<'tcx>> {
@@ -2063,125 +2164,109 @@ impl<'tcx> TyCtxt<'tcx> {
                 .all(|[a, b]| a.skip_binder().stable_cmp(self, &b.skip_binder())
                     != Ordering::Greater)
         );
-        self._intern_poly_existential_predicates(eps)
+        self.intern_poly_existential_predicates(eps)
     }
 
-    pub fn intern_predicates(self, preds: &[Predicate<'tcx>]) -> &'tcx List<Predicate<'tcx>> {
+    pub fn mk_predicates(self, preds: &[Predicate<'tcx>]) -> &'tcx List<Predicate<'tcx>> {
         // FIXME consider asking the input slice to be sorted to avoid
         // re-interning permutations, in which case that would be asserted
         // here.
-        if preds.is_empty() {
-            // The macro-generated method below asserts we don't intern an empty slice.
-            List::empty()
-        } else {
-            self._intern_predicates(preds)
-        }
-    }
-
-    pub fn mk_const_list<I: InternAs<ty::Const<'tcx>, &'tcx List<ty::Const<'tcx>>>>(
-        self,
-        iter: I,
-    ) -> I::Output {
-        iter.intern_with(|xs| self.intern_const_list(xs))
-    }
-
-    pub fn intern_const_list(self, cs: &[ty::Const<'tcx>]) -> &'tcx List<ty::Const<'tcx>> {
-        if cs.is_empty() { List::empty() } else { self._intern_const_list(cs) }
-    }
-
-    pub fn intern_type_list(self, ts: &[Ty<'tcx>]) -> &'tcx List<Ty<'tcx>> {
-        if ts.is_empty() {
-            List::empty()
-        } else {
-            // Actually intern type lists as lists of `GenericArg`s.
-            //
-            // Transmuting from `Ty<'tcx>` to `GenericArg<'tcx>` is sound
-            // as explained in ty_slice_as_generic_arg`. With this,
-            // we guarantee that even when transmuting between `List<Ty<'tcx>>`
-            // and `List<GenericArg<'tcx>>`, the uniqueness requirement for
-            // lists is upheld.
-            let substs = self._intern_substs(ty::subst::ty_slice_as_generic_args(ts));
-            substs.try_as_type_list().unwrap()
-        }
-    }
-
-    pub fn intern_substs(self, ts: &[GenericArg<'tcx>]) -> &'tcx List<GenericArg<'tcx>> {
-        if ts.is_empty() { List::empty() } else { self._intern_substs(ts) }
+        self.intern_predicates(preds)
     }
 
-    pub fn intern_projs(self, ps: &[ProjectionKind]) -> &'tcx List<ProjectionKind> {
-        if ps.is_empty() { List::empty() } else { self._intern_projs(ps) }
-    }
-
-    pub fn intern_place_elems(self, ts: &[PlaceElem<'tcx>]) -> &'tcx List<PlaceElem<'tcx>> {
-        if ts.is_empty() { List::empty() } else { self._intern_place_elems(ts) }
-    }
-
-    pub fn intern_canonical_var_infos(
-        self,
-        ts: &[CanonicalVarInfo<'tcx>],
-    ) -> CanonicalVarInfos<'tcx> {
-        if ts.is_empty() { List::empty() } else { self._intern_canonical_var_infos(ts) }
-    }
-
-    pub fn intern_bound_variable_kinds(
-        self,
-        ts: &[ty::BoundVariableKind],
-    ) -> &'tcx List<ty::BoundVariableKind> {
-        if ts.is_empty() { List::empty() } else { self._intern_bound_variable_kinds(ts) }
+    pub fn mk_const_list_from_iter<I, T>(self, iter: I) -> T::Output
+    where
+        I: Iterator<Item = T>,
+        T: CollectAndApply<ty::Const<'tcx>, &'tcx List<ty::Const<'tcx>>>,
+    {
+        T::collect_and_apply(iter, |xs| self.mk_const_list(xs))
     }
 
-    pub fn mk_fn_sig<I>(
+    pub fn mk_type_list(self, ts: &[Ty<'tcx>]) -> &'tcx List<Ty<'tcx>> {
+        // Actually intern type lists as lists of `GenericArg`s.
+        //
+        // Transmuting from `Ty<'tcx>` to `GenericArg<'tcx>` is sound
+        // as explained in `ty_slice_as_generic_arg`. With this,
+        // we guarantee that even when transmuting between `List<Ty<'tcx>>`
+        // and `List<GenericArg<'tcx>>`, the uniqueness requirement for
+        // lists is upheld.
+        let substs = self.mk_substs(ty::subst::ty_slice_as_generic_args(ts));
+        substs.try_as_type_list().unwrap()
+    }
+
+    // Unlike various other `mk_*_from_iter` functions, this one uses `I:
+    // IntoIterator` instead of `I: Iterator`, and it doesn't have a slice
+    // variant, because of the need to combine `inputs` and `output`. This
+    // explains the lack of `_from_iter` suffix.
+    pub fn mk_fn_sig<I, T>(
         self,
         inputs: I,
         output: I::Item,
         c_variadic: bool,
         unsafety: hir::Unsafety,
         abi: abi::Abi,
-    ) -> <I::Item as InternIteratorElement<Ty<'tcx>, ty::FnSig<'tcx>>>::Output
+    ) -> T::Output
     where
-        I: Iterator<Item: InternIteratorElement<Ty<'tcx>, ty::FnSig<'tcx>>>,
+        I: IntoIterator<Item = T>,
+        T: CollectAndApply<Ty<'tcx>, ty::FnSig<'tcx>>,
     {
-        inputs.chain(iter::once(output)).intern_with(|xs| ty::FnSig {
-            inputs_and_output: self.intern_type_list(xs),
+        T::collect_and_apply(inputs.into_iter().chain(iter::once(output)), |xs| ty::FnSig {
+            inputs_and_output: self.mk_type_list(xs),
             c_variadic,
             unsafety,
             abi,
         })
     }
 
-    pub fn mk_poly_existential_predicates<
-        I: InternAs<PolyExistentialPredicate<'tcx>, &'tcx List<PolyExistentialPredicate<'tcx>>>,
-    >(
-        self,
-        iter: I,
-    ) -> I::Output {
-        iter.intern_with(|xs| self.intern_poly_existential_predicates(xs))
+    pub fn mk_poly_existential_predicates_from_iter<I, T>(self, iter: I) -> T::Output
+    where
+        I: Iterator<Item = T>,
+        T: CollectAndApply<
+                PolyExistentialPredicate<'tcx>,
+                &'tcx List<PolyExistentialPredicate<'tcx>>,
+            >,
+    {
+        T::collect_and_apply(iter, |xs| self.mk_poly_existential_predicates(xs))
     }
 
-    pub fn mk_predicates<I: InternAs<Predicate<'tcx>, &'tcx List<Predicate<'tcx>>>>(
-        self,
-        iter: I,
-    ) -> I::Output {
-        iter.intern_with(|xs| self.intern_predicates(xs))
+    pub fn mk_predicates_from_iter<I, T>(self, iter: I) -> T::Output
+    where
+        I: Iterator<Item = T>,
+        T: CollectAndApply<Predicate<'tcx>, &'tcx List<Predicate<'tcx>>>,
+    {
+        T::collect_and_apply(iter, |xs| self.mk_predicates(xs))
     }
 
-    pub fn mk_type_list<I: InternAs<Ty<'tcx>, &'tcx List<Ty<'tcx>>>>(self, iter: I) -> I::Output {
-        iter.intern_with(|xs| self.intern_type_list(xs))
+    pub fn mk_type_list_from_iter<I, T>(self, iter: I) -> T::Output
+    where
+        I: Iterator<Item = T>,
+        T: CollectAndApply<Ty<'tcx>, &'tcx List<Ty<'tcx>>>,
+    {
+        T::collect_and_apply(iter, |xs| self.mk_type_list(xs))
     }
 
-    pub fn mk_substs<I: InternAs<GenericArg<'tcx>, &'tcx List<GenericArg<'tcx>>>>(
-        self,
-        iter: I,
-    ) -> I::Output {
-        iter.intern_with(|xs| self.intern_substs(xs))
+    pub fn mk_substs_from_iter<I, T>(self, iter: I) -> T::Output
+    where
+        I: Iterator<Item = T>,
+        T: CollectAndApply<GenericArg<'tcx>, &'tcx List<GenericArg<'tcx>>>,
+    {
+        T::collect_and_apply(iter, |xs| self.mk_substs(xs))
     }
 
-    pub fn mk_place_elems<I: InternAs<PlaceElem<'tcx>, &'tcx List<PlaceElem<'tcx>>>>(
-        self,
-        iter: I,
-    ) -> I::Output {
-        iter.intern_with(|xs| self.intern_place_elems(xs))
+    pub fn mk_canonical_var_infos_from_iter<I, T>(self, iter: I) -> T::Output
+    where
+        I: Iterator<Item = T>,
+        T: CollectAndApply<CanonicalVarInfo<'tcx>, &'tcx List<CanonicalVarInfo<'tcx>>>,
+    {
+        T::collect_and_apply(iter, |xs| self.mk_canonical_var_infos(xs))
+    }
+
+    pub fn mk_place_elems_from_iter<I, T>(self, iter: I) -> T::Output
+    where
+        I: Iterator<Item = T>,
+        T: CollectAndApply<PlaceElem<'tcx>, &'tcx List<PlaceElem<'tcx>>>,
+    {
+        T::collect_and_apply(iter, |xs| self.mk_place_elems(xs))
     }
 
     pub fn mk_substs_trait(
@@ -2189,34 +2274,33 @@ impl<'tcx> TyCtxt<'tcx> {
         self_ty: Ty<'tcx>,
         rest: impl IntoIterator<Item = GenericArg<'tcx>>,
     ) -> SubstsRef<'tcx> {
-        self.mk_substs(iter::once(self_ty.into()).chain(rest))
+        self.mk_substs_from_iter(iter::once(self_ty.into()).chain(rest))
     }
 
     pub fn mk_trait_ref(
         self,
         trait_def_id: DefId,
-        substs: impl IntoIterator<Item = impl Into<GenericArg<'tcx>>>,
+        substs: impl IntoIterator<Item: Into<GenericArg<'tcx>>>,
     ) -> ty::TraitRef<'tcx> {
-        let substs = self.check_substs(trait_def_id, substs);
+        let substs = self.check_and_mk_substs(trait_def_id, substs);
         ty::TraitRef { def_id: trait_def_id, substs, _use_mk_trait_ref_instead: () }
     }
 
     pub fn mk_alias_ty(
         self,
         def_id: DefId,
-        substs: impl IntoIterator<Item = impl Into<GenericArg<'tcx>>>,
+        substs: impl IntoIterator<Item: Into<GenericArg<'tcx>>>,
     ) -> ty::AliasTy<'tcx> {
-        let substs = self.check_substs(def_id, substs);
+        let substs = self.check_and_mk_substs(def_id, substs);
         ty::AliasTy { def_id, substs, _use_mk_alias_ty_instead: () }
     }
 
-    pub fn mk_bound_variable_kinds<
-        I: InternAs<ty::BoundVariableKind, &'tcx List<ty::BoundVariableKind>>,
-    >(
-        self,
-        iter: I,
-    ) -> I::Output {
-        iter.intern_with(|xs| self.intern_bound_variable_kinds(xs))
+    pub fn mk_bound_variable_kinds_from_iter<I, T>(self, iter: I) -> T::Output
+    where
+        I: Iterator<Item = T>,
+        T: CollectAndApply<ty::BoundVariableKind, &'tcx List<ty::BoundVariableKind>>,
+    {
+        T::collect_and_apply(iter, |xs| self.mk_bound_variable_kinds(xs))
     }
 
     /// Emit a lint at `span` from a lint struct (some type that implements `DecorateLint`,
@@ -2291,26 +2375,23 @@ impl<'tcx> TyCtxt<'tcx> {
         Some(&*candidates)
     }
 
-    pub fn named_region(self, id: HirId) -> Option<resolve_lifetime::Region> {
+    pub fn named_bound_var(self, id: HirId) -> Option<resolve_bound_vars::ResolvedArg> {
         debug!(?id, "named_region");
-        self.named_region_map(id.owner).and_then(|map| map.get(&id.local_id).cloned())
+        self.named_variable_map(id.owner).and_then(|map| map.get(&id.local_id).cloned())
     }
 
     pub fn is_late_bound(self, id: HirId) -> bool {
-        self.is_late_bound_map(id.owner.def_id).map_or(false, |set| {
-            let def_id = self.hir().local_def_id(id);
-            set.contains(&def_id)
-        })
+        self.is_late_bound_map(id.owner).map_or(false, |set| set.contains(&id.local_id))
     }
 
     pub fn late_bound_vars(self, id: HirId) -> &'tcx List<ty::BoundVariableKind> {
         self.mk_bound_variable_kinds(
-            self.late_bound_vars_map(id.owner)
+            &self
+                .late_bound_vars_map(id.owner)
                 .and_then(|map| map.get(&id.local_id).cloned())
                 .unwrap_or_else(|| {
-                    bug!("No bound vars found for {:?} ({:?})", self.hir().node_to_string(id), id)
-                })
-                .iter(),
+                    bug!("No bound vars found for {}", self.hir().node_to_string(id))
+                }),
         )
     }
 
@@ -2351,16 +2432,24 @@ impl<'tcx> TyCtxt<'tcx> {
             })
         )
     }
+
+    pub fn local_def_id_to_hir_id(self, local_def_id: LocalDefId) -> HirId {
+        self.opt_local_def_id_to_hir_id(local_def_id).unwrap()
+    }
+
+    pub fn trait_solver_next(self) -> bool {
+        self.sess.opts.unstable_opts.trait_solver == rustc_session::config::TraitSolver::Next
+    }
 }
 
 impl<'tcx> TyCtxtAt<'tcx> {
     /// Constructs a `TyKind::Error` type and registers a `delay_span_bug` to ensure it gets used.
     #[track_caller]
-    pub fn ty_error(self) -> Ty<'tcx> {
+    pub fn ty_error_misc(self) -> Ty<'tcx> {
         self.tcx.ty_error_with_message(self.span, "TyKind::Error constructed but no error reported")
     }
 
-    /// Constructs a `TyKind::Error` type and registers a `delay_span_bug` with the given `msg to
+    /// Constructs a `TyKind::Error` type and registers a `delay_span_bug` with the given `msg` to
     /// ensure it gets used.
     #[track_caller]
     pub fn ty_error_with_message(self, msg: &str) -> Ty<'tcx> {
@@ -2370,7 +2459,7 @@ impl<'tcx> TyCtxtAt<'tcx> {
     pub fn mk_trait_ref(
         self,
         trait_lang_item: LangItem,
-        substs: impl IntoIterator<Item = impl Into<ty::GenericArg<'tcx>>>,
+        substs: impl IntoIterator<Item: Into<ty::GenericArg<'tcx>>>,
     ) -> ty::TraitRef<'tcx> {
         let trait_def_id = self.require_lang_item(trait_lang_item, Some(self.span));
         self.tcx.mk_trait_ref(trait_def_id, substs)
@@ -2392,19 +2481,11 @@ pub struct DeducedParamAttrs {
     pub read_only: bool,
 }
 
-// We are comparing types with different invariant lifetimes, so `ptr::eq`
-// won't work for us.
-fn ptr_eq<T, U>(t: *const T, u: *const U) -> bool {
-    t as *const () == u as *const ()
-}
-
 pub fn provide(providers: &mut ty::query::Providers) {
     providers.module_reexports =
         |tcx, id| tcx.resolutions(()).reexport_map.get(&id).map(|v| &v[..]);
     providers.maybe_unused_trait_imports =
         |tcx, ()| &tcx.resolutions(()).maybe_unused_trait_imports;
-    providers.maybe_unused_extern_crates =
-        |tcx, ()| &tcx.resolutions(()).maybe_unused_extern_crates[..];
     providers.names_imported_by_glob_use = |tcx, id| {
         tcx.arena.alloc(tcx.resolutions(()).glob_map.get(&id).cloned().unwrap_or_default())
     };
@@ -2424,6 +2505,5 @@ pub fn provide(providers: &mut ty::query::Providers) {
         // We want to check if the panic handler was defined in this crate
         tcx.lang_items().panic_impl().map_or(false, |did| did.is_local())
     };
-    providers.source_span =
-        |tcx, def_id| tcx.untracked.source_span.get(def_id).copied().unwrap_or(DUMMY_SP);
+    providers.source_span = |tcx, def_id| tcx.untracked.source_span.get(def_id).unwrap_or(DUMMY_SP);
 }
diff --git a/compiler/rustc_middle/src/ty/context/tls.rs b/compiler/rustc_middle/src/ty/context/tls.rs
new file mode 100644
index 000000000..5426ac8d7
--- /dev/null
+++ b/compiler/rustc_middle/src/ty/context/tls.rs
@@ -0,0 +1,186 @@
+use super::{GlobalCtxt, TyCtxt};
+
+use crate::dep_graph::TaskDepsRef;
+use crate::ty::query;
+use rustc_data_structures::sync::{self, Lock};
+use rustc_errors::Diagnostic;
+use std::mem;
+use std::ptr;
+use thin_vec::ThinVec;
+
+/// This is the implicit state of rustc. It contains the current
+/// `TyCtxt` and query. It is updated when creating a local interner or
+/// executing a new query. Whenever there's a `TyCtxt` value available
+/// you should also have access to an `ImplicitCtxt` through the functions
+/// in this module.
+#[derive(Clone)]
+pub struct ImplicitCtxt<'a, 'tcx> {
+    /// The current `TyCtxt`.
+    pub tcx: TyCtxt<'tcx>,
+
+    /// The current query job, if any. This is updated by `JobOwner::start` in
+    /// `ty::query::plumbing` when executing a query.
+    pub query: Option<query::QueryJobId>,
+
+    /// Where to store diagnostics for the current query job, if any.
+    /// This is updated by `JobOwner::start` in `ty::query::plumbing` when executing a query.
+    pub diagnostics: Option<&'a Lock<ThinVec<Diagnostic>>>,
+
+    /// Used to prevent queries from calling too deeply.
+    pub query_depth: usize,
+
+    /// The current dep graph task. This is used to add dependencies to queries
+    /// when executing them.
+    pub task_deps: TaskDepsRef<'a>,
+}
+
+impl<'a, 'tcx> ImplicitCtxt<'a, 'tcx> {
+    pub fn new(gcx: &'tcx GlobalCtxt<'tcx>) -> Self {
+        let tcx = TyCtxt { gcx };
+        ImplicitCtxt {
+            tcx,
+            query: None,
+            diagnostics: None,
+            query_depth: 0,
+            task_deps: TaskDepsRef::Ignore,
+        }
+    }
+}
+
+#[cfg(parallel_compiler)]
+mod tlv {
+    use rustc_rayon_core as rayon_core;
+    use std::ptr;
+
+    /// Gets Rayon's thread-local variable, which is preserved for Rayon jobs.
+    /// This is used to get the pointer to the current `ImplicitCtxt`.
+    #[inline]
+    pub(super) fn get_tlv() -> *const () {
+        ptr::from_exposed_addr(rayon_core::tlv::get())
+    }
+
+    /// Sets Rayon's thread-local variable, which is preserved for Rayon jobs
+    /// to `value` during the call to `f`. It is restored to its previous value after.
+    /// This is used to set the pointer to the new `ImplicitCtxt`.
+    #[inline]
+    pub(super) fn with_tlv<F: FnOnce() -> R, R>(value: *const (), f: F) -> R {
+        rayon_core::tlv::with(value.expose_addr(), f)
+    }
+}
+
+#[cfg(not(parallel_compiler))]
+mod tlv {
+    use std::cell::Cell;
+    use std::ptr;
+
+    thread_local! {
+        /// A thread local variable that stores a pointer to the current `ImplicitCtxt`.
+        static TLV: Cell<*const ()> = const { Cell::new(ptr::null()) };
+    }
+
+    /// Gets the pointer to the current `ImplicitCtxt`.
+    #[inline]
+    pub(super) fn get_tlv() -> *const () {
+        TLV.with(|tlv| tlv.get())
+    }
+
+    /// Sets TLV to `value` during the call to `f`.
+    /// It is restored to its previous value after.
+    /// This is used to set the pointer to the new `ImplicitCtxt`.
+    #[inline]
+    pub(super) fn with_tlv<F: FnOnce() -> R, R>(value: *const (), f: F) -> R {
+        let old = TLV.replace(value);
+        let _reset = rustc_data_structures::OnDrop(move || TLV.set(old));
+        f()
+    }
+}
+
+#[inline]
+fn erase(context: &ImplicitCtxt<'_, '_>) -> *const () {
+    context as *const _ as *const ()
+}
+
+#[inline]
+unsafe fn downcast<'a, 'tcx>(context: *const ()) -> &'a ImplicitCtxt<'a, 'tcx> {
+    &*(context as *const ImplicitCtxt<'a, 'tcx>)
+}
+
+/// Sets `context` as the new current `ImplicitCtxt` for the duration of the function `f`.
+#[inline]
+pub fn enter_context<'a, 'tcx, F, R>(context: &ImplicitCtxt<'a, 'tcx>, f: F) -> R
+where
+    F: FnOnce() -> R,
+{
+    tlv::with_tlv(erase(context), f)
+}
+
+/// Allows access to the current `ImplicitCtxt` in a closure if one is available.
+#[inline]
+pub fn with_context_opt<F, R>(f: F) -> R
+where
+    F: for<'a, 'tcx> FnOnce(Option<&ImplicitCtxt<'a, 'tcx>>) -> R,
+{
+    let context = tlv::get_tlv();
+    if context.is_null() {
+        f(None)
+    } else {
+        // We could get an `ImplicitCtxt` pointer from another thread.
+        // Ensure that `ImplicitCtxt` is `Sync`.
+        sync::assert_sync::<ImplicitCtxt<'_, '_>>();
+
+        unsafe { f(Some(downcast(context))) }
+    }
+}
+
+/// Allows access to the current `ImplicitCtxt`.
+/// Panics if there is no `ImplicitCtxt` available.
+#[inline]
+pub fn with_context<F, R>(f: F) -> R
+where
+    F: for<'a, 'tcx> FnOnce(&ImplicitCtxt<'a, 'tcx>) -> R,
+{
+    with_context_opt(|opt_context| f(opt_context.expect("no ImplicitCtxt stored in tls")))
+}
+
+/// Allows access to the current `ImplicitCtxt` whose tcx field is the same as the tcx argument
+/// passed in. This means the closure is given an `ImplicitCtxt` with the same `'tcx` lifetime
+/// as the `TyCtxt` passed in.
+/// This will panic if you pass it a `TyCtxt` which is different from the current
+/// `ImplicitCtxt`'s `tcx` field.
+#[inline]
+pub fn with_related_context<'tcx, F, R>(tcx: TyCtxt<'tcx>, f: F) -> R
+where
+    F: FnOnce(&ImplicitCtxt<'_, 'tcx>) -> R,
+{
+    with_context(|context| {
+        // The two gcx have different invariant lifetimes, so we need to erase them for the comparison.
+        assert!(ptr::eq(
+            context.tcx.gcx as *const _ as *const (),
+            tcx.gcx as *const _ as *const ()
+        ));
+
+        let context: &ImplicitCtxt<'_, '_> = unsafe { mem::transmute(context) };
+
+        f(context)
+    })
+}
+
+/// Allows access to the `TyCtxt` in the current `ImplicitCtxt`.
+/// Panics if there is no `ImplicitCtxt` available.
+#[inline]
+pub fn with<F, R>(f: F) -> R
+where
+    F: for<'tcx> FnOnce(TyCtxt<'tcx>) -> R,
+{
+    with_context(|context| f(context.tcx))
+}
+
+/// Allows access to the `TyCtxt` in the current `ImplicitCtxt`.
+/// The closure is passed None if there is no `ImplicitCtxt` available.
+#[inline]
+pub fn with_opt<F, R>(f: F) -> R
+where
+    F: for<'tcx> FnOnce(Option<TyCtxt<'tcx>>) -> R,
+{
+    with_context_opt(|opt_context| f(opt_context.map(|context| context.tcx)))
+}
diff --git a/compiler/rustc_middle/src/ty/diagnostics.rs b/compiler/rustc_middle/src/ty/diagnostics.rs
index 4b4518f61..e894e1aaf 100644
--- a/compiler/rustc_middle/src/ty/diagnostics.rs
+++ b/compiler/rustc_middle/src/ty/diagnostics.rs
@@ -3,8 +3,9 @@
 use std::ops::ControlFlow;
 
 use crate::ty::{
-    visit::TypeVisitable, AliasTy, Const, ConstKind, DefIdTree, InferConst, InferTy, Opaque,
-    PolyTraitPredicate, Projection, Ty, TyCtxt, TypeSuperVisitable, TypeVisitor,
+    AliasTy, Const, ConstKind, DefIdTree, FallibleTypeFolder, InferConst, InferTy, Opaque,
+    PolyTraitPredicate, Projection, Ty, TyCtxt, TypeFoldable, TypeSuperFoldable,
+    TypeSuperVisitable, TypeVisitable, TypeVisitor,
 };
 
 use rustc_data_structures::fx::FxHashMap;
@@ -76,7 +77,7 @@ impl<'tcx> Ty<'tcx> {
     }
 }
 
-pub trait IsSuggestable<'tcx> {
+pub trait IsSuggestable<'tcx>: Sized {
     /// Whether this makes sense to suggest in a diagnostic.
     ///
     /// We filter out certain types and constants since they don't provide
@@ -87,15 +88,21 @@ pub trait IsSuggestable<'tcx> {
     /// Only if `infer_suggestable` is true, we consider type and const
     /// inference variables to be suggestable.
     fn is_suggestable(self, tcx: TyCtxt<'tcx>, infer_suggestable: bool) -> bool;
+
+    fn make_suggestable(self, tcx: TyCtxt<'tcx>, infer_suggestable: bool) -> Option<Self>;
 }
 
 impl<'tcx, T> IsSuggestable<'tcx> for T
 where
-    T: TypeVisitable<'tcx>,
+    T: TypeVisitable<TyCtxt<'tcx>> + TypeFoldable<TyCtxt<'tcx>>,
 {
     fn is_suggestable(self, tcx: TyCtxt<'tcx>, infer_suggestable: bool) -> bool {
         self.visit_with(&mut IsSuggestableVisitor { tcx, infer_suggestable }).is_continue()
     }
+
+    fn make_suggestable(self, tcx: TyCtxt<'tcx>, infer_suggestable: bool) -> Option<T> {
+        self.try_fold_with(&mut MakeSuggestableFolder { tcx, infer_suggestable }).ok()
+    }
 }
 
 pub fn suggest_arbitrary_trait_bound<'tcx>(
@@ -186,6 +193,9 @@ fn suggest_removing_unsized_bound(
 }
 
 /// Suggest restricting a type param with a new bound.
+///
+/// If `span_to_replace` is provided, then that span will be replaced with the
+/// `constraint`. If one wasn't provided, then the full bound will be suggested.
 pub fn suggest_constraining_type_param(
     tcx: TyCtxt<'_>,
     generics: &hir::Generics<'_>,
@@ -193,12 +203,14 @@ pub fn suggest_constraining_type_param(
     param_name: &str,
     constraint: &str,
     def_id: Option<DefId>,
+    span_to_replace: Option<Span>,
 ) -> bool {
     suggest_constraining_type_params(
         tcx,
         generics,
         err,
         [(param_name, constraint, def_id)].into_iter(),
+        span_to_replace,
     )
 }
 
@@ -208,6 +220,7 @@ pub fn suggest_constraining_type_params<'a>(
     generics: &hir::Generics<'_>,
     err: &mut Diagnostic,
     param_names_and_constraints: impl Iterator<Item = (&'a str, &'a str, Option<DefId>)>,
+    span_to_replace: Option<Span>,
 ) -> bool {
     let mut grouped = FxHashMap::default();
     param_names_and_constraints.for_each(|(param_name, constraint, def_id)| {
@@ -246,7 +259,9 @@ pub fn suggest_constraining_type_params<'a>(
         let mut suggest_restrict = |span, bound_list_non_empty| {
             suggestions.push((
                 span,
-                if bound_list_non_empty {
+                if span_to_replace.is_some() {
+                    constraint.clone()
+                } else if bound_list_non_empty {
                     format!(" + {}", constraint)
                 } else {
                     format!(" {}", constraint)
@@ -255,6 +270,11 @@ pub fn suggest_constraining_type_params<'a>(
             ))
         };
 
+        if let Some(span) = span_to_replace {
+            suggest_restrict(span, true);
+            continue;
+        }
+
         // When the type parameter has been provided bounds
         //
         //    Message:
@@ -440,7 +460,7 @@ pub struct IsSuggestableVisitor<'tcx> {
     infer_suggestable: bool,
 }
 
-impl<'tcx> TypeVisitor<'tcx> for IsSuggestableVisitor<'tcx> {
+impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for IsSuggestableVisitor<'tcx> {
     type BreakTy = ();
 
     fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
@@ -460,8 +480,9 @@ impl<'tcx> TypeVisitor<'tcx> for IsSuggestableVisitor<'tcx> {
 
             Alias(Opaque, AliasTy { def_id, .. }) => {
                 let parent = self.tcx.parent(def_id);
+                let parent_ty = self.tcx.type_of(parent).subst_identity();
                 if let DefKind::TyAlias | DefKind::AssocTy = self.tcx.def_kind(parent)
-                    && let Alias(Opaque, AliasTy { def_id: parent_opaque_def_id, .. }) = *self.tcx.type_of(parent).kind()
+                    && let Alias(Opaque, AliasTy { def_id: parent_opaque_def_id, .. }) = *parent_ty.kind()
                     && parent_opaque_def_id == def_id
                 {
                     // Okay
@@ -509,3 +530,92 @@ impl<'tcx> TypeVisitor<'tcx> for IsSuggestableVisitor<'tcx> {
         c.super_visit_with(self)
     }
 }
+
+pub struct MakeSuggestableFolder<'tcx> {
+    tcx: TyCtxt<'tcx>,
+    infer_suggestable: bool,
+}
+
+impl<'tcx> FallibleTypeFolder<TyCtxt<'tcx>> for MakeSuggestableFolder<'tcx> {
+    type Error = ();
+
+    fn interner(&self) -> TyCtxt<'tcx> {
+        self.tcx
+    }
+
+    fn try_fold_ty(&mut self, t: Ty<'tcx>) -> Result<Ty<'tcx>, Self::Error> {
+        let t = match *t.kind() {
+            Infer(InferTy::TyVar(_)) if self.infer_suggestable => t,
+
+            FnDef(def_id, substs) => {
+                self.tcx.mk_fn_ptr(self.tcx.fn_sig(def_id).subst(self.tcx, substs))
+            }
+
+            // FIXME(compiler-errors): We could replace these with infer, I guess.
+            Closure(..)
+            | Infer(..)
+            | Generator(..)
+            | GeneratorWitness(..)
+            | Bound(_, _)
+            | Placeholder(_)
+            | Error(_) => {
+                return Err(());
+            }
+
+            Alias(Opaque, AliasTy { def_id, .. }) => {
+                let parent = self.tcx.parent(def_id);
+                let parent_ty = self.tcx.type_of(parent).subst_identity();
+                if let hir::def::DefKind::TyAlias | hir::def::DefKind::AssocTy = self.tcx.def_kind(parent)
+                    && let Alias(Opaque, AliasTy { def_id: parent_opaque_def_id, .. }) = *parent_ty.kind()
+                    && parent_opaque_def_id == def_id
+                {
+                    t
+                } else {
+                    return Err(());
+                }
+            }
+
+            Param(param) => {
+                // FIXME: It would be nice to make this not use string manipulation,
+                // but it's pretty hard to do this, since `ty::ParamTy` is missing
+                // sufficient info to determine if it is synthetic, and we don't
+                // always have a convenient way of getting `ty::Generics` at the call
+                // sites we invoke `IsSuggestable::is_suggestable`.
+                if param.name.as_str().starts_with("impl ") {
+                    return Err(());
+                }
+
+                t
+            }
+
+            _ => t,
+        };
+
+        t.try_super_fold_with(self)
+    }
+
+    fn try_fold_const(&mut self, c: Const<'tcx>) -> Result<Const<'tcx>, ()> {
+        let c = match c.kind() {
+            ConstKind::Infer(InferConst::Var(_)) if self.infer_suggestable => c,
+
+            ConstKind::Infer(..)
+            | ConstKind::Bound(..)
+            | ConstKind::Placeholder(..)
+            | ConstKind::Error(..) => {
+                return Err(());
+            }
+
+            _ => c,
+        };
+
+        c.try_super_fold_with(self)
+    }
+}
+
+#[derive(Diagnostic)]
+#[diag(middle_const_not_used_in_type_alias)]
+pub(super) struct ConstNotUsedTraitAlias {
+    pub ct: String,
+    #[primary_span]
+    pub span: Span,
+}
diff --git a/compiler/rustc_middle/src/ty/erase_regions.rs b/compiler/rustc_middle/src/ty/erase_regions.rs
index 9e4f90caa..383773248 100644
--- a/compiler/rustc_middle/src/ty/erase_regions.rs
+++ b/compiler/rustc_middle/src/ty/erase_regions.rs
@@ -1,6 +1,5 @@
 use crate::ty::fold::{TypeFoldable, TypeFolder, TypeSuperFoldable};
-use crate::ty::visit::TypeVisitable;
-use crate::ty::{self, Ty, TyCtxt, TypeFlags};
+use crate::ty::{self, Ty, TyCtxt, TypeFlags, TypeVisitableExt};
 
 pub(super) fn provide(providers: &mut ty::query::Providers) {
     *providers = ty::query::Providers { erase_regions_ty, ..*providers };
@@ -18,7 +17,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// subtyping, but they are anonymized and normalized as well)..
     pub fn erase_regions<T>(self, value: T) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         // If there's nothing to erase avoid performing the query at all
         if !value.has_type_flags(TypeFlags::HAS_LATE_BOUND | TypeFlags::HAS_FREE_REGIONS) {
@@ -35,8 +34,8 @@ struct RegionEraserVisitor<'tcx> {
     tcx: TyCtxt<'tcx>,
 }
 
-impl<'tcx> TypeFolder<'tcx> for RegionEraserVisitor<'tcx> {
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for RegionEraserVisitor<'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
@@ -46,7 +45,7 @@ impl<'tcx> TypeFolder<'tcx> for RegionEraserVisitor<'tcx> {
 
     fn fold_binder<T>(&mut self, t: ty::Binder<'tcx, T>) -> ty::Binder<'tcx, T>
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         let u = self.tcx.anonymize_bound_vars(t);
         u.super_fold_with(self)
diff --git a/compiler/rustc_middle/src/ty/error.rs b/compiler/rustc_middle/src/ty/error.rs
index 5d394f71f..9c171a69d 100644
--- a/compiler/rustc_middle/src/ty/error.rs
+++ b/compiler/rustc_middle/src/ty/error.rs
@@ -1,24 +1,16 @@
-use crate::traits::{ObligationCause, ObligationCauseCode};
-use crate::ty::diagnostics::suggest_constraining_type_param;
-use crate::ty::print::{with_forced_trimmed_paths, FmtPrinter, Printer};
+use crate::ty::print::{with_forced_trimmed_paths, FmtPrinter, PrettyPrinter};
 use crate::ty::{self, BoundRegionKind, Region, Ty, TyCtxt};
-use rustc_errors::Applicability::{MachineApplicable, MaybeIncorrect};
-use rustc_errors::{pluralize, Diagnostic, MultiSpan};
+use rustc_errors::pluralize;
 use rustc_hir as hir;
 use rustc_hir::def::{CtorOf, DefKind};
 use rustc_hir::def_id::DefId;
-use rustc_span::symbol::{sym, Symbol};
-use rustc_span::{BytePos, Span};
+use rustc_span::symbol::Symbol;
 use rustc_target::spec::abi;
-
 use std::borrow::Cow;
 use std::collections::hash_map::DefaultHasher;
-use std::fmt;
 use std::hash::{Hash, Hasher};
 use std::path::PathBuf;
 
-use super::print::PrettyPrinter;
-
 #[derive(Clone, Copy, Debug, PartialEq, Eq, TypeFoldable, TypeVisitable, Lift)]
 pub struct ExpectedFound<T> {
     pub expected: T,
@@ -36,7 +28,7 @@ impl<T> ExpectedFound<T> {
 }
 
 // Data structures used in type unification
-#[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable, Lift)]
+#[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable, Lift, PartialEq, Eq)]
 #[rustc_pass_by_value]
 pub enum TypeError<'tcx> {
     Mismatch,
@@ -93,20 +85,16 @@ impl TypeError<'_> {
 /// in parentheses after some larger message. You should also invoke `note_and_explain_type_err()`
 /// afterwards to present additional details, particularly when it comes to lifetime-related
 /// errors.
-impl<'tcx> fmt::Display for TypeError<'tcx> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+impl<'tcx> TypeError<'tcx> {
+    pub fn to_string(self, tcx: TyCtxt<'tcx>) -> Cow<'static, str> {
         use self::TypeError::*;
-        fn report_maybe_different(
-            f: &mut fmt::Formatter<'_>,
-            expected: &str,
-            found: &str,
-        ) -> fmt::Result {
+        fn report_maybe_different(expected: &str, found: &str) -> String {
             // A naive approach to making sure that we're not reporting silly errors such as:
             // (expected closure, found closure).
             if expected == found {
-                write!(f, "expected {}, found a different {}", expected, found)
+                format!("expected {}, found a different {}", expected, found)
             } else {
-                write!(f, "expected {}, found {}", expected, found)
+                format!("expected {}, found {}", expected, found)
             }
         }
 
@@ -115,64 +103,63 @@ impl<'tcx> fmt::Display for TypeError<'tcx> {
             _ => String::new(),
         };
 
-        match *self {
-            CyclicTy(_) => write!(f, "cyclic type of infinite size"),
-            CyclicConst(_) => write!(f, "encountered a self-referencing constant"),
-            Mismatch => write!(f, "types differ"),
+        match self {
+            CyclicTy(_) => "cyclic type of infinite size".into(),
+            CyclicConst(_) => "encountered a self-referencing constant".into(),
+            Mismatch => "types differ".into(),
             ConstnessMismatch(values) => {
-                write!(f, "expected {} bound, found {} bound", values.expected, values.found)
+                format!("expected {} bound, found {} bound", values.expected, values.found).into()
             }
             PolarityMismatch(values) => {
-                write!(f, "expected {} polarity, found {} polarity", values.expected, values.found)
+                format!("expected {} polarity, found {} polarity", values.expected, values.found)
+                    .into()
             }
             UnsafetyMismatch(values) => {
-                write!(f, "expected {} fn, found {} fn", values.expected, values.found)
+                format!("expected {} fn, found {} fn", values.expected, values.found).into()
             }
             AbiMismatch(values) => {
-                write!(f, "expected {} fn, found {} fn", values.expected, values.found)
+                format!("expected {} fn, found {} fn", values.expected, values.found).into()
             }
-            ArgumentMutability(_) | Mutability => write!(f, "types differ in mutability"),
-            TupleSize(values) => write!(
-                f,
+            ArgumentMutability(_) | Mutability => "types differ in mutability".into(),
+            TupleSize(values) => format!(
                 "expected a tuple with {} element{}, found one with {} element{}",
                 values.expected,
                 pluralize!(values.expected),
                 values.found,
                 pluralize!(values.found)
-            ),
-            FixedArraySize(values) => write!(
-                f,
+            )
+            .into(),
+            FixedArraySize(values) => format!(
                 "expected an array with a fixed size of {} element{}, found one with {} element{}",
                 values.expected,
                 pluralize!(values.expected),
                 values.found,
                 pluralize!(values.found)
-            ),
-            ArgCount => write!(f, "incorrect number of function parameters"),
-            FieldMisMatch(adt, field) => write!(f, "field type mismatch: {}.{}", adt, field),
-            RegionsDoesNotOutlive(..) => write!(f, "lifetime mismatch"),
+            )
+            .into(),
+            ArgCount => "incorrect number of function parameters".into(),
+            FieldMisMatch(adt, field) => format!("field type mismatch: {}.{}", adt, field).into(),
+            RegionsDoesNotOutlive(..) => "lifetime mismatch".into(),
             // Actually naming the region here is a bit confusing because context is lacking
             RegionsInsufficientlyPolymorphic(..) => {
-                write!(f, "one type is more general than the other")
+                "one type is more general than the other".into()
             }
-            RegionsOverlyPolymorphic(br, _) => write!(
-                f,
+            RegionsOverlyPolymorphic(br, _) => format!(
                 "expected concrete lifetime, found bound lifetime parameter{}",
                 br_string(br)
-            ),
-            RegionsPlaceholderMismatch => write!(f, "one type is more general than the other"),
-            ArgumentSorts(values, _) | Sorts(values) => ty::tls::with(|tcx| {
-                let (mut expected, mut found) = with_forced_trimmed_paths!((
-                    values.expected.sort_string(tcx),
-                    values.found.sort_string(tcx),
-                ));
+            )
+            .into(),
+            RegionsPlaceholderMismatch => "one type is more general than the other".into(),
+            ArgumentSorts(values, _) | Sorts(values) => {
+                let mut expected = values.expected.sort_string(tcx);
+                let mut found = values.found.sort_string(tcx);
                 if expected == found {
                     expected = values.expected.sort_string(tcx);
                     found = values.found.sort_string(tcx);
                 }
-                report_maybe_different(f, &expected, &found)
-            }),
-            Traits(values) => ty::tls::with(|tcx| {
+                report_maybe_different(&expected, &found).into()
+            }
+            Traits(values) => {
                 let (mut expected, mut found) = with_forced_trimmed_paths!((
                     tcx.def_path_str(values.expected),
                     tcx.def_path_str(values.found),
@@ -181,12 +168,9 @@ impl<'tcx> fmt::Display for TypeError<'tcx> {
                     expected = tcx.def_path_str(values.expected);
                     found = tcx.def_path_str(values.found);
                 }
-                report_maybe_different(
-                    f,
-                    &format!("trait `{expected}`"),
-                    &format!("trait `{found}`"),
-                )
-            }),
+                report_maybe_different(&format!("trait `{expected}`"), &format!("trait `{found}`"))
+                    .into()
+            }
             IntMismatch(ref values) => {
                 let expected = match values.expected {
                     ty::IntVarValue::IntType(ty) => ty.name_str(),
@@ -196,43 +180,38 @@ impl<'tcx> fmt::Display for TypeError<'tcx> {
                     ty::IntVarValue::IntType(ty) => ty.name_str(),
                     ty::IntVarValue::UintType(ty) => ty.name_str(),
                 };
-                write!(f, "expected `{}`, found `{}`", expected, found)
+                format!("expected `{}`, found `{}`", expected, found).into()
             }
-            FloatMismatch(ref values) => {
-                write!(
-                    f,
-                    "expected `{}`, found `{}`",
-                    values.expected.name_str(),
-                    values.found.name_str()
-                )
-            }
-            VariadicMismatch(ref values) => write!(
-                f,
+            FloatMismatch(ref values) => format!(
+                "expected `{}`, found `{}`",
+                values.expected.name_str(),
+                values.found.name_str()
+            )
+            .into(),
+            VariadicMismatch(ref values) => format!(
                 "expected {} fn, found {} function",
                 if values.expected { "variadic" } else { "non-variadic" },
                 if values.found { "variadic" } else { "non-variadic" }
-            ),
-            ProjectionMismatched(ref values) => ty::tls::with(|tcx| {
-                write!(
-                    f,
-                    "expected {}, found {}",
-                    tcx.def_path_str(values.expected),
-                    tcx.def_path_str(values.found)
-                )
-            }),
+            )
+            .into(),
+            ProjectionMismatched(ref values) => format!(
+                "expected `{}`, found `{}`",
+                tcx.def_path_str(values.expected),
+                tcx.def_path_str(values.found)
+            )
+            .into(),
             ExistentialMismatch(ref values) => report_maybe_different(
-                f,
                 &format!("trait `{}`", values.expected),
                 &format!("trait `{}`", values.found),
-            ),
+            )
+            .into(),
             ConstMismatch(ref values) => {
-                write!(f, "expected `{}`, found `{}`", values.expected, values.found)
+                format!("expected `{}`, found `{}`", values.expected, values.found).into()
+            }
+            IntrinsicCast => "cannot coerce intrinsics to function pointers".into(),
+            TargetFeatureCast(_) => {
+                "cannot coerce functions with `#[target_feature]` to safe function pointers".into()
             }
-            IntrinsicCast => write!(f, "cannot coerce intrinsics to function pointers"),
-            TargetFeatureCast(_) => write!(
-                f,
-                "cannot coerce functions with `#[target_feature]` to safe function pointers"
-            ),
         }
     }
 }
@@ -265,60 +244,9 @@ impl<'tcx> TypeError<'tcx> {
 }
 
 impl<'tcx> Ty<'tcx> {
-    pub fn sort_string(self, tcx: TyCtxt<'_>) -> Cow<'static, str> {
+    pub fn sort_string(self, tcx: TyCtxt<'tcx>) -> Cow<'static, str> {
         match *self.kind() {
-            ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::Str | ty::Never => {
-                format!("`{}`", self).into()
-            }
-            ty::Tuple(ref tys) if tys.is_empty() => format!("`{}`", self).into(),
-
-            ty::Adt(def, _) => format!("{} `{}`", def.descr(), tcx.def_path_str(def.did())).into(),
             ty::Foreign(def_id) => format!("extern type `{}`", tcx.def_path_str(def_id)).into(),
-            ty::Array(t, n) => {
-                if t.is_simple_ty() {
-                    return format!("array `{}`", self).into();
-                }
-
-                let n = tcx.lift(n).unwrap();
-                if let ty::ConstKind::Value(v) = n.kind() {
-                    if let Some(n) = v.try_to_machine_usize(tcx) {
-                        return format!("array of {} element{}", n, pluralize!(n)).into();
-                    }
-                }
-                "array".into()
-            }
-            ty::Slice(ty) if ty.is_simple_ty() => format!("slice `{}`", self).into(),
-            ty::Slice(_) => "slice".into(),
-            ty::RawPtr(tymut) => {
-                let tymut_string = match tymut.mutbl {
-                    hir::Mutability::Mut => tymut.to_string(),
-                    hir::Mutability::Not => format!("const {}", tymut.ty),
-                };
-
-                if tymut_string != "_" && (tymut.ty.is_simple_text() || tymut_string.len() < "const raw pointer".len()) {
-                    format!("`*{}`", tymut_string).into()
-                } else {
-                    // Unknown type name, it's long or has type arguments
-                    "raw pointer".into()
-                }
-            },
-            ty::Ref(_, ty, mutbl) => {
-                let tymut = ty::TypeAndMut { ty, mutbl };
-                let tymut_string = tymut.to_string();
-
-                if tymut_string != "_"
-                    && (ty.is_simple_text() || tymut_string.len() < "mutable reference".len())
-                {
-                    format!("`&{}`", tymut_string).into()
-                } else {
-                    // Unknown type name, it's long or has type arguments
-                    match mutbl {
-                        hir::Mutability::Mut => "mutable reference",
-                        _ => "reference",
-                    }
-                    .into()
-                }
-            }
             ty::FnDef(def_id, ..) => match tcx.def_kind(def_id) {
                 DefKind::Ctor(CtorOf::Struct, _) => "struct constructor".into(),
                 DefKind::Ctor(CtorOf::Variant, _) => "enum constructor".into(),
@@ -326,13 +254,13 @@ impl<'tcx> Ty<'tcx> {
             },
             ty::FnPtr(_) => "fn pointer".into(),
             ty::Dynamic(ref inner, ..) if let Some(principal) = inner.principal() => {
-                format!("trait object `dyn {}`", tcx.def_path_str(principal.def_id())).into()
+                format!("`dyn {}`", tcx.def_path_str(principal.def_id())).into()
             }
             ty::Dynamic(..) => "trait object".into(),
             ty::Closure(..) => "closure".into(),
             ty::Generator(def_id, ..) => tcx.generator_kind(def_id).unwrap().descr().into(),
-            ty::GeneratorWitness(..) => "generator witness".into(),
-            ty::Tuple(..) => "tuple".into(),
+            ty::GeneratorWitness(..) |
+            ty::GeneratorWitnessMIR(..) => "generator witness".into(),
             ty::Infer(ty::TyVar(_)) => "inferred type".into(),
             ty::Infer(ty::IntVar(_)) => "integer".into(),
             ty::Infer(ty::FloatVar(_)) => "floating-point number".into(),
@@ -342,9 +270,14 @@ impl<'tcx> Ty<'tcx> {
             ty::Infer(ty::FreshIntTy(_)) => "fresh integral type".into(),
             ty::Infer(ty::FreshFloatTy(_)) => "fresh floating-point type".into(),
             ty::Alias(ty::Projection, _) => "associated type".into(),
-            ty::Param(p) => format!("type parameter `{}`", p).into(),
-            ty::Alias(ty::Opaque, ..) => "opaque type".into(),
+            ty::Param(p) => format!("type parameter `{p}`").into(),
+            ty::Alias(ty::Opaque, ..) => if tcx.ty_is_opaque_future(self) { "future".into() } else { "opaque type".into() },
             ty::Error(_) => "type error".into(),
+            _ => {
+                let width = tcx.sess.diagnostic_width();
+                let length_limit = std::cmp::max(width / 4, 15);
+                format!("`{}`", tcx.ty_string_with_limit(self, length_limit)).into()
+            }
         }
     }
 
@@ -379,7 +312,7 @@ impl<'tcx> Ty<'tcx> {
             ty::Dynamic(..) => "trait object".into(),
             ty::Closure(..) => "closure".into(),
             ty::Generator(def_id, ..) => tcx.generator_kind(def_id).unwrap().descr().into(),
-            ty::GeneratorWitness(..) => "generator witness".into(),
+            ty::GeneratorWitness(..) | ty::GeneratorWitnessMIR(..) => "generator witness".into(),
             ty::Tuple(..) => "tuple".into(),
             ty::Placeholder(..) => "higher-ranked type".into(),
             ty::Bound(..) => "bound type variable".into(),
@@ -391,630 +324,14 @@ impl<'tcx> Ty<'tcx> {
 }
 
 impl<'tcx> TyCtxt<'tcx> {
-    pub fn note_and_explain_type_err(
-        self,
-        diag: &mut Diagnostic,
-        err: TypeError<'tcx>,
-        cause: &ObligationCause<'tcx>,
-        sp: Span,
-        body_owner_def_id: DefId,
-    ) {
-        use self::TypeError::*;
-        debug!("note_and_explain_type_err err={:?} cause={:?}", err, cause);
-        match err {
-            ArgumentSorts(values, _) | Sorts(values) => {
-                match (values.expected.kind(), values.found.kind()) {
-                    (ty::Closure(..), ty::Closure(..)) => {
-                        diag.note("no two closures, even if identical, have the same type");
-                        diag.help("consider boxing your closure and/or using it as a trait object");
-                    }
-                    (ty::Alias(ty::Opaque, ..), ty::Alias(ty::Opaque, ..)) => {
-                        // Issue #63167
-                        diag.note("distinct uses of `impl Trait` result in different opaque types");
-                    }
-                    (ty::Float(_), ty::Infer(ty::IntVar(_)))
-                        if let Ok(
-                            // Issue #53280
-                            snippet,
-                        ) = self.sess.source_map().span_to_snippet(sp) =>
-                    {
-                        if snippet.chars().all(|c| c.is_digit(10) || c == '-' || c == '_') {
-                            diag.span_suggestion(
-                                sp,
-                                "use a float literal",
-                                format!("{}.0", snippet),
-                                MachineApplicable,
-                            );
-                        }
-                    }
-                    (ty::Param(expected), ty::Param(found)) => {
-                        let generics = self.generics_of(body_owner_def_id);
-                        let e_span = self.def_span(generics.type_param(expected, self).def_id);
-                        if !sp.contains(e_span) {
-                            diag.span_label(e_span, "expected type parameter");
-                        }
-                        let f_span = self.def_span(generics.type_param(found, self).def_id);
-                        if !sp.contains(f_span) {
-                            diag.span_label(f_span, "found type parameter");
-                        }
-                        diag.note(
-                            "a type parameter was expected, but a different one was found; \
-                             you might be missing a type parameter or trait bound",
-                        );
-                        diag.note(
-                            "for more information, visit \
-                             https://doc.rust-lang.org/book/ch10-02-traits.html\
-                             #traits-as-parameters",
-                        );
-                    }
-                    (ty::Alias(ty::Projection, _), ty::Alias(ty::Projection, _)) => {
-                        diag.note("an associated type was expected, but a different one was found");
-                    }
-                    (ty::Param(p), ty::Alias(ty::Projection, proj)) | (ty::Alias(ty::Projection, proj), ty::Param(p))
-                        if self.def_kind(proj.def_id) != DefKind::ImplTraitPlaceholder =>
-                    {
-                        let generics = self.generics_of(body_owner_def_id);
-                        let p_span = self.def_span(generics.type_param(p, self).def_id);
-                        if !sp.contains(p_span) {
-                            diag.span_label(p_span, "this type parameter");
-                        }
-                        let hir = self.hir();
-                        let mut note = true;
-                        if let Some(generics) = generics
-                            .type_param(p, self)
-                            .def_id
-                            .as_local()
-                            .map(|id| hir.local_def_id_to_hir_id(id))
-                            .and_then(|id| self.hir().find_parent(id))
-                            .as_ref()
-                            .and_then(|node| node.generics())
-                        {
-                            // Synthesize the associated type restriction `Add<Output = Expected>`.
-                            // FIXME: extract this logic for use in other diagnostics.
-                            let (trait_ref, assoc_substs) = proj.trait_ref_and_own_substs(self);
-                            let path =
-                                self.def_path_str_with_substs(trait_ref.def_id, trait_ref.substs);
-                            let item_name = self.item_name(proj.def_id);
-                            let item_args = self.format_generic_args(assoc_substs);
-
-                            let path = if path.ends_with('>') {
-                                format!(
-                                    "{}, {}{} = {}>",
-                                    &path[..path.len() - 1],
-                                    item_name,
-                                    item_args,
-                                    p
-                                )
-                            } else {
-                                format!("{}<{}{} = {}>", path, item_name, item_args, p)
-                            };
-                            note = !suggest_constraining_type_param(
-                                self,
-                                generics,
-                                diag,
-                                &format!("{}", proj.self_ty()),
-                                &path,
-                                None,
-                            );
-                        }
-                        if note {
-                            diag.note("you might be missing a type parameter or trait bound");
-                        }
-                    }
-                    (ty::Param(p), ty::Dynamic(..) | ty::Alias(ty::Opaque, ..))
-                    | (ty::Dynamic(..) | ty::Alias(ty::Opaque, ..), ty::Param(p)) => {
-                        let generics = self.generics_of(body_owner_def_id);
-                        let p_span = self.def_span(generics.type_param(p, self).def_id);
-                        if !sp.contains(p_span) {
-                            diag.span_label(p_span, "this type parameter");
-                        }
-                        diag.help("type parameters must be constrained to match other types");
-                        if self.sess.teach(&diag.get_code().unwrap()) {
-                            diag.help(
-                                "given a type parameter `T` and a method `foo`:
-```
-trait Trait<T> { fn foo(&self) -> T; }
-```
-the only ways to implement method `foo` are:
-- constrain `T` with an explicit type:
-```
-impl Trait<String> for X {
-    fn foo(&self) -> String { String::new() }
-}
-```
-- add a trait bound to `T` and call a method on that trait that returns `Self`:
-```
-impl<T: std::default::Default> Trait<T> for X {
-    fn foo(&self) -> T { <T as std::default::Default>::default() }
-}
-```
-- change `foo` to return an argument of type `T`:
-```
-impl<T> Trait<T> for X {
-    fn foo(&self, x: T) -> T { x }
-}
-```",
-                            );
-                        }
-                        diag.note(
-                            "for more information, visit \
-                             https://doc.rust-lang.org/book/ch10-02-traits.html\
-                             #traits-as-parameters",
-                        );
-                    }
-                    (ty::Param(p), ty::Closure(..) | ty::Generator(..)) => {
-                        let generics = self.generics_of(body_owner_def_id);
-                        let p_span = self.def_span(generics.type_param(p, self).def_id);
-                        if !sp.contains(p_span) {
-                            diag.span_label(p_span, "this type parameter");
-                        }
-                        diag.help(&format!(
-                            "every closure has a distinct type and so could not always match the \
-                             caller-chosen type of parameter `{}`",
-                            p
-                        ));
-                    }
-                    (ty::Param(p), _) | (_, ty::Param(p)) => {
-                        let generics = self.generics_of(body_owner_def_id);
-                        let p_span = self.def_span(generics.type_param(p, self).def_id);
-                        if !sp.contains(p_span) {
-                            diag.span_label(p_span, "this type parameter");
-                        }
-                    }
-                    (ty::Alias(ty::Projection, proj_ty), _) if self.def_kind(proj_ty.def_id) != DefKind::ImplTraitPlaceholder => {
-                        self.expected_projection(
-                            diag,
-                            proj_ty,
-                            values,
-                            body_owner_def_id,
-                            cause.code(),
-                        );
-                    }
-                    (_, ty::Alias(ty::Projection, proj_ty)) if self.def_kind(proj_ty.def_id) != DefKind::ImplTraitPlaceholder => {
-                        let msg = format!(
-                            "consider constraining the associated type `{}` to `{}`",
-                            values.found, values.expected,
-                        );
-                        if !(self.suggest_constraining_opaque_associated_type(
-                            diag,
-                            &msg,
-                            proj_ty,
-                            values.expected,
-                        ) || self.suggest_constraint(
-                            diag,
-                            &msg,
-                            body_owner_def_id,
-                            proj_ty,
-                            values.expected,
-                        )) {
-                            diag.help(&msg);
-                            diag.note(
-                                "for more information, visit \
-                                https://doc.rust-lang.org/book/ch19-03-advanced-traits.html",
-                            );
-                        }
-                    }
-                    _ => {}
-                }
-                debug!(
-                    "note_and_explain_type_err expected={:?} ({:?}) found={:?} ({:?})",
-                    values.expected,
-                    values.expected.kind(),
-                    values.found,
-                    values.found.kind(),
-                );
-            }
-            CyclicTy(ty) => {
-                // Watch out for various cases of cyclic types and try to explain.
-                if ty.is_closure() || ty.is_generator() {
-                    diag.note(
-                        "closures cannot capture themselves or take themselves as argument;\n\
-                         this error may be the result of a recent compiler bug-fix,\n\
-                         see issue #46062 <https://github.com/rust-lang/rust/issues/46062>\n\
-                         for more information",
-                    );
-                }
-            }
-            TargetFeatureCast(def_id) => {
-                let target_spans =
-                    self.get_attrs(def_id, sym::target_feature).map(|attr| attr.span);
-                diag.note(
-                    "functions with `#[target_feature]` can only be coerced to `unsafe` function pointers"
-                );
-                diag.span_labels(target_spans, "`#[target_feature]` added here");
-            }
-            _ => {}
-        }
-    }
-
-    fn suggest_constraint(
-        self,
-        diag: &mut Diagnostic,
-        msg: &str,
-        body_owner_def_id: DefId,
-        proj_ty: &ty::AliasTy<'tcx>,
-        ty: Ty<'tcx>,
-    ) -> bool {
-        let assoc = self.associated_item(proj_ty.def_id);
-        let (trait_ref, assoc_substs) = proj_ty.trait_ref_and_own_substs(self);
-        if let Some(item) = self.hir().get_if_local(body_owner_def_id) {
-            if let Some(hir_generics) = item.generics() {
-                // Get the `DefId` for the type parameter corresponding to `A` in `<A as T>::Foo`.
-                // This will also work for `impl Trait`.
-                let def_id = if let ty::Param(param_ty) = proj_ty.self_ty().kind() {
-                    let generics = self.generics_of(body_owner_def_id);
-                    generics.type_param(param_ty, self).def_id
-                } else {
-                    return false;
-                };
-                let Some(def_id) = def_id.as_local() else {
-                    return false;
-                };
-
-                // First look in the `where` clause, as this might be
-                // `fn foo<T>(x: T) where T: Trait`.
-                for pred in hir_generics.bounds_for_param(def_id) {
-                    if self.constrain_generic_bound_associated_type_structured_suggestion(
-                        diag,
-                        &trait_ref,
-                        pred.bounds,
-                        &assoc,
-                        assoc_substs,
-                        ty,
-                        msg,
-                        false,
-                    ) {
-                        return true;
-                    }
-                }
-            }
-        }
-        false
-    }
-
-    /// An associated type was expected and a different type was found.
-    ///
-    /// We perform a few different checks to see what we can suggest:
-    ///
-    ///  - In the current item, look for associated functions that return the expected type and
-    ///    suggest calling them. (Not a structured suggestion.)
-    ///  - If any of the item's generic bounds can be constrained, we suggest constraining the
-    ///    associated type to the found type.
-    ///  - If the associated type has a default type and was expected inside of a `trait`, we
-    ///    mention that this is disallowed.
-    ///  - If all other things fail, and the error is not because of a mismatch between the `trait`
-    ///    and the `impl`, we provide a generic `help` to constrain the assoc type or call an assoc
-    ///    fn that returns the type.
-    fn expected_projection(
-        self,
-        diag: &mut Diagnostic,
-        proj_ty: &ty::AliasTy<'tcx>,
-        values: ExpectedFound<Ty<'tcx>>,
-        body_owner_def_id: DefId,
-        cause_code: &ObligationCauseCode<'_>,
-    ) {
-        let msg = format!(
-            "consider constraining the associated type `{}` to `{}`",
-            values.expected, values.found
-        );
-        let body_owner = self.hir().get_if_local(body_owner_def_id);
-        let current_method_ident = body_owner.and_then(|n| n.ident()).map(|i| i.name);
-
-        // We don't want to suggest calling an assoc fn in a scope where that isn't feasible.
-        let callable_scope = matches!(
-            body_owner,
-            Some(
-                hir::Node::Item(hir::Item { kind: hir::ItemKind::Fn(..), .. })
-                    | hir::Node::TraitItem(hir::TraitItem { kind: hir::TraitItemKind::Fn(..), .. })
-                    | hir::Node::ImplItem(hir::ImplItem { kind: hir::ImplItemKind::Fn(..), .. }),
-            )
-        );
-        let impl_comparison =
-            matches!(cause_code, ObligationCauseCode::CompareImplItemObligation { .. });
-        let assoc = self.associated_item(proj_ty.def_id);
-        if !callable_scope || impl_comparison {
-            // We do not want to suggest calling functions when the reason of the
-            // type error is a comparison of an `impl` with its `trait` or when the
-            // scope is outside of a `Body`.
-        } else {
-            // If we find a suitable associated function that returns the expected type, we don't
-            // want the more general suggestion later in this method about "consider constraining
-            // the associated type or calling a method that returns the associated type".
-            let point_at_assoc_fn = self.point_at_methods_that_satisfy_associated_type(
-                diag,
-                assoc.container_id(self),
-                current_method_ident,
-                proj_ty.def_id,
-                values.expected,
-            );
-            // Possibly suggest constraining the associated type to conform to the
-            // found type.
-            if self.suggest_constraint(diag, &msg, body_owner_def_id, proj_ty, values.found)
-                || point_at_assoc_fn
-            {
-                return;
-            }
-        }
-
-        self.suggest_constraining_opaque_associated_type(diag, &msg, proj_ty, values.found);
-
-        if self.point_at_associated_type(diag, body_owner_def_id, values.found) {
-            return;
-        }
-
-        if !impl_comparison {
-            // Generic suggestion when we can't be more specific.
-            if callable_scope {
-                diag.help(&format!(
-                    "{} or calling a method that returns `{}`",
-                    msg, values.expected
-                ));
-            } else {
-                diag.help(&msg);
-            }
-            diag.note(
-                "for more information, visit \
-                 https://doc.rust-lang.org/book/ch19-03-advanced-traits.html",
-            );
-        }
-        if self.sess.teach(&diag.get_code().unwrap()) {
-            diag.help(
-                "given an associated type `T` and a method `foo`:
-```
-trait Trait {
-type T;
-fn foo(&self) -> Self::T;
-}
-```
-the only way of implementing method `foo` is to constrain `T` with an explicit associated type:
-```
-impl Trait for X {
-type T = String;
-fn foo(&self) -> Self::T { String::new() }
-}
-```",
-            );
-        }
-    }
-
-    /// When the expected `impl Trait` is not defined in the current item, it will come from
-    /// a return type. This can occur when dealing with `TryStream` (#71035).
-    fn suggest_constraining_opaque_associated_type(
-        self,
-        diag: &mut Diagnostic,
-        msg: &str,
-        proj_ty: &ty::AliasTy<'tcx>,
-        ty: Ty<'tcx>,
-    ) -> bool {
-        let assoc = self.associated_item(proj_ty.def_id);
-        if let ty::Alias(ty::Opaque, ty::AliasTy { def_id, .. }) = *proj_ty.self_ty().kind() {
-            let opaque_local_def_id = def_id.as_local();
-            let opaque_hir_ty = if let Some(opaque_local_def_id) = opaque_local_def_id {
-                match &self.hir().expect_item(opaque_local_def_id).kind {
-                    hir::ItemKind::OpaqueTy(opaque_hir_ty) => opaque_hir_ty,
-                    _ => bug!("The HirId comes from a `ty::Opaque`"),
-                }
-            } else {
-                return false;
-            };
-
-            let (trait_ref, assoc_substs) = proj_ty.trait_ref_and_own_substs(self);
-
-            self.constrain_generic_bound_associated_type_structured_suggestion(
-                diag,
-                &trait_ref,
-                opaque_hir_ty.bounds,
-                assoc,
-                assoc_substs,
-                ty,
-                msg,
-                true,
-            )
-        } else {
-            false
-        }
-    }
-
-    fn point_at_methods_that_satisfy_associated_type(
-        self,
-        diag: &mut Diagnostic,
-        assoc_container_id: DefId,
-        current_method_ident: Option<Symbol>,
-        proj_ty_item_def_id: DefId,
-        expected: Ty<'tcx>,
-    ) -> bool {
-        let items = self.associated_items(assoc_container_id);
-        // Find all the methods in the trait that could be called to construct the
-        // expected associated type.
-        // FIXME: consider suggesting the use of associated `const`s.
-        let methods: Vec<(Span, String)> = items
-            .items
-            .iter()
-            .filter(|(name, item)| {
-                ty::AssocKind::Fn == item.kind && Some(**name) != current_method_ident
-            })
-            .filter_map(|(_, item)| {
-                let method = self.fn_sig(item.def_id);
-                match *method.output().skip_binder().kind() {
-                    ty::Alias(ty::Projection, ty::AliasTy { def_id: item_def_id, .. })
-                        if item_def_id == proj_ty_item_def_id =>
-                    {
-                        Some((
-                            self.def_span(item.def_id),
-                            format!("consider calling `{}`", self.def_path_str(item.def_id)),
-                        ))
-                    }
-                    _ => None,
-                }
-            })
-            .collect();
-        if !methods.is_empty() {
-            // Use a single `help:` to show all the methods in the trait that can
-            // be used to construct the expected associated type.
-            let mut span: MultiSpan =
-                methods.iter().map(|(sp, _)| *sp).collect::<Vec<Span>>().into();
-            let msg = format!(
-                "{some} method{s} {are} available that return{r} `{ty}`",
-                some = if methods.len() == 1 { "a" } else { "some" },
-                s = pluralize!(methods.len()),
-                are = pluralize!("is", methods.len()),
-                r = if methods.len() == 1 { "s" } else { "" },
-                ty = expected
-            );
-            for (sp, label) in methods.into_iter() {
-                span.push_span_label(sp, label);
-            }
-            diag.span_help(span, &msg);
-            return true;
-        }
-        false
-    }
-
-    fn point_at_associated_type(
-        self,
-        diag: &mut Diagnostic,
-        body_owner_def_id: DefId,
-        found: Ty<'tcx>,
-    ) -> bool {
-        let Some(hir_id) = body_owner_def_id.as_local() else {
-            return false;
-        };
-        let hir_id = self.hir().local_def_id_to_hir_id(hir_id);
-        // When `body_owner` is an `impl` or `trait` item, look in its associated types for
-        // `expected` and point at it.
-        let parent_id = self.hir().get_parent_item(hir_id);
-        let item = self.hir().find_by_def_id(parent_id.def_id);
-        debug!("expected_projection parent item {:?}", item);
-        match item {
-            Some(hir::Node::Item(hir::Item { kind: hir::ItemKind::Trait(.., items), .. })) => {
-                // FIXME: account for `#![feature(specialization)]`
-                for item in &items[..] {
-                    match item.kind {
-                        hir::AssocItemKind::Type => {
-                            // FIXME: account for returning some type in a trait fn impl that has
-                            // an assoc type as a return type (#72076).
-                            if let hir::Defaultness::Default { has_value: true } =
-                                self.impl_defaultness(item.id.owner_id)
-                            {
-                                if self.type_of(item.id.owner_id) == found {
-                                    diag.span_label(
-                                        item.span,
-                                        "associated type defaults can't be assumed inside the \
-                                            trait defining them",
-                                    );
-                                    return true;
-                                }
-                            }
-                        }
-                        _ => {}
-                    }
-                }
-            }
-            Some(hir::Node::Item(hir::Item {
-                kind: hir::ItemKind::Impl(hir::Impl { items, .. }),
-                ..
-            })) => {
-                for item in &items[..] {
-                    if let hir::AssocItemKind::Type = item.kind {
-                        if self.type_of(item.id.owner_id) == found {
-                            diag.span_label(item.span, "expected this associated type");
-                            return true;
-                        }
-                    }
-                }
-            }
-            _ => {}
-        }
-        false
-    }
-
-    /// Given a slice of `hir::GenericBound`s, if any of them corresponds to the `trait_ref`
-    /// requirement, provide a structured suggestion to constrain it to a given type `ty`.
-    ///
-    /// `is_bound_surely_present` indicates whether we know the bound we're looking for is
-    /// inside `bounds`. If that's the case then we can consider `bounds` containing only one
-    /// trait bound as the one we're looking for. This can help in cases where the associated
-    /// type is defined on a supertrait of the one present in the bounds.
-    fn constrain_generic_bound_associated_type_structured_suggestion(
-        self,
-        diag: &mut Diagnostic,
-        trait_ref: &ty::TraitRef<'tcx>,
-        bounds: hir::GenericBounds<'_>,
-        assoc: &ty::AssocItem,
-        assoc_substs: &[ty::GenericArg<'tcx>],
-        ty: Ty<'tcx>,
-        msg: &str,
-        is_bound_surely_present: bool,
-    ) -> bool {
-        // FIXME: we would want to call `resolve_vars_if_possible` on `ty` before suggesting.
-
-        let trait_bounds = bounds.iter().filter_map(|bound| match bound {
-            hir::GenericBound::Trait(ptr, hir::TraitBoundModifier::None) => Some(ptr),
-            _ => None,
-        });
-
-        let matching_trait_bounds = trait_bounds
-            .clone()
-            .filter(|ptr| ptr.trait_ref.trait_def_id() == Some(trait_ref.def_id))
-            .collect::<Vec<_>>();
-
-        let span = match &matching_trait_bounds[..] {
-            &[ptr] => ptr.span,
-            &[] if is_bound_surely_present => match &trait_bounds.collect::<Vec<_>>()[..] {
-                &[ptr] => ptr.span,
-                _ => return false,
-            },
-            _ => return false,
-        };
-
-        self.constrain_associated_type_structured_suggestion(
-            diag,
-            span,
-            assoc,
-            assoc_substs,
-            ty,
-            msg,
-        )
-    }
-
-    /// Given a span corresponding to a bound, provide a structured suggestion to set an
-    /// associated type to a given type `ty`.
-    fn constrain_associated_type_structured_suggestion(
-        self,
-        diag: &mut Diagnostic,
-        span: Span,
-        assoc: &ty::AssocItem,
-        assoc_substs: &[ty::GenericArg<'tcx>],
-        ty: Ty<'tcx>,
-        msg: &str,
-    ) -> bool {
-        if let Ok(has_params) =
-            self.sess.source_map().span_to_snippet(span).map(|snippet| snippet.ends_with('>'))
-        {
-            let (span, sugg) = if has_params {
-                let pos = span.hi() - BytePos(1);
-                let span = Span::new(pos, pos, span.ctxt(), span.parent());
-                (span, format!(", {} = {}", assoc.ident(self), ty))
-            } else {
-                let item_args = self.format_generic_args(assoc_substs);
-                (span.shrink_to_hi(), format!("<{}{} = {}>", assoc.ident(self), item_args, ty))
-            };
-            diag.span_suggestion_verbose(span, msg, sugg, MaybeIncorrect);
-            return true;
-        }
-        false
-    }
-
-    pub fn short_ty_string(self, ty: Ty<'tcx>) -> (String, Option<PathBuf>) {
-        let width = self.sess.diagnostic_width();
-        let length_limit = width.saturating_sub(30);
+    pub fn ty_string_with_limit(self, ty: Ty<'tcx>, length_limit: usize) -> String {
         let mut type_limit = 50;
         let regular = FmtPrinter::new(self, hir::def::Namespace::TypeNS)
             .pretty_print_type(ty)
             .expect("could not write to `String`")
             .into_buffer();
-        if regular.len() <= width {
-            return (regular, None);
+        if regular.len() <= length_limit {
+            return regular;
         }
         let mut short;
         loop {
@@ -1034,6 +351,20 @@ fn foo(&self) -> Self::T { String::new() }
             }
             type_limit -= 1;
         }
+        short
+    }
+
+    pub fn short_ty_string(self, ty: Ty<'tcx>) -> (String, Option<PathBuf>) {
+        let width = self.sess.diagnostic_width();
+        let length_limit = width.saturating_sub(30);
+        let regular = FmtPrinter::new(self, hir::def::Namespace::TypeNS)
+            .pretty_print_type(ty)
+            .expect("could not write to `String`")
+            .into_buffer();
+        if regular.len() <= width {
+            return (regular, None);
+        }
+        let short = self.ty_string_with_limit(ty, length_limit);
         if regular == short {
             return (regular, None);
         }
@@ -1047,11 +378,4 @@ fn foo(&self) -> Self::T { String::new() }
             Err(_) => (regular, None),
         }
     }
-
-    fn format_generic_args(self, args: &[ty::GenericArg<'tcx>]) -> String {
-        FmtPrinter::new(self, hir::def::Namespace::TypeNS)
-            .path_generic_args(Ok, args)
-            .expect("could not write to `String`.")
-            .into_buffer()
-    }
 }
diff --git a/compiler/rustc_middle/src/ty/fast_reject.rs b/compiler/rustc_middle/src/ty/fast_reject.rs
index f785fb5c4..59deade0a 100644
--- a/compiler/rustc_middle/src/ty/fast_reject.rs
+++ b/compiler/rustc_middle/src/ty/fast_reject.rs
@@ -1,6 +1,6 @@
 use crate::mir::Mutability;
 use crate::ty::subst::GenericArgKind;
-use crate::ty::{self, Ty, TyCtxt, TypeVisitable};
+use crate::ty::{self, Ty, TyCtxt, TypeVisitableExt};
 use rustc_hir::def_id::DefId;
 use std::fmt::Debug;
 use std::hash::Hash;
@@ -32,6 +32,7 @@ pub enum SimplifiedType {
     ClosureSimplifiedType(DefId),
     GeneratorSimplifiedType(DefId),
     GeneratorWitnessSimplifiedType(usize),
+    GeneratorWitnessMIRSimplifiedType(DefId),
     FunctionSimplifiedType(usize),
     PlaceholderSimplifiedType,
 }
@@ -108,6 +109,7 @@ pub fn simplify_type<'tcx>(
         ty::FnDef(def_id, _) | ty::Closure(def_id, _) => Some(ClosureSimplifiedType(def_id)),
         ty::Generator(def_id, _, _) => Some(GeneratorSimplifiedType(def_id)),
         ty::GeneratorWitness(tys) => Some(GeneratorWitnessSimplifiedType(tys.skip_binder().len())),
+        ty::GeneratorWitnessMIR(def_id, _) => Some(GeneratorWitnessMIRSimplifiedType(def_id)),
         ty::Never => Some(NeverSimplifiedType),
         ty::Tuple(tys) => Some(TupleSimplifiedType(tys.len())),
         ty::FnPtr(f) => Some(FunctionSimplifiedType(f.skip_binder().inputs().len())),
@@ -139,7 +141,8 @@ impl SimplifiedType {
             | ForeignSimplifiedType(d)
             | TraitSimplifiedType(d)
             | ClosureSimplifiedType(d)
-            | GeneratorSimplifiedType(d) => Some(d),
+            | GeneratorSimplifiedType(d)
+            | GeneratorWitnessMIRSimplifiedType(d) => Some(d),
             _ => None,
         }
     }
@@ -208,6 +211,7 @@ impl DeepRejectCtxt {
             | ty::Closure(..)
             | ty::Generator(..)
             | ty::GeneratorWitness(..)
+            | ty::GeneratorWitnessMIR(..)
             | ty::Placeholder(..)
             | ty::Bound(..)
             | ty::Infer(_) => bug!("unexpected impl_ty: {impl_ty}"),
@@ -286,7 +290,7 @@ impl DeepRejectCtxt {
             // Impls cannot contain these types as these cannot be named directly.
             ty::FnDef(..) | ty::Closure(..) | ty::Generator(..) => false,
 
-            ty::Placeholder(..) => false,
+            ty::Placeholder(..) | ty::Bound(..) => false,
 
             // Depending on the value of `treat_obligation_params`, we either
             // treat generic parameters like placeholders or like inference variables.
@@ -306,7 +310,7 @@ impl DeepRejectCtxt {
 
             ty::Error(_) => true,
 
-            ty::GeneratorWitness(..) | ty::Bound(..) => {
+            ty::GeneratorWitness(..) | ty::GeneratorWitnessMIR(..) => {
                 bug!("unexpected obligation type: {:?}", obligation_ty)
             }
         }
diff --git a/compiler/rustc_middle/src/ty/flags.rs b/compiler/rustc_middle/src/ty/flags.rs
index b7eafc4b4..91241ff40 100644
--- a/compiler/rustc_middle/src/ty/flags.rs
+++ b/compiler/rustc_middle/src/ty/flags.rs
@@ -125,6 +125,16 @@ impl FlagComputation {
                 self.bound_computation(ts, |flags, ts| flags.add_tys(ts));
             }
 
+            ty::GeneratorWitnessMIR(_, substs) => {
+                let should_remove_further_specializable =
+                    !self.flags.contains(TypeFlags::STILL_FURTHER_SPECIALIZABLE);
+                self.add_substs(substs);
+                if should_remove_further_specializable {
+                    self.flags -= TypeFlags::STILL_FURTHER_SPECIALIZABLE;
+                }
+                self.add_flags(TypeFlags::HAS_TY_GENERATOR);
+            }
+
             &ty::Closure(_, substs) => {
                 let substs = substs.as_closure();
                 let should_remove_further_specializable =
@@ -241,6 +251,10 @@ impl FlagComputation {
                 self.add_ty(ty);
                 self.add_region(region);
             }
+            ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(ct, ty)) => {
+                self.add_const(ct);
+                self.add_ty(ty);
+            }
             ty::PredicateKind::Subtype(ty::SubtypePredicate { a_is_expected: _, a, b }) => {
                 self.add_ty(a);
                 self.add_ty(b);
@@ -254,10 +268,7 @@ impl FlagComputation {
                 term,
             })) => {
                 self.add_projection_ty(projection_ty);
-                match term.unpack() {
-                    ty::TermKind::Ty(ty) => self.add_ty(ty),
-                    ty::TermKind::Const(c) => self.add_const(c),
-                }
+                self.add_term(term);
             }
             ty::PredicateKind::WellFormed(arg) => {
                 self.add_substs(slice::from_ref(&arg));
@@ -277,6 +288,10 @@ impl FlagComputation {
                 self.add_ty(ty);
             }
             ty::PredicateKind::Ambiguous => {}
+            ty::PredicateKind::AliasEq(t1, t2) => {
+                self.add_term(t1);
+                self.add_term(t2);
+            }
         }
     }
 
@@ -370,4 +385,11 @@ impl FlagComputation {
             }
         }
     }
+
+    fn add_term(&mut self, term: ty::Term<'_>) {
+        match term.unpack() {
+            ty::TermKind::Ty(ty) => self.add_ty(ty),
+            ty::TermKind::Const(ct) => self.add_const(ct),
+        }
+    }
 }
diff --git a/compiler/rustc_middle/src/ty/fold.rs b/compiler/rustc_middle/src/ty/fold.rs
index 6b9a37d84..d66f436f9 100644
--- a/compiler/rustc_middle/src/ty/fold.rs
+++ b/compiler/rustc_middle/src/ty/fold.rs
@@ -1,210 +1,10 @@
-//! A folding traversal mechanism for complex data structures that contain type
-//! information.
-//!
-//! This is a modifying traversal. It consumes the data structure, producing a
-//! (possibly) modified version of it. Both fallible and infallible versions are
-//! available. The name is potentially confusing, because this traversal is more
-//! like `Iterator::map` than `Iterator::fold`.
-//!
-//! This traversal has limited flexibility. Only a small number of "types of
-//! interest" within the complex data structures can receive custom
-//! modification. These are the ones containing the most important type-related
-//! information, such as `Ty`, `Predicate`, `Region`, and `Const`.
-//!
-//! There are three groups of traits involved in each traversal.
-//! - `TypeFoldable`. This is implemented once for many types, including:
-//!   - Types of interest, for which the methods delegate to the folder.
-//!   - All other types, including generic containers like `Vec` and `Option`.
-//!     It defines a "skeleton" of how they should be folded.
-//! - `TypeSuperFoldable`. This is implemented only for each type of interest,
-//!   and defines the folding "skeleton" for these types.
-//! - `TypeFolder`/`FallibleTypeFolder. One of these is implemented for each
-//!   folder. This defines how types of interest are folded.
-//!
-//! This means each fold is a mixture of (a) generic folding operations, and (b)
-//! custom fold operations that are specific to the folder.
-//! - The `TypeFoldable` impls handle most of the traversal, and call into
-//!   `TypeFolder`/`FallibleTypeFolder` when they encounter a type of interest.
-//! - A `TypeFolder`/`FallibleTypeFolder` may call into another `TypeFoldable`
-//!   impl, because some of the types of interest are recursive and can contain
-//!   other types of interest.
-//! - A `TypeFolder`/`FallibleTypeFolder` may also call into a `TypeSuperFoldable`
-//!   impl, because each folder might provide custom handling only for some types
-//!   of interest, or only for some variants of each type of interest, and then
-//!   use default traversal for the remaining cases.
-//!
-//! For example, if you have `struct S(Ty, U)` where `S: TypeFoldable` and `U:
-//! TypeFoldable`, and an instance `s = S(ty, u)`, it would be folded like so:
-//! ```text
-//! s.fold_with(folder) calls
-//! - ty.fold_with(folder) calls
-//!   - folder.fold_ty(ty) may call
-//!     - ty.super_fold_with(folder)
-//! - u.fold_with(folder)
-//! ```
-use crate::ty::{self, Binder, BoundTy, Ty, TyCtxt, TypeVisitable};
+use crate::ty::{self, Binder, BoundTy, Ty, TyCtxt, TypeVisitableExt};
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_hir::def_id::DefId;
 
 use std::collections::BTreeMap;
 
-/// This trait is implemented for every type that can be folded,
-/// providing the skeleton of the traversal.
-///
-/// To implement this conveniently, use the derive macro located in
-/// `rustc_macros`.
-pub trait TypeFoldable<'tcx>: TypeVisitable<'tcx> {
-    /// The entry point for folding. To fold a value `t` with a folder `f`
-    /// call: `t.try_fold_with(f)`.
-    ///
-    /// For most types, this just traverses the value, calling `try_fold_with`
-    /// on each field/element.
-    ///
-    /// For types of interest (such as `Ty`), the implementation of method
-    /// calls a folder method specifically for that type (such as
-    /// `F::try_fold_ty`). This is where control transfers from `TypeFoldable`
-    /// to `TypeFolder`.
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error>;
-
-    /// A convenient alternative to `try_fold_with` for use with infallible
-    /// folders. Do not override this method, to ensure coherence with
-    /// `try_fold_with`.
-    fn fold_with<F: TypeFolder<'tcx>>(self, folder: &mut F) -> Self {
-        self.try_fold_with(folder).into_ok()
-    }
-}
-
-// This trait is implemented for types of interest.
-pub trait TypeSuperFoldable<'tcx>: TypeFoldable<'tcx> {
-    /// Provides a default fold for a type of interest. This should only be
-    /// called within `TypeFolder` methods, when a non-custom traversal is
-    /// desired for the value of the type of interest passed to that method.
-    /// For example, in `MyFolder::try_fold_ty(ty)`, it is valid to call
-    /// `ty.try_super_fold_with(self)`, but any other folding should be done
-    /// with `xyz.try_fold_with(self)`.
-    fn try_super_fold_with<F: FallibleTypeFolder<'tcx>>(
-        self,
-        folder: &mut F,
-    ) -> Result<Self, F::Error>;
-
-    /// A convenient alternative to `try_super_fold_with` for use with
-    /// infallible folders. Do not override this method, to ensure coherence
-    /// with `try_super_fold_with`.
-    fn super_fold_with<F: TypeFolder<'tcx>>(self, folder: &mut F) -> Self {
-        self.try_super_fold_with(folder).into_ok()
-    }
-}
-
-/// This trait is implemented for every infallible folding traversal. There is
-/// a fold method defined for every type of interest. Each such method has a
-/// default that does an "identity" fold. Implementations of these methods
-/// often fall back to a `super_fold_with` method if the primary argument
-/// doesn't satisfy a particular condition.
-///
-/// A blanket implementation of [`FallibleTypeFolder`] will defer to
-/// the infallible methods of this trait to ensure that the two APIs
-/// are coherent.
-pub trait TypeFolder<'tcx>: FallibleTypeFolder<'tcx, Error = !> {
-    fn tcx<'a>(&'a self) -> TyCtxt<'tcx>;
-
-    fn fold_binder<T>(&mut self, t: Binder<'tcx, T>) -> Binder<'tcx, T>
-    where
-        T: TypeFoldable<'tcx>,
-    {
-        t.super_fold_with(self)
-    }
-
-    fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
-        t.super_fold_with(self)
-    }
-
-    fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
-        r.super_fold_with(self)
-    }
-
-    fn fold_const(&mut self, c: ty::Const<'tcx>) -> ty::Const<'tcx> {
-        c.super_fold_with(self)
-    }
-
-    fn fold_predicate(&mut self, p: ty::Predicate<'tcx>) -> ty::Predicate<'tcx> {
-        p.super_fold_with(self)
-    }
-}
-
-/// This trait is implemented for every folding traversal. There is a fold
-/// method defined for every type of interest. Each such method has a default
-/// that does an "identity" fold.
-///
-/// A blanket implementation of this trait (that defers to the relevant
-/// method of [`TypeFolder`]) is provided for all infallible folders in
-/// order to ensure the two APIs are coherent.
-pub trait FallibleTypeFolder<'tcx>: Sized {
-    type Error;
-
-    fn tcx<'a>(&'a self) -> TyCtxt<'tcx>;
-
-    fn try_fold_binder<T>(&mut self, t: Binder<'tcx, T>) -> Result<Binder<'tcx, T>, Self::Error>
-    where
-        T: TypeFoldable<'tcx>,
-    {
-        t.try_super_fold_with(self)
-    }
-
-    fn try_fold_ty(&mut self, t: Ty<'tcx>) -> Result<Ty<'tcx>, Self::Error> {
-        t.try_super_fold_with(self)
-    }
-
-    fn try_fold_region(&mut self, r: ty::Region<'tcx>) -> Result<ty::Region<'tcx>, Self::Error> {
-        r.try_super_fold_with(self)
-    }
-
-    fn try_fold_const(&mut self, c: ty::Const<'tcx>) -> Result<ty::Const<'tcx>, Self::Error> {
-        c.try_super_fold_with(self)
-    }
-
-    fn try_fold_predicate(
-        &mut self,
-        p: ty::Predicate<'tcx>,
-    ) -> Result<ty::Predicate<'tcx>, Self::Error> {
-        p.try_super_fold_with(self)
-    }
-}
-
-// This blanket implementation of the fallible trait for infallible folders
-// delegates to infallible methods to ensure coherence.
-impl<'tcx, F> FallibleTypeFolder<'tcx> for F
-where
-    F: TypeFolder<'tcx>,
-{
-    type Error = !;
-
-    fn tcx<'a>(&'a self) -> TyCtxt<'tcx> {
-        TypeFolder::tcx(self)
-    }
-
-    fn try_fold_binder<T>(&mut self, t: Binder<'tcx, T>) -> Result<Binder<'tcx, T>, !>
-    where
-        T: TypeFoldable<'tcx>,
-    {
-        Ok(self.fold_binder(t))
-    }
-
-    fn try_fold_ty(&mut self, t: Ty<'tcx>) -> Result<Ty<'tcx>, !> {
-        Ok(self.fold_ty(t))
-    }
-
-    fn try_fold_region(&mut self, r: ty::Region<'tcx>) -> Result<ty::Region<'tcx>, !> {
-        Ok(self.fold_region(r))
-    }
-
-    fn try_fold_const(&mut self, c: ty::Const<'tcx>) -> Result<ty::Const<'tcx>, !> {
-        Ok(self.fold_const(c))
-    }
-
-    fn try_fold_predicate(&mut self, p: ty::Predicate<'tcx>) -> Result<ty::Predicate<'tcx>, !> {
-        Ok(self.fold_predicate(p))
-    }
-}
+pub use rustc_type_ir::fold::{FallibleTypeFolder, TypeFoldable, TypeFolder, TypeSuperFoldable};
 
 ///////////////////////////////////////////////////////////////////////////
 // Some sample folders
@@ -221,13 +21,13 @@ where
     pub ct_op: H,
 }
 
-impl<'tcx, F, G, H> TypeFolder<'tcx> for BottomUpFolder<'tcx, F, G, H>
+impl<'tcx, F, G, H> TypeFolder<TyCtxt<'tcx>> for BottomUpFolder<'tcx, F, G, H>
 where
     F: FnMut(Ty<'tcx>) -> Ty<'tcx>,
     G: FnMut(ty::Region<'tcx>) -> ty::Region<'tcx>,
     H: FnMut(ty::Const<'tcx>) -> ty::Const<'tcx>,
 {
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
@@ -260,7 +60,7 @@ impl<'tcx> TyCtxt<'tcx> {
         mut f: impl FnMut(ty::Region<'tcx>, ty::DebruijnIndex) -> ty::Region<'tcx>,
     ) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         value.fold_with(&mut RegionFolder::new(self, &mut f))
     }
@@ -271,7 +71,7 @@ impl<'tcx> TyCtxt<'tcx> {
         mut f: impl FnMut(ty::Region<'tcx>, ty::DebruijnIndex) -> ty::Region<'tcx>,
     ) -> T
     where
-        T: TypeSuperFoldable<'tcx>,
+        T: TypeSuperFoldable<TyCtxt<'tcx>>,
     {
         value.super_fold_with(&mut RegionFolder::new(self, &mut f))
     }
@@ -311,12 +111,12 @@ impl<'a, 'tcx> RegionFolder<'a, 'tcx> {
     }
 }
 
-impl<'a, 'tcx> TypeFolder<'tcx> for RegionFolder<'a, 'tcx> {
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for RegionFolder<'a, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
-    fn fold_binder<T: TypeFoldable<'tcx>>(
+    fn fold_binder<T: TypeFoldable<TyCtxt<'tcx>>>(
         &mut self,
         t: ty::Binder<'tcx, T>,
     ) -> ty::Binder<'tcx, T> {
@@ -385,15 +185,15 @@ impl<'tcx, D: BoundVarReplacerDelegate<'tcx>> BoundVarReplacer<'tcx, D> {
     }
 }
 
-impl<'tcx, D> TypeFolder<'tcx> for BoundVarReplacer<'tcx, D>
+impl<'tcx, D> TypeFolder<TyCtxt<'tcx>> for BoundVarReplacer<'tcx, D>
 where
     D: BoundVarReplacerDelegate<'tcx>,
 {
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
-    fn fold_binder<T: TypeFoldable<'tcx>>(
+    fn fold_binder<T: TypeFoldable<TyCtxt<'tcx>>>(
         &mut self,
         t: ty::Binder<'tcx, T>,
     ) -> ty::Binder<'tcx, T> {
@@ -425,7 +225,7 @@ where
                     // debruijn index. Then we adjust it to the
                     // correct depth.
                     assert_eq!(debruijn1, ty::INNERMOST);
-                    self.tcx.reuse_or_mk_region(region, ty::ReLateBound(debruijn, br))
+                    self.tcx.mk_re_late_bound(debruijn, br)
                 } else {
                     region
                 }
@@ -471,7 +271,7 @@ impl<'tcx> TyCtxt<'tcx> {
     ) -> (T, BTreeMap<ty::BoundRegion, ty::Region<'tcx>>)
     where
         F: FnMut(ty::BoundRegion) -> ty::Region<'tcx>,
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         let mut region_map = BTreeMap::new();
         let real_fld_r = |br: ty::BoundRegion| *region_map.entry(br).or_insert_with(|| fld_r(br));
@@ -486,7 +286,7 @@ impl<'tcx> TyCtxt<'tcx> {
     ) -> T
     where
         F: FnMut(ty::BoundRegion) -> ty::Region<'tcx>,
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         let value = value.skip_binder();
         if !value.has_escaping_bound_vars() {
@@ -505,7 +305,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Replaces all escaping bound vars. The `fld_r` closure replaces escaping
     /// bound regions; the `fld_t` closure replaces escaping bound types and the `fld_c`
     /// closure replaces escaping bound consts.
-    pub fn replace_escaping_bound_vars_uncached<T: TypeFoldable<'tcx>>(
+    pub fn replace_escaping_bound_vars_uncached<T: TypeFoldable<TyCtxt<'tcx>>>(
         self,
         value: T,
         delegate: impl BoundVarReplacerDelegate<'tcx>,
@@ -521,7 +321,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Replaces all types or regions bound by the given `Binder`. The `fld_r`
     /// closure replaces bound regions, the `fld_t` closure replaces bound
     /// types, and `fld_c` replaces bound constants.
-    pub fn replace_bound_vars_uncached<T: TypeFoldable<'tcx>>(
+    pub fn replace_bound_vars_uncached<T: TypeFoldable<TyCtxt<'tcx>>>(
         self,
         value: Binder<'tcx, T>,
         delegate: impl BoundVarReplacerDelegate<'tcx>,
@@ -537,35 +337,29 @@ impl<'tcx> TyCtxt<'tcx> {
         value: ty::Binder<'tcx, T>,
     ) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         self.replace_late_bound_regions_uncached(value, |br| {
-            self.mk_region(ty::ReFree(ty::FreeRegion {
-                scope: all_outlive_scope,
-                bound_region: br.kind,
-            }))
+            self.mk_re_free(all_outlive_scope, br.kind)
         })
     }
 
     pub fn shift_bound_var_indices<T>(self, bound_vars: usize, value: T) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         let shift_bv = |bv: ty::BoundVar| ty::BoundVar::from_usize(bv.as_usize() + bound_vars);
         self.replace_escaping_bound_vars_uncached(
             value,
             FnMutDelegate {
                 regions: &mut |r: ty::BoundRegion| {
-                    self.mk_region(ty::ReLateBound(
+                    self.mk_re_late_bound(
                         ty::INNERMOST,
                         ty::BoundRegion { var: shift_bv(r.var), kind: r.kind },
-                    ))
+                    )
                 },
                 types: &mut |t: ty::BoundTy| {
-                    self.mk_ty(ty::Bound(
-                        ty::INNERMOST,
-                        ty::BoundTy { var: shift_bv(t.var), kind: t.kind },
-                    ))
+                    self.mk_bound(ty::INNERMOST, ty::BoundTy { var: shift_bv(t.var), kind: t.kind })
                 },
                 consts: &mut |c, ty: Ty<'tcx>| {
                     self.mk_const(ty::ConstKind::Bound(ty::INNERMOST, shift_bv(c)), ty)
@@ -578,7 +372,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// method lookup and a few other places where precise region relationships are not required.
     pub fn erase_late_bound_regions<T>(self, value: Binder<'tcx, T>) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         self.replace_late_bound_regions(value, |_| self.lifetimes.re_erased).0
     }
@@ -586,7 +380,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Anonymize all bound variables in `value`, this is mostly used to improve caching.
     pub fn anonymize_bound_vars<T>(self, value: Binder<'tcx, T>) -> Binder<'tcx, T>
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         struct Anonymize<'a, 'tcx> {
             tcx: TyCtxt<'tcx>,
@@ -603,16 +397,18 @@ impl<'tcx> TyCtxt<'tcx> {
                     })
                     .expect_region();
                 let br = ty::BoundRegion { var, kind };
-                self.tcx.mk_region(ty::ReLateBound(ty::INNERMOST, br))
+                self.tcx.mk_re_late_bound(ty::INNERMOST, br)
             }
             fn replace_ty(&mut self, bt: ty::BoundTy) -> Ty<'tcx> {
                 let entry = self.map.entry(bt.var);
                 let index = entry.index();
                 let var = ty::BoundVar::from_usize(index);
                 let kind = entry
-                    .or_insert_with(|| ty::BoundVariableKind::Ty(ty::BoundTyKind::Anon))
+                    .or_insert_with(|| {
+                        ty::BoundVariableKind::Ty(ty::BoundTyKind::Anon(index as u32))
+                    })
                     .expect_ty();
-                self.tcx.mk_ty(ty::Bound(ty::INNERMOST, BoundTy { var, kind }))
+                self.tcx.mk_bound(ty::INNERMOST, BoundTy { var, kind })
             }
             fn replace_const(&mut self, bv: ty::BoundVar, ty: Ty<'tcx>) -> ty::Const<'tcx> {
                 let entry = self.map.entry(bv);
@@ -626,7 +422,7 @@ impl<'tcx> TyCtxt<'tcx> {
         let mut map = Default::default();
         let delegate = Anonymize { tcx: self, map: &mut map };
         let inner = self.replace_escaping_bound_vars_uncached(value.skip_binder(), delegate);
-        let bound_vars = self.mk_bound_variable_kinds(map.into_values());
+        let bound_vars = self.mk_bound_variable_kinds_from_iter(map.into_values());
         Binder::bind_with_vars(inner, bound_vars)
     }
 }
@@ -652,12 +448,12 @@ impl<'tcx> Shifter<'tcx> {
     }
 }
 
-impl<'tcx> TypeFolder<'tcx> for Shifter<'tcx> {
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Shifter<'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
-    fn fold_binder<T: TypeFoldable<'tcx>>(
+    fn fold_binder<T: TypeFoldable<TyCtxt<'tcx>>>(
         &mut self,
         t: ty::Binder<'tcx, T>,
     ) -> ty::Binder<'tcx, T> {
@@ -671,8 +467,7 @@ impl<'tcx> TypeFolder<'tcx> for Shifter<'tcx> {
         match *r {
             ty::ReLateBound(debruijn, br) if debruijn >= self.current_index => {
                 let debruijn = debruijn.shifted_in(self.amount);
-                let shifted = ty::ReLateBound(debruijn, br);
-                self.tcx.mk_region(shifted)
+                self.tcx.mk_re_late_bound(debruijn, br)
             }
             _ => r,
         }
@@ -682,7 +477,7 @@ impl<'tcx> TypeFolder<'tcx> for Shifter<'tcx> {
         match *ty.kind() {
             ty::Bound(debruijn, bound_ty) if debruijn >= self.current_index => {
                 let debruijn = debruijn.shifted_in(self.amount);
-                self.tcx.mk_ty(ty::Bound(debruijn, bound_ty))
+                self.tcx.mk_bound(debruijn, bound_ty)
             }
 
             _ if ty.has_vars_bound_at_or_above(self.current_index) => ty.super_fold_with(self),
@@ -713,7 +508,7 @@ pub fn shift_region<'tcx>(
 ) -> ty::Region<'tcx> {
     match *region {
         ty::ReLateBound(debruijn, br) if amount > 0 => {
-            tcx.mk_region(ty::ReLateBound(debruijn.shifted_in(amount), br))
+            tcx.mk_re_late_bound(debruijn.shifted_in(amount), br)
         }
         _ => region,
     }
@@ -721,7 +516,7 @@ pub fn shift_region<'tcx>(
 
 pub fn shift_vars<'tcx, T>(tcx: TyCtxt<'tcx>, value: T, amount: u32) -> T
 where
-    T: TypeFoldable<'tcx>,
+    T: TypeFoldable<TyCtxt<'tcx>>,
 {
     debug!("shift_vars(value={:?}, amount={})", value, amount);
 
diff --git a/compiler/rustc_middle/src/ty/generics.rs b/compiler/rustc_middle/src/ty/generics.rs
index 801ca6004..baef4ffed 100644
--- a/compiler/rustc_middle/src/ty/generics.rs
+++ b/compiler/rustc_middle/src/ty/generics.rs
@@ -85,7 +85,7 @@ impl GenericParamDef {
     ) -> Option<EarlyBinder<ty::GenericArg<'tcx>>> {
         match self.kind {
             GenericParamDefKind::Type { has_default, .. } if has_default => {
-                Some(tcx.bound_type_of(self.def_id).map_bound(|t| t.into()))
+                Some(tcx.type_of(self.def_id).map_bound(|t| t.into()))
             }
             GenericParamDefKind::Const { has_default } if has_default => {
                 Some(tcx.const_param_default(self.def_id).map_bound(|c| c.into()))
@@ -100,10 +100,10 @@ impl GenericParamDef {
         preceding_substs: &[ty::GenericArg<'tcx>],
     ) -> ty::GenericArg<'tcx> {
         match &self.kind {
-            ty::GenericParamDefKind::Lifetime => tcx.lifetimes.re_static.into(),
-            ty::GenericParamDefKind::Type { .. } => tcx.ty_error().into(),
+            ty::GenericParamDefKind::Lifetime => tcx.mk_re_error_misc().into(),
+            ty::GenericParamDefKind::Type { .. } => tcx.ty_error_misc().into(),
             ty::GenericParamDefKind::Const { .. } => {
-                tcx.const_error(tcx.bound_type_of(self.def_id).subst(tcx, preceding_substs)).into()
+                tcx.const_error(tcx.type_of(self.def_id).subst(tcx, preceding_substs)).into()
             }
         }
     }
diff --git a/compiler/rustc_middle/src/ty/impls_ty.rs b/compiler/rustc_middle/src/ty/impls_ty.rs
index 3e59c0b96..4c7822acd 100644
--- a/compiler/rustc_middle/src/ty/impls_ty.rs
+++ b/compiler/rustc_middle/src/ty/impls_ty.rs
@@ -79,7 +79,7 @@ impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for ty::subst::GenericArgKin
             // WARNING: We dedup cache the `HashStable` results for `List`
             // while ignoring types and freely transmute
             // between `List<Ty<'tcx>>` and `List<GenericArg<'tcx>>`.
-            // See `fn intern_type_list` for more details.
+            // See `fn mk_type_list` for more details.
             //
             // We therefore hash types without adding a hash for their discriminant.
             //
diff --git a/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs b/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs
index 33f727297..e268553f8 100644
--- a/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs
+++ b/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs
@@ -57,7 +57,7 @@ impl<'tcx> InhabitedPredicate<'tcx> {
         match self {
             Self::False => Ok(false),
             Self::True => Ok(true),
-            Self::ConstIsZero(const_) => match const_.try_eval_usize(tcx, param_env) {
+            Self::ConstIsZero(const_) => match const_.try_eval_target_usize(tcx, param_env) {
                 None | Some(0) => Ok(true),
                 Some(1..) => Ok(false),
             },
@@ -159,7 +159,7 @@ impl<'tcx> InhabitedPredicate<'tcx> {
         match self {
             Self::ConstIsZero(c) => {
                 let c = ty::EarlyBinder(c).subst(tcx, substs);
-                let pred = match c.kind().try_to_machine_usize(tcx) {
+                let pred = match c.kind().try_to_target_usize(tcx) {
                     Some(0) => Self::True,
                     Some(1..) => Self::False,
                     None => Self::ConstIsZero(c),
diff --git a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
index 5d5089cec..92a040068 100644
--- a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
+++ b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
@@ -87,7 +87,7 @@ impl<'tcx> VariantDef {
         InhabitedPredicate::all(
             tcx,
             self.fields.iter().map(|field| {
-                let pred = tcx.type_of(field.did).inhabited_predicate(tcx);
+                let pred = tcx.type_of(field.did).subst_identity().inhabited_predicate(tcx);
                 if adt.is_enum() {
                     return pred;
                 }
@@ -105,7 +105,7 @@ impl<'tcx> VariantDef {
 impl<'tcx> Ty<'tcx> {
     pub fn inhabited_predicate(self, tcx: TyCtxt<'tcx>) -> InhabitedPredicate<'tcx> {
         match self.kind() {
-            // For now, union`s are always considered inhabited
+            // For now, unions are always considered inhabited
             Adt(adt, _) if adt.is_union() => InhabitedPredicate::True,
             // Non-exhaustive ADTs from other crates are always considered inhabited
             Adt(adt, _) if adt.is_variant_list_non_exhaustive() && !adt.did().is_local() => {
@@ -191,7 +191,7 @@ fn inhabited_predicate_type<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> InhabitedP
 
         // If we can evaluate the array length before having a `ParamEnv`, then
         // we can simplify the predicate. This is an optimization.
-        Array(ty, len) => match len.kind().try_to_machine_usize(tcx) {
+        Array(ty, len) => match len.kind().try_to_target_usize(tcx) {
             Some(0) => InhabitedPredicate::True,
             Some(1..) => ty.inhabited_predicate(tcx),
             None => ty.inhabited_predicate(tcx).or(tcx, InhabitedPredicate::ConstIsZero(len)),
diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs
index 6ac00d16c..f4028a5a9 100644
--- a/compiler/rustc_middle/src/ty/instance.rs
+++ b/compiler/rustc_middle/src/ty/instance.rs
@@ -1,7 +1,7 @@
 use crate::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use crate::ty::print::{FmtPrinter, Printer};
-use crate::ty::{self, Ty, TyCtxt, TypeFoldable, TypeSuperFoldable, TypeVisitable};
-use crate::ty::{EarlyBinder, InternalSubsts, SubstsRef};
+use crate::ty::{self, Ty, TyCtxt, TypeFoldable, TypeSuperFoldable};
+use crate::ty::{EarlyBinder, InternalSubsts, SubstsRef, TypeVisitableExt};
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def::Namespace;
 use rustc_hir::def_id::{CrateNum, DefId};
@@ -103,7 +103,7 @@ impl<'tcx> Instance<'tcx> {
     /// lifetimes erased, allowing a `ParamEnv` to be specified for use during normalization.
     pub fn ty(&self, tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>) -> Ty<'tcx> {
         let ty = tcx.type_of(self.def.def_id());
-        tcx.subst_and_normalize_erasing_regions(self.substs, param_env, ty)
+        tcx.subst_and_normalize_erasing_regions(self.substs, param_env, ty.skip_binder())
     }
 
     /// Finds a crate that contains a monomorphization of this instance that
@@ -459,7 +459,7 @@ impl<'tcx> Instance<'tcx> {
         substs: SubstsRef<'tcx>,
     ) -> Option<Instance<'tcx>> {
         debug!("resolve_for_vtable(def_id={:?}, substs={:?})", def_id, substs);
-        let fn_sig = tcx.fn_sig(def_id);
+        let fn_sig = tcx.fn_sig(def_id).subst_identity();
         let is_vtable_shim = !fn_sig.inputs().skip_binder().is_empty()
             && fn_sig.input(0).skip_binder().is_param(0)
             && tcx.generics_of(def_id).has_self;
@@ -540,7 +540,7 @@ impl<'tcx> Instance<'tcx> {
 
     pub fn resolve_drop_in_place(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ty::Instance<'tcx> {
         let def_id = tcx.require_lang_item(LangItem::DropInPlace, None);
-        let substs = tcx.intern_substs(&[ty.into()]);
+        let substs = tcx.mk_substs(&[ty.into()]);
         Instance::expect_resolve(tcx, ty::ParamEnv::reveal_all(), def_id, substs)
     }
 
@@ -584,12 +584,12 @@ impl<'tcx> Instance<'tcx> {
     /// this function returns `None`, then the MIR body does not require substitution during
     /// codegen.
     fn substs_for_mir_body(&self) -> Option<SubstsRef<'tcx>> {
-        if self.def.has_polymorphic_mir_body() { Some(self.substs) } else { None }
+        self.def.has_polymorphic_mir_body().then_some(self.substs)
     }
 
     pub fn subst_mir<T>(&self, tcx: TyCtxt<'tcx>, v: &T) -> T
     where
-        T: TypeFoldable<'tcx> + Copy,
+        T: TypeFoldable<TyCtxt<'tcx>> + Copy,
     {
         if let Some(substs) = self.substs_for_mir_body() {
             EarlyBinder(*v).subst(tcx, substs)
@@ -606,7 +606,7 @@ impl<'tcx> Instance<'tcx> {
         v: T,
     ) -> T
     where
-        T: TypeFoldable<'tcx> + Clone,
+        T: TypeFoldable<TyCtxt<'tcx>> + Clone,
     {
         if let Some(substs) = self.substs_for_mir_body() {
             tcx.subst_and_normalize_erasing_regions(substs, param_env, v)
@@ -623,7 +623,7 @@ impl<'tcx> Instance<'tcx> {
         v: T,
     ) -> Result<T, NormalizationError<'tcx>>
     where
-        T: TypeFoldable<'tcx> + Clone,
+        T: TypeFoldable<TyCtxt<'tcx>> + Clone,
     {
         if let Some(substs) = self.substs_for_mir_body() {
             tcx.try_subst_and_normalize_erasing_regions(substs, param_env, v)
@@ -662,7 +662,7 @@ fn polymorphize<'tcx>(
     let def_id = instance.def_id();
     let upvars_ty = if tcx.is_closure(def_id) {
         Some(substs.as_closure().tupled_upvars_ty())
-    } else if tcx.type_of(def_id).is_generator() {
+    } else if tcx.type_of(def_id).skip_binder().is_generator() {
         Some(substs.as_generator().tupled_upvars_ty())
     } else {
         None
@@ -674,8 +674,8 @@ fn polymorphize<'tcx>(
         tcx: TyCtxt<'tcx>,
     }
 
-    impl<'tcx> ty::TypeFolder<'tcx> for PolymorphizationFolder<'tcx> {
-        fn tcx<'a>(&'a self) -> TyCtxt<'tcx> {
+    impl<'tcx> ty::TypeFolder<TyCtxt<'tcx>> for PolymorphizationFolder<'tcx> {
+        fn interner(&self) -> TyCtxt<'tcx> {
             self.tcx
         }
 
diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs
index dfd016569..254ffc33c 100644
--- a/compiler/rustc_middle/src/ty/layout.rs
+++ b/compiler/rustc_middle/src/ty/layout.rs
@@ -1,11 +1,13 @@
+use crate::fluent_generated as fluent;
 use crate::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use crate::ty::normalize_erasing_regions::NormalizationError;
-use crate::ty::{self, ReprOptions, Ty, TyCtxt, TypeVisitable};
+use crate::ty::{self, ReprOptions, Ty, TyCtxt, TypeVisitableExt};
 use rustc_errors::{DiagnosticBuilder, Handler, IntoDiagnostic};
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 use rustc_index::vec::Idx;
 use rustc_session::config::OptLevel;
+use rustc_span::symbol::{sym, Symbol};
 use rustc_span::{Span, DUMMY_SP};
 use rustc_target::abi::call::FnAbi;
 use rustc_target::abi::*;
@@ -91,7 +93,7 @@ impl IntegerExt for Integer {
             if discr < fit {
                 bug!(
                     "Integer::repr_discr: `#[repr]` hint too small for \
-                      discriminant range of enum `{}",
+                      discriminant range of enum `{}`",
                     ty
                 )
             }
@@ -128,7 +130,8 @@ impl PrimitiveExt for Primitive {
             Int(i, signed) => i.to_ty(tcx, signed),
             F32 => tcx.types.f32,
             F64 => tcx.types.f64,
-            Pointer => tcx.mk_mut_ptr(tcx.mk_unit()),
+            // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
+            Pointer(_) => tcx.mk_mut_ptr(tcx.mk_unit()),
         }
     }
 
@@ -138,7 +141,11 @@ impl PrimitiveExt for Primitive {
     fn to_int_ty<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
         match *self {
             Int(i, signed) => i.to_ty(tcx, signed),
-            Pointer => tcx.types.usize,
+            // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
+            Pointer(_) => {
+                let signed = false;
+                tcx.data_layout().ptr_sized_integer().to_ty(tcx, signed)
+            }
             F32 | F64 => bug!("floats do not have an int type"),
         }
     }
@@ -163,6 +170,41 @@ pub const FAT_PTR_EXTRA: usize = 1;
 /// * Cranelift stores the base-2 log of the lane count in a 4 bit integer.
 pub const MAX_SIMD_LANES: u64 = 1 << 0xF;
 
+/// Used in `check_validity_requirement` to indicate the kind of initialization
+/// that is checked to be valid
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, HashStable)]
+pub enum ValidityRequirement {
+    Inhabited,
+    Zero,
+    /// The return value of mem::uninitialized, 0x01
+    /// (unless -Zstrict-init-checks is on, in which case it's the same as Uninit).
+    UninitMitigated0x01Fill,
+    /// True uninitialized memory.
+    Uninit,
+}
+
+impl ValidityRequirement {
+    pub fn from_intrinsic(intrinsic: Symbol) -> Option<Self> {
+        match intrinsic {
+            sym::assert_inhabited => Some(Self::Inhabited),
+            sym::assert_zero_valid => Some(Self::Zero),
+            sym::assert_mem_uninitialized_valid => Some(Self::UninitMitigated0x01Fill),
+            _ => None,
+        }
+    }
+}
+
+impl fmt::Display for ValidityRequirement {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Inhabited => f.write_str("is inhabited"),
+            Self::Zero => f.write_str("allows being left zeroed"),
+            Self::UninitMitigated0x01Fill => f.write_str("allows being filled with 0x01"),
+            Self::Uninit => f.write_str("allows being left uninitialized"),
+        }
+    }
+}
+
 #[derive(Copy, Clone, Debug, HashStable, TyEncodable, TyDecodable)]
 pub enum LayoutError<'tcx> {
     Unknown(Ty<'tcx>),
@@ -177,16 +219,16 @@ impl IntoDiagnostic<'_, !> for LayoutError<'_> {
         match self {
             LayoutError::Unknown(ty) => {
                 diag.set_arg("ty", ty);
-                diag.set_primary_message(rustc_errors::fluent::middle_unknown_layout);
+                diag.set_primary_message(fluent::middle_unknown_layout);
             }
             LayoutError::SizeOverflow(ty) => {
                 diag.set_arg("ty", ty);
-                diag.set_primary_message(rustc_errors::fluent::middle_values_too_big);
+                diag.set_primary_message(fluent::middle_values_too_big);
             }
             LayoutError::NormalizationFailure(ty, e) => {
                 diag.set_arg("ty", ty);
                 diag.set_arg("failure_ty", e.get_type_for_failure());
-                diag.set_primary_message(rustc_errors::fluent::middle_cannot_be_normalized);
+                diag.set_primary_message(fluent::middle_cannot_be_normalized);
             }
         }
         diag
@@ -590,7 +632,7 @@ where
                     ty::Adt(def, _) => def.variant(variant_index).fields.len(),
                     _ => bug!(),
                 };
-                tcx.intern_layout(LayoutS {
+                tcx.mk_layout(LayoutS {
                     variants: Variants::Single { index: variant_index },
                     fields: match NonZeroUsize::new(fields) {
                         Some(fields) => FieldsShape::Union(fields),
@@ -603,7 +645,7 @@ where
                 })
             }
 
-            Variants::Multiple { ref variants, .. } => cx.tcx().intern_layout(variants[variant_index].clone()),
+            Variants::Multiple { ref variants, .. } => cx.tcx().mk_layout(variants[variant_index].clone()),
         };
 
         assert_eq!(*layout.variants(), Variants::Single { index: variant_index });
@@ -625,7 +667,7 @@ where
             let tcx = cx.tcx();
             let tag_layout = |tag: Scalar| -> TyAndLayout<'tcx> {
                 TyAndLayout {
-                    layout: tcx.intern_layout(LayoutS::scalar(cx, tag)),
+                    layout: tcx.mk_layout(LayoutS::scalar(cx, tag)),
                     ty: tag.primitive().to_ty(tcx),
                 }
             };
@@ -640,6 +682,7 @@ where
                 | ty::Never
                 | ty::FnDef(..)
                 | ty::GeneratorWitness(..)
+                | ty::GeneratorWitnessMIR(..)
                 | ty::Foreign(..)
                 | ty::Dynamic(_, _, ty::Dyn) => {
                     bug!("TyAndLayout::field({:?}): not applicable", this)
@@ -764,7 +807,7 @@ where
 
                 ty::Dynamic(_, _, ty::DynStar) => {
                     if i == 0 {
-                        TyMaybeWithLayout::Ty(tcx.types.usize)
+                        TyMaybeWithLayout::Ty(tcx.mk_mut_ptr(tcx.types.unit))
                     } else if i == 1 {
                         // FIXME(dyn-star) same FIXME as above applies here too
                         TyMaybeWithLayout::Ty(
@@ -812,17 +855,12 @@ where
         let tcx = cx.tcx();
         let param_env = cx.param_env();
 
-        let addr_space_of_ty = |ty: Ty<'tcx>| {
-            if ty.is_fn() { cx.data_layout().instruction_address_space } else { AddressSpace::DATA }
-        };
-
         let pointee_info = match *this.ty.kind() {
             ty::RawPtr(mt) if offset.bytes() == 0 => {
                 tcx.layout_of(param_env.and(mt.ty)).ok().map(|layout| PointeeInfo {
                     size: layout.size,
                     align: layout.align.abi,
                     safe: None,
-                    address_space: addr_space_of_ty(mt.ty),
                 })
             }
             ty::FnPtr(fn_sig) if offset.bytes() == 0 => {
@@ -830,44 +868,26 @@ where
                     size: layout.size,
                     align: layout.align.abi,
                     safe: None,
-                    address_space: cx.data_layout().instruction_address_space,
                 })
             }
             ty::Ref(_, ty, mt) if offset.bytes() == 0 => {
-                let address_space = addr_space_of_ty(ty);
-                let kind = if tcx.sess.opts.optimize == OptLevel::No {
-                    // Use conservative pointer kind if not optimizing. This saves us the
-                    // Freeze/Unpin queries, and can save time in the codegen backend (noalias
-                    // attributes in LLVM have compile-time cost even in unoptimized builds).
-                    PointerKind::SharedMutable
-                } else {
-                    match mt {
-                        hir::Mutability::Not => {
-                            if ty.is_freeze(tcx, cx.param_env()) {
-                                PointerKind::Frozen
-                            } else {
-                                PointerKind::SharedMutable
-                            }
-                        }
-                        hir::Mutability::Mut => {
-                            // References to self-referential structures should not be considered
-                            // noalias, as another pointer to the structure can be obtained, that
-                            // is not based-on the original reference. We consider all !Unpin
-                            // types to be potentially self-referential here.
-                            if ty.is_unpin(tcx, cx.param_env()) {
-                                PointerKind::UniqueBorrowed
-                            } else {
-                                PointerKind::UniqueBorrowedPinned
-                            }
-                        }
-                    }
+                // Use conservative pointer kind if not optimizing. This saves us the
+                // Freeze/Unpin queries, and can save time in the codegen backend (noalias
+                // attributes in LLVM have compile-time cost even in unoptimized builds).
+                let optimize = tcx.sess.opts.optimize != OptLevel::No;
+                let kind = match mt {
+                    hir::Mutability::Not => PointerKind::SharedRef {
+                        frozen: optimize && ty.is_freeze(tcx, cx.param_env()),
+                    },
+                    hir::Mutability::Mut => PointerKind::MutableRef {
+                        unpin: optimize && ty.is_unpin(tcx, cx.param_env()),
+                    },
                 };
 
                 tcx.layout_of(param_env.and(ty)).ok().map(|layout| PointeeInfo {
                     size: layout.size,
                     align: layout.align.abi,
                     safe: Some(kind),
-                    address_space,
                 })
             }
 
@@ -904,7 +924,9 @@ where
                 let mut result = None;
 
                 if let Some(variant) = data_variant {
-                    let ptr_end = offset + Pointer.size(cx);
+                    // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
+                    // (requires passing in the expected address space from the caller)
+                    let ptr_end = offset + Pointer(AddressSpace::DATA).size(cx);
                     for i in 0..variant.fields.count() {
                         let field_start = variant.fields.offset(i);
                         if field_start <= offset {
@@ -930,7 +952,10 @@ where
                 if let Some(ref mut pointee) = result {
                     if let ty::Adt(def, _) = this.ty.kind() {
                         if def.is_box() && offset.bytes() == 0 {
-                            pointee.safe = Some(PointerKind::UniqueOwned);
+                            let optimize = tcx.sess.opts.optimize != OptLevel::No;
+                            pointee.safe = Some(PointerKind::Box {
+                                unpin: optimize && this.ty.boxed_ty().is_unpin(tcx, cx.param_env()),
+                            });
                         }
                     }
                 }
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index 7dfcd1bb5..dce18a585 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -12,7 +12,7 @@
 #![allow(rustc::usage_of_ty_tykind)]
 
 pub use self::fold::{FallibleTypeFolder, TypeFoldable, TypeFolder, TypeSuperFoldable};
-pub use self::visit::{TypeSuperVisitable, TypeVisitable, TypeVisitor};
+pub use self::visit::{TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor};
 pub use self::AssocItemContainer::*;
 pub use self::BorrowKind::*;
 pub use self::IntVarValue::*;
@@ -31,23 +31,22 @@ pub use generics::*;
 use rustc_ast as ast;
 use rustc_ast::node_id::NodeMap;
 use rustc_attr as attr;
-use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap, FxIndexSet};
 use rustc_data_structures::intern::Interned;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::tagged_ptr::CopyTaggedPtr;
+use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
-use rustc_hir::def::{CtorKind, CtorOf, DefKind, LifetimeRes, Res};
+use rustc_hir::def::{CtorKind, CtorOf, DefKind, DocLinkResMap, LifetimeRes, Res};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LocalDefIdMap};
 use rustc_hir::Node;
 use rustc_index::vec::IndexVec;
 use rustc_macros::HashStable;
 use rustc_query_system::ich::StableHashingContext;
 use rustc_serialize::{Decodable, Encodable};
-use rustc_session::cstore::Untracked;
 use rustc_span::hygiene::MacroKind;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
-use rustc_span::{ExpnId, Span};
+use rustc_span::{ExpnId, ExpnKind, Span};
 use rustc_target::abi::{Align, Integer, IntegerType, VariantIdx};
 pub use rustc_target::abi::{ReprFlags, ReprOptions};
 use rustc_type_ir::WithCachedTypeInfo;
@@ -74,7 +73,7 @@ pub use self::binding::BindingMode;
 pub use self::binding::BindingMode::*;
 pub use self::closure::{
     is_ancestor_or_same_capture, place_to_string_for_capture, BorrowKind, CaptureInfo,
-    CapturedPlace, ClosureKind, MinCaptureInformationMap, MinCaptureList,
+    CapturedPlace, ClosureKind, ClosureTypeInfo, MinCaptureInformationMap, MinCaptureList,
     RootVariableMinCaptureList, UpvarCapture, UpvarCaptureMap, UpvarId, UpvarListMap, UpvarPath,
     CAPTURE_STRUCT_LOCAL,
 };
@@ -154,7 +153,6 @@ pub type RegisteredTools = FxHashSet<Ident>;
 pub struct ResolverOutputs {
     pub global_ctxt: ResolverGlobalCtxt,
     pub ast_lowering: ResolverAstLowering,
-    pub untracked: Untracked,
 }
 
 #[derive(Debug)]
@@ -167,12 +165,8 @@ pub struct ResolverGlobalCtxt {
     pub effective_visibilities: EffectiveVisibilities,
     pub extern_crate_map: FxHashMap<LocalDefId, CrateNum>,
     pub maybe_unused_trait_imports: FxIndexSet<LocalDefId>,
-    pub maybe_unused_extern_crates: Vec<(LocalDefId, Span)>,
     pub reexport_map: FxHashMap<LocalDefId, Vec<ModChild>>,
     pub glob_map: FxHashMap<LocalDefId, FxHashSet<Symbol>>,
-    /// Extern prelude entries. The value is `true` if the entry was introduced
-    /// via `extern crate` item and not `--extern` option or compiler built-in.
-    pub extern_prelude: FxHashMap<Symbol, bool>,
     pub main_def: Option<MainDefinition>,
     pub trait_impls: FxIndexMap<DefId, Vec<LocalDefId>>,
     /// A list of proc macro LocalDefIds, written out in the order in which
@@ -182,6 +176,9 @@ pub struct ResolverGlobalCtxt {
     /// exist under `std`. For example, wrote `str::from_utf8` instead of `std::str::from_utf8`.
     pub confused_type_with_std_module: FxHashMap<Span, Span>,
     pub registered_tools: RegisteredTools,
+    pub doc_link_resolutions: FxHashMap<LocalDefId, DocLinkResMap>,
+    pub doc_link_traits_in_scope: FxHashMap<LocalDefId, Vec<DefId>>,
+    pub all_macro_rules: FxHashMap<Symbol, Res<ast::NodeId>>,
 }
 
 /// Resolutions that should only be used for lowering.
@@ -453,18 +450,6 @@ pub struct CReaderCacheKey {
 #[rustc_pass_by_value]
 pub struct Ty<'tcx>(Interned<'tcx, WithCachedTypeInfo<TyKind<'tcx>>>);
 
-impl<'tcx> TyCtxt<'tcx> {
-    /// A "bool" type used in rustc_mir_transform unit tests when we
-    /// have not spun up a TyCtxt.
-    pub const BOOL_TY_FOR_UNIT_TESTING: Ty<'tcx> =
-        Ty(Interned::new_unchecked(&WithCachedTypeInfo {
-            internee: ty::Bool,
-            stable_hash: Fingerprint::ZERO,
-            flags: TypeFlags::empty(),
-            outer_exclusive_binder: DebruijnIndex::from_usize(0),
-        }));
-}
-
 impl ty::EarlyBoundRegion {
     /// Does this early bound region have a name? Early bound regions normally
     /// always have names except when using anonymous lifetimes (`'_`).
@@ -558,6 +543,8 @@ impl<'tcx> Predicate<'tcx> {
             | PredicateKind::Clause(Clause::RegionOutlives(_))
             | PredicateKind::Clause(Clause::TypeOutlives(_))
             | PredicateKind::Clause(Clause::Projection(_))
+            | PredicateKind::Clause(Clause::ConstArgHasType(..))
+            | PredicateKind::AliasEq(..)
             | PredicateKind::ObjectSafe(_)
             | PredicateKind::ClosureKind(_, _, _)
             | PredicateKind::Subtype(_)
@@ -595,6 +582,10 @@ pub enum Clause<'tcx> {
     /// `where <T as TraitRef>::Name == X`, approximately.
     /// See the `ProjectionPredicate` struct for details.
     Projection(ProjectionPredicate<'tcx>),
+
+    /// Ensures that a const generic argument to a parameter `const N: u8`
+    /// is of type `u8`.
+    ConstArgHasType(Const<'tcx>, Ty<'tcx>),
 }
 
 #[derive(Clone, Copy, PartialEq, Eq, Hash, TyEncodable, TyDecodable)]
@@ -645,6 +636,12 @@ pub enum PredicateKind<'tcx> {
     /// A marker predicate that is always ambiguous.
     /// Used for coherence to mark opaque types as possibly equal to each other but ambiguous.
     Ambiguous,
+
+    /// Separate from `Clause::Projection` which is used for normalization in new solver.
+    /// This predicate requires two terms to be equal to eachother.
+    ///
+    /// Only used for new solver
+    AliasEq(Term<'tcx>, Term<'tcx>),
 }
 
 /// The crate outlives map is computed during typeck and contains the
@@ -713,7 +710,7 @@ impl<'tcx> Predicate<'tcx> {
         //   The substitution from the input trait-ref is therefore going to be
         //   `'a => 'x` (where `'x` has a DB index of 1).
         // - The supertrait-ref is `for<'b> Bar1<'a,'b>`, where `'a` is an
-        //   early-bound parameter and `'b' is a late-bound parameter with a
+        //   early-bound parameter and `'b` is a late-bound parameter with a
         //   DB index of 1.
         // - If we replace `'a` with `'x` from the input, it too will have
         //   a DB index of 1, and thus we'll have `for<'x,'b> Bar1<'x,'b>`
@@ -756,7 +753,7 @@ impl<'tcx> Predicate<'tcx> {
         let new = EarlyBinder(shifted_pred).subst(tcx, trait_ref.skip_binder().substs);
         // 3) ['x] + ['b] -> ['x, 'b]
         let bound_vars =
-            tcx.mk_bound_variable_kinds(trait_bound_vars.iter().chain(pred_bound_vars));
+            tcx.mk_bound_variable_kinds_from_iter(trait_bound_vars.iter().chain(pred_bound_vars));
         tcx.reuse_or_mk_predicate(self, ty::Binder::bind_with_vars(new, bound_vars))
     }
 }
@@ -917,14 +914,17 @@ impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for Term<'tcx> {
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for Term<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for Term<'tcx> {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self, F::Error> {
         Ok(self.unpack().try_fold_with(folder)?.pack())
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx> for Term<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for Term<'tcx> {
+    fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
         self.unpack().visit_with(visitor)
     }
 }
@@ -976,6 +976,33 @@ impl<'tcx> Term<'tcx> {
             TermKind::Const(c) => c.into(),
         }
     }
+
+    /// This function returns `None` for `AliasKind::Opaque`.
+    ///
+    /// FIXME: rename `AliasTy` to `AliasTerm` and make sure we correctly
+    /// deal with constants.
+    pub fn to_alias_term_no_opaque(&self, tcx: TyCtxt<'tcx>) -> Option<AliasTy<'tcx>> {
+        match self.unpack() {
+            TermKind::Ty(ty) => match ty.kind() {
+                ty::Alias(kind, alias_ty) => match kind {
+                    AliasKind::Projection => Some(*alias_ty),
+                    AliasKind::Opaque => None,
+                },
+                _ => None,
+            },
+            TermKind::Const(ct) => match ct.kind() {
+                ConstKind::Unevaluated(uv) => Some(tcx.mk_alias_ty(uv.def.did, uv.substs)),
+                _ => None,
+            },
+        }
+    }
+
+    pub fn is_infer(&self) -> bool {
+        match self.unpack() {
+            TermKind::Ty(ty) => ty.is_ty_or_numeric_infer(),
+            TermKind::Const(ct) => ct.is_ct_infer(),
+        }
+    }
 }
 
 const TAG_MASK: usize = 0b11;
@@ -1165,6 +1192,8 @@ impl<'tcx> Predicate<'tcx> {
         match predicate.skip_binder() {
             PredicateKind::Clause(Clause::Trait(t)) => Some(predicate.rebind(t)),
             PredicateKind::Clause(Clause::Projection(..))
+            | PredicateKind::Clause(Clause::ConstArgHasType(..))
+            | PredicateKind::AliasEq(..)
             | PredicateKind::Subtype(..)
             | PredicateKind::Coerce(..)
             | PredicateKind::Clause(Clause::RegionOutlives(..))
@@ -1184,6 +1213,8 @@ impl<'tcx> Predicate<'tcx> {
         match predicate.skip_binder() {
             PredicateKind::Clause(Clause::Projection(t)) => Some(predicate.rebind(t)),
             PredicateKind::Clause(Clause::Trait(..))
+            | PredicateKind::Clause(Clause::ConstArgHasType(..))
+            | PredicateKind::AliasEq(..)
             | PredicateKind::Subtype(..)
             | PredicateKind::Coerce(..)
             | PredicateKind::Clause(Clause::RegionOutlives(..))
@@ -1203,7 +1234,9 @@ impl<'tcx> Predicate<'tcx> {
         match predicate.skip_binder() {
             PredicateKind::Clause(Clause::TypeOutlives(data)) => Some(predicate.rebind(data)),
             PredicateKind::Clause(Clause::Trait(..))
+            | PredicateKind::Clause(Clause::ConstArgHasType(..))
             | PredicateKind::Clause(Clause::Projection(..))
+            | PredicateKind::AliasEq(..)
             | PredicateKind::Subtype(..)
             | PredicateKind::Coerce(..)
             | PredicateKind::Clause(Clause::RegionOutlives(..))
@@ -1322,7 +1355,7 @@ pub struct OpaqueHiddenType<'tcx> {
 }
 
 impl<'tcx> OpaqueHiddenType<'tcx> {
-    pub fn report_mismatch(&self, other: &Self, tcx: TyCtxt<'tcx>) {
+    pub fn report_mismatch(&self, other: &Self, tcx: TyCtxt<'tcx>) -> ErrorGuaranteed {
         // Found different concrete types for the opaque type.
         let sub_diag = if self.span == other.span {
             TypeMismatchReason::ConflictType { span: self.span }
@@ -1334,7 +1367,7 @@ impl<'tcx> OpaqueHiddenType<'tcx> {
             other_ty: other.ty,
             other_span: other.span,
             sub: sub_diag,
-        });
+        })
     }
 
     #[instrument(level = "debug", skip(tcx), ret)]
@@ -1382,7 +1415,7 @@ pub struct Placeholder<T> {
 
 pub type PlaceholderRegion = Placeholder<BoundRegionKind>;
 
-pub type PlaceholderType = Placeholder<BoundVar>;
+pub type PlaceholderType = Placeholder<BoundTyKind>;
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, HashStable)]
 #[derive(TyEncodable, TyDecodable, PartialOrd, Ord)]
@@ -1589,8 +1622,8 @@ impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for ParamEnv<'tcx> {
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for ParamEnv<'tcx> {
-    fn try_fold_with<F: ty::fold::FallibleTypeFolder<'tcx>>(
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for ParamEnv<'tcx> {
+    fn try_fold_with<F: ty::fold::FallibleTypeFolder<TyCtxt<'tcx>>>(
         self,
         folder: &mut F,
     ) -> Result<Self, F::Error> {
@@ -1602,8 +1635,8 @@ impl<'tcx> TypeFoldable<'tcx> for ParamEnv<'tcx> {
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx> for ParamEnv<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for ParamEnv<'tcx> {
+    fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
         self.caller_bounds().visit_with(visitor)?;
         self.reveal().visit_with(visitor)
     }
@@ -1728,7 +1761,7 @@ impl<'tcx> ParamEnv<'tcx> {
     /// `where Box<u32>: Copy`, which are clearly never
     /// satisfiable. We generally want to behave as if they were true,
     /// although the surrounding function is never reachable.
-    pub fn and<T: TypeVisitable<'tcx>>(self, value: T) -> ParamEnvAnd<'tcx, T> {
+    pub fn and<T: TypeVisitable<TyCtxt<'tcx>>>(self, value: T) -> ParamEnvAnd<'tcx, T> {
         match self.reveal() {
             Reveal::UserFacing => ParamEnvAnd { param_env: self, value },
 
@@ -1986,7 +2019,7 @@ impl<'tcx> FieldDef {
     /// Returns the type of this field. The resulting type is not normalized. The `subst` is
     /// typically obtained via the second field of [`TyKind::Adt`].
     pub fn ty(&self, tcx: TyCtxt<'tcx>, subst: SubstsRef<'tcx>) -> Ty<'tcx> {
-        tcx.bound_type_of(self.did).subst(tcx, subst)
+        tcx.type_of(self.did).subst(tcx, subst)
     }
 
     /// Computes the `Ident` of this variant by looking up the `Span`
@@ -2038,6 +2071,12 @@ pub enum ImplOverlapKind {
     Issue33140,
 }
 
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, TyEncodable, TyDecodable, HashStable)]
+pub enum ImplTraitInTraitData {
+    Trait { fn_def_id: DefId, opaque_def_id: DefId },
+    Impl { fn_def_id: DefId },
+}
+
 impl<'tcx> TyCtxt<'tcx> {
     pub fn typeck_body(self, body: hir::BodyId) -> &'tcx TypeckResults<'tcx> {
         self.typeck(self.hir().body_owner_def_id(body))
@@ -2167,7 +2206,7 @@ impl<'tcx> TyCtxt<'tcx> {
         Some(Ident::new(def, span))
     }
 
-    pub fn opt_associated_item(self, def_id: DefId) -> Option<&'tcx AssocItem> {
+    pub fn opt_associated_item(self, def_id: DefId) -> Option<AssocItem> {
         if let DefKind::AssocConst | DefKind::AssocFn | DefKind::AssocTy = self.def_kind(def_id) {
             Some(self.associated_item(def_id))
         } else {
@@ -2357,15 +2396,17 @@ impl<'tcx> TyCtxt<'tcx> {
         self.trait_def(trait_def_id).has_auto_impl
     }
 
+    /// Returns `true` if this is coinductive, either because it is
+    /// an auto trait or because it has the `#[rustc_coinductive]` attribute.
+    pub fn trait_is_coinductive(self, trait_def_id: DefId) -> bool {
+        self.trait_def(trait_def_id).is_coinductive
+    }
+
     /// Returns `true` if this is a trait alias.
     pub fn trait_is_alias(self, trait_def_id: DefId) -> bool {
         self.def_kind(trait_def_id) == DefKind::TraitAlias
     }
 
-    pub fn trait_is_coinductive(self, trait_def_id: DefId) -> bool {
-        self.trait_is_auto(trait_def_id) || self.lang_items().sized_trait() == Some(trait_def_id)
-    }
-
     /// Returns layout of a generator. Layout might be unavailable if the
     /// generator is tainted by errors.
     pub fn generator_layout(self, def_id: DefId) -> Option<&'tcx GeneratorLayout<'tcx>> {
@@ -2396,15 +2437,30 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn impl_of_method(self, def_id: DefId) -> Option<DefId> {
         if let DefKind::AssocConst | DefKind::AssocFn | DefKind::AssocTy = self.def_kind(def_id) {
             let parent = self.parent(def_id);
-            if let DefKind::Impl = self.def_kind(parent) {
+            if let DefKind::Impl { .. } = self.def_kind(parent) {
                 return Some(parent);
             }
         }
         None
     }
 
-    /// If the given `DefId` belongs to a trait that was automatically derived, returns `true`.
-    pub fn is_builtin_derive(self, def_id: DefId) -> bool {
+    /// Check if the given `DefId` is `#\[automatically_derived\]`, *and*
+    /// whether it was produced by expanding a builtin derive macro.
+    pub fn is_builtin_derived(self, def_id: DefId) -> bool {
+        if self.is_automatically_derived(def_id)
+            && let Some(def_id) = def_id.as_local()
+            && let outer = self.def_span(def_id).ctxt().outer_expn_data()
+            && matches!(outer.kind, ExpnKind::Macro(MacroKind::Derive, _))
+            && self.has_attr(outer.macro_def_id.unwrap(), sym::rustc_builtin_macro)
+        {
+            true
+        } else {
+            false
+        }
+    }
+
+    /// Check if the given `DefId` is `#\[automatically_derived\]`.
+    pub fn is_automatically_derived(self, def_id: DefId) -> bool {
         self.has_attr(def_id, sym::automatically_derived)
     }
 
@@ -2437,6 +2493,7 @@ impl<'tcx> TyCtxt<'tcx> {
         ident
     }
 
+    // FIXME(vincenzoapalzzo): move the HirId to a LocalDefId
     pub fn adjust_ident_and_get_scope(
         self,
         mut ident: Ident,
@@ -2470,10 +2527,6 @@ impl<'tcx> TyCtxt<'tcx> {
             }
     }
 
-    pub fn is_object_safe(self, key: DefId) -> bool {
-        self.object_safety_violations(key).is_empty()
-    }
-
     #[inline]
     pub fn is_const_fn_raw(self, def_id: DefId) -> bool {
         matches!(
@@ -2494,6 +2547,34 @@ impl<'tcx> TyCtxt<'tcx> {
         }
         def_id
     }
+
+    pub fn impl_method_has_trait_impl_trait_tys(self, def_id: DefId) -> bool {
+        if self.def_kind(def_id) != DefKind::AssocFn {
+            return false;
+        }
+
+        let Some(item) = self.opt_associated_item(def_id) else { return false; };
+        if item.container != ty::AssocItemContainer::ImplContainer {
+            return false;
+        }
+
+        let Some(trait_item_def_id) = item.trait_item_def_id else { return false; };
+
+        // FIXME(RPITIT): This does a somewhat manual walk through the signature
+        // of the trait fn to look for any RPITITs, but that's kinda doing a lot
+        // of work. We can probably remove this when we refactor RPITITs to be
+        // associated types.
+        self.fn_sig(trait_item_def_id).subst_identity().skip_binder().output().walk().any(|arg| {
+            if let ty::GenericArgKind::Type(ty) = arg.unpack()
+                && let ty::Alias(ty::Projection, data) = ty.kind()
+                && self.def_kind(data.def_id) == DefKind::ImplTraitPlaceholder
+            {
+                true
+            } else {
+                false
+            }
+        })
+    }
 }
 
 /// Yields the parent function's `LocalDefId` if `def_id` is an `impl Trait` definition.
@@ -2619,7 +2700,7 @@ impl<'tcx> fmt::Debug for SymbolName<'tcx> {
 }
 
 #[derive(Debug, Default, Copy, Clone)]
-pub struct FoundRelationships {
+pub struct InferVarInfo {
     /// This is true if we identified that this Ty (`?T`) is found in a `?T: Foo`
     /// obligation, where:
     ///
diff --git a/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs b/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs
index ee13920d5..7c59879a1 100644
--- a/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs
+++ b/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs
@@ -7,16 +7,14 @@
 //! `normalize_generic_arg_after_erasing_regions` query for each type
 //! or constant found within. (This underlying query is what is cached.)
 
-use crate::mir;
 use crate::traits::query::NoSolution;
 use crate::ty::fold::{FallibleTypeFolder, TypeFoldable, TypeFolder};
-use crate::ty::{self, EarlyBinder, SubstsRef, Ty, TyCtxt};
+use crate::ty::{self, EarlyBinder, SubstsRef, Ty, TyCtxt, TypeVisitableExt};
 
 #[derive(Debug, Copy, Clone, HashStable, TyEncodable, TyDecodable)]
 pub enum NormalizationError<'tcx> {
     Type(Ty<'tcx>),
     Const(ty::Const<'tcx>),
-    ConstantKind(mir::ConstantKind<'tcx>),
 }
 
 impl<'tcx> NormalizationError<'tcx> {
@@ -24,7 +22,6 @@ impl<'tcx> NormalizationError<'tcx> {
         match self {
             NormalizationError::Type(t) => format!("{}", t),
             NormalizationError::Const(c) => format!("{}", c),
-            NormalizationError::ConstantKind(ck) => format!("{}", ck),
         }
     }
 }
@@ -38,7 +35,7 @@ impl<'tcx> TyCtxt<'tcx> {
     #[tracing::instrument(level = "debug", skip(self, param_env))]
     pub fn normalize_erasing_regions<T>(self, param_env: ty::ParamEnv<'tcx>, value: T) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         debug!(
             "normalize_erasing_regions::<{}>(value={:?}, param_env={:?})",
@@ -70,7 +67,7 @@ impl<'tcx> TyCtxt<'tcx> {
         value: T,
     ) -> Result<T, NormalizationError<'tcx>>
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         debug!(
             "try_normalize_erasing_regions::<{}>(value={:?}, param_env={:?})",
@@ -107,7 +104,7 @@ impl<'tcx> TyCtxt<'tcx> {
         value: ty::Binder<'tcx, T>,
     ) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         let value = self.erase_late_bound_regions(value);
         self.normalize_erasing_regions(param_env, value)
@@ -127,7 +124,7 @@ impl<'tcx> TyCtxt<'tcx> {
         value: ty::Binder<'tcx, T>,
     ) -> Result<T, NormalizationError<'tcx>>
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         let value = self.erase_late_bound_regions(value);
         self.try_normalize_erasing_regions(param_env, value)
@@ -145,7 +142,7 @@ impl<'tcx> TyCtxt<'tcx> {
         value: T,
     ) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         debug!(
             "subst_and_normalize_erasing_regions(\
@@ -169,7 +166,7 @@ impl<'tcx> TyCtxt<'tcx> {
         value: T,
     ) -> Result<T, NormalizationError<'tcx>>
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         debug!(
             "subst_and_normalize_erasing_regions(\
@@ -196,14 +193,14 @@ impl<'tcx> NormalizeAfterErasingRegionsFolder<'tcx> {
         let arg = self.param_env.and(arg);
 
         self.tcx.try_normalize_generic_arg_after_erasing_regions(arg).unwrap_or_else(|_| bug!(
-                "Failed to normalize {:?}, maybe try to call `try_normalize_erasing_regions` instead",
-                arg.value
-            ))
+            "Failed to normalize {:?}, maybe try to call `try_normalize_erasing_regions` instead",
+            arg.value
+        ))
     }
 }
 
-impl<'tcx> TypeFolder<'tcx> for NormalizeAfterErasingRegionsFolder<'tcx> {
-    fn tcx(&self) -> TyCtxt<'tcx> {
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for NormalizeAfterErasingRegionsFolder<'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
@@ -238,10 +235,10 @@ impl<'tcx> TryNormalizeAfterErasingRegionsFolder<'tcx> {
     }
 }
 
-impl<'tcx> FallibleTypeFolder<'tcx> for TryNormalizeAfterErasingRegionsFolder<'tcx> {
+impl<'tcx> FallibleTypeFolder<TyCtxt<'tcx>> for TryNormalizeAfterErasingRegionsFolder<'tcx> {
     type Error = NormalizationError<'tcx>;
 
-    fn tcx(&self) -> TyCtxt<'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
diff --git a/compiler/rustc_middle/src/ty/opaque_types.rs b/compiler/rustc_middle/src/ty/opaque_types.rs
index 98cd92007..751f3066c 100644
--- a/compiler/rustc_middle/src/ty/opaque_types.rs
+++ b/compiler/rustc_middle/src/ty/opaque_types.rs
@@ -3,6 +3,7 @@ use crate::ty::fold::{TypeFolder, TypeSuperFoldable};
 use crate::ty::subst::{GenericArg, GenericArgKind};
 use crate::ty::{self, Ty, TyCtxt, TypeFoldable};
 use rustc_data_structures::fx::FxHashMap;
+use rustc_span::def_id::DefId;
 use rustc_span::Span;
 
 /// Converts generic params of a TypeFoldable from one
@@ -47,10 +48,51 @@ impl<'tcx> ReverseMapper<'tcx> {
         assert!(!self.do_not_error);
         kind.fold_with(self)
     }
+
+    fn fold_closure_substs(
+        &mut self,
+        def_id: DefId,
+        substs: ty::SubstsRef<'tcx>,
+    ) -> ty::SubstsRef<'tcx> {
+        // I am a horrible monster and I pray for death. When
+        // we encounter a closure here, it is always a closure
+        // from within the function that we are currently
+        // type-checking -- one that is now being encapsulated
+        // in an opaque type. Ideally, we would
+        // go through the types/lifetimes that it references
+        // and treat them just like we would any other type,
+        // which means we would error out if we find any
+        // reference to a type/region that is not in the
+        // "reverse map".
+        //
+        // **However,** in the case of closures, there is a
+        // somewhat subtle (read: hacky) consideration. The
+        // problem is that our closure types currently include
+        // all the lifetime parameters declared on the
+        // enclosing function, even if they are unused by the
+        // closure itself. We can't readily filter them out,
+        // so here we replace those values with `'empty`. This
+        // can't really make a difference to the rest of the
+        // compiler; those regions are ignored for the
+        // outlives relation, and hence don't affect trait
+        // selection or auto traits, and they are erased
+        // during codegen.
+
+        let generics = self.tcx.generics_of(def_id);
+        self.tcx.mk_substs_from_iter(substs.iter().enumerate().map(|(index, kind)| {
+            if index < generics.parent_count {
+                // Accommodate missing regions in the parent kinds...
+                self.fold_kind_no_missing_regions_error(kind)
+            } else {
+                // ...but not elsewhere.
+                self.fold_kind_normally(kind)
+            }
+        }))
+    }
 }
 
-impl<'tcx> TypeFolder<'tcx> for ReverseMapper<'tcx> {
-    fn tcx(&self) -> TyCtxt<'tcx> {
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ReverseMapper<'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
@@ -67,6 +109,8 @@ impl<'tcx> TypeFolder<'tcx> for ReverseMapper<'tcx> {
             // them.
             ty::ReErased => return r,
 
+            ty::ReError(_) => return r,
+
             // The regions that we expect from borrow checking.
             ty::ReEarlyBound(_) | ty::ReFree(_) => {}
 
@@ -83,20 +127,21 @@ impl<'tcx> TypeFolder<'tcx> for ReverseMapper<'tcx> {
             Some(u) => panic!("region mapped to unexpected kind: {:?}", u),
             None if self.do_not_error => self.tcx.lifetimes.re_static,
             None => {
-                self.tcx
+                let e = self
+                    .tcx
                     .sess
                     .struct_span_err(self.span, "non-defining opaque type use in defining scope")
                     .span_label(
                         self.span,
                         format!(
                             "lifetime `{}` is part of concrete type but not used in \
-                                 parameter list of the `impl Trait` type alias",
+                             parameter list of the `impl Trait` type alias",
                             r
                         ),
                     )
                     .emit();
 
-                self.tcx().lifetimes.re_static
+                self.interner().mk_re_error(e)
             }
         }
     }
@@ -104,59 +149,20 @@ impl<'tcx> TypeFolder<'tcx> for ReverseMapper<'tcx> {
     fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
         match *ty.kind() {
             ty::Closure(def_id, substs) => {
-                // I am a horrible monster and I pray for death. When
-                // we encounter a closure here, it is always a closure
-                // from within the function that we are currently
-                // type-checking -- one that is now being encapsulated
-                // in an opaque type. Ideally, we would
-                // go through the types/lifetimes that it references
-                // and treat them just like we would any other type,
-                // which means we would error out if we find any
-                // reference to a type/region that is not in the
-                // "reverse map".
-                //
-                // **However,** in the case of closures, there is a
-                // somewhat subtle (read: hacky) consideration. The
-                // problem is that our closure types currently include
-                // all the lifetime parameters declared on the
-                // enclosing function, even if they are unused by the
-                // closure itself. We can't readily filter them out,
-                // so here we replace those values with `'empty`. This
-                // can't really make a difference to the rest of the
-                // compiler; those regions are ignored for the
-                // outlives relation, and hence don't affect trait
-                // selection or auto traits, and they are erased
-                // during codegen.
-
-                let generics = self.tcx.generics_of(def_id);
-                let substs = self.tcx.mk_substs(substs.iter().enumerate().map(|(index, kind)| {
-                    if index < generics.parent_count {
-                        // Accommodate missing regions in the parent kinds...
-                        self.fold_kind_no_missing_regions_error(kind)
-                    } else {
-                        // ...but not elsewhere.
-                        self.fold_kind_normally(kind)
-                    }
-                }));
-
+                let substs = self.fold_closure_substs(def_id, substs);
                 self.tcx.mk_closure(def_id, substs)
             }
 
             ty::Generator(def_id, substs, movability) => {
-                let generics = self.tcx.generics_of(def_id);
-                let substs = self.tcx.mk_substs(substs.iter().enumerate().map(|(index, kind)| {
-                    if index < generics.parent_count {
-                        // Accommodate missing regions in the parent kinds...
-                        self.fold_kind_no_missing_regions_error(kind)
-                    } else {
-                        // ...but not elsewhere.
-                        self.fold_kind_normally(kind)
-                    }
-                }));
-
+                let substs = self.fold_closure_substs(def_id, substs);
                 self.tcx.mk_generator(def_id, substs, movability)
             }
 
+            ty::GeneratorWitnessMIR(def_id, substs) => {
+                let substs = self.fold_closure_substs(def_id, substs);
+                self.tcx.mk_generator_witness_mir(def_id, substs)
+            }
+
             ty::Param(param) => {
                 // Look it up in the substitution list.
                 match self.map.get(&ty.into()).map(|k| k.unpack()) {
@@ -180,7 +186,7 @@ impl<'tcx> TypeFolder<'tcx> for ReverseMapper<'tcx> {
                                 .emit();
                         }
 
-                        self.tcx().ty_error()
+                        self.interner().ty_error_misc()
                     }
                 }
             }
@@ -208,7 +214,7 @@ impl<'tcx> TypeFolder<'tcx> for ReverseMapper<'tcx> {
                             });
                         }
 
-                        self.tcx().const_error(ct.ty())
+                        self.interner().const_error(ct.ty())
                     }
                 }
             }
diff --git a/compiler/rustc_middle/src/ty/parameterized.rs b/compiler/rustc_middle/src/ty/parameterized.rs
index 24f3d1acf..8849e7eab 100644
--- a/compiler/rustc_middle/src/ty/parameterized.rs
+++ b/compiler/rustc_middle/src/ty/parameterized.rs
@@ -57,7 +57,7 @@ trivially_parameterized_over_tcx! {
     crate::metadata::ModChild,
     crate::middle::codegen_fn_attrs::CodegenFnAttrs,
     crate::middle::exported_symbols::SymbolExportInfo,
-    crate::middle::resolve_lifetime::ObjectLifetimeDefault,
+    crate::middle::resolve_bound_vars::ObjectLifetimeDefault,
     crate::mir::ConstQualifs,
     ty::AssocItemContainer,
     ty::DeducedParamAttrs,
@@ -81,6 +81,8 @@ trivially_parameterized_over_tcx! {
     rustc_hir::IsAsync,
     rustc_hir::LangItem,
     rustc_hir::def::DefKind,
+    rustc_hir::def::DocLinkResMap,
+    rustc_hir::def_id::DefId,
     rustc_hir::def_id::DefIndex,
     rustc_hir::definitions::DefKey,
     rustc_index::bit_set::BitSet<u32>,
@@ -117,6 +119,7 @@ macro_rules! parameterized_over_tcx {
 parameterized_over_tcx! {
     crate::middle::exported_symbols::ExportedSymbol,
     crate::mir::Body,
+    crate::mir::GeneratorLayout,
     ty::Ty,
     ty::FnSig,
     ty::GenericPredicates,
diff --git a/compiler/rustc_middle/src/ty/print/mod.rs b/compiler/rustc_middle/src/ty/print/mod.rs
index c302c4611..021c20b58 100644
--- a/compiler/rustc_middle/src/ty/print/mod.rs
+++ b/compiler/rustc_middle/src/ty/print/mod.rs
@@ -115,7 +115,7 @@ pub trait Printer<'tcx>: Sized {
 
             DefPathData::Impl => {
                 let generics = self.tcx().generics_of(def_id);
-                let self_ty = self.tcx().bound_type_of(def_id);
+                let self_ty = self.tcx().type_of(def_id);
                 let impl_trait_ref = self.tcx().impl_trait_ref(def_id);
                 let (self_ty, impl_trait_ref) = if substs.len() >= generics.count() {
                     (
@@ -265,6 +265,7 @@ fn characteristic_def_id_of_type_cached<'a>(
         ty::FnDef(def_id, _)
         | ty::Closure(def_id, _)
         | ty::Generator(def_id, _, _)
+        | ty::GeneratorWitnessMIR(def_id, _)
         | ty::Foreign(def_id) => Some(def_id),
 
         ty::Bool
diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index ae7c20fff..6a053c368 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -1,7 +1,7 @@
 use crate::mir::interpret::{AllocRange, GlobalAlloc, Pointer, Provenance, Scalar};
 use crate::ty::{
     self, ConstInt, DefIdTree, ParamConst, ScalarInt, Term, TermKind, Ty, TyCtxt, TypeFoldable,
-    TypeSuperFoldable, TypeSuperVisitable, TypeVisitable,
+    TypeSuperFoldable, TypeSuperVisitable, TypeVisitable, TypeVisitableExt,
 };
 use crate::ty::{GenericArg, GenericArgKind};
 use rustc_apfloat::ieee::{Double, Single};
@@ -22,7 +22,6 @@ use rustc_target::spec::abi::Abi;
 use smallvec::SmallVec;
 
 use std::cell::Cell;
-use std::char;
 use std::collections::BTreeMap;
 use std::fmt::{self, Write as _};
 use std::iter;
@@ -183,7 +182,7 @@ impl<'tcx> RegionHighlightMode<'tcx> {
 
     /// Convenience wrapper for `highlighting_region`.
     pub fn highlighting_region_vid(&mut self, vid: ty::RegionVid, number: usize) {
-        self.highlighting_region(self.tcx.mk_region(ty::ReVar(vid)), number)
+        self.highlighting_region(self.tcx.mk_re_var(vid), number)
     }
 
     /// Returns `Some(n)` with the number to use for the given region, if any.
@@ -226,7 +225,7 @@ pub trait PrettyPrinter<'tcx>:
 
     fn in_binder<T>(self, value: &ty::Binder<'tcx, T>) -> Result<Self, Self::Error>
     where
-        T: Print<'tcx, Self, Output = Self, Error = Self::Error> + TypeFoldable<'tcx>,
+        T: Print<'tcx, Self, Output = Self, Error = Self::Error> + TypeFoldable<TyCtxt<'tcx>>,
     {
         value.as_ref().skip_binder().print(self)
     }
@@ -237,7 +236,7 @@ pub trait PrettyPrinter<'tcx>:
         f: F,
     ) -> Result<Self, Self::Error>
     where
-        T: Print<'tcx, Self, Output = Self, Error = Self::Error> + TypeFoldable<'tcx>,
+        T: Print<'tcx, Self, Output = Self, Error = Self::Error> + TypeFoldable<TyCtxt<'tcx>>,
     {
         f(value.as_ref().skip_binder(), self)
     }
@@ -675,8 +674,12 @@ pub trait PrettyPrinter<'tcx>:
                 p!(")")
             }
             ty::FnDef(def_id, substs) => {
-                let sig = self.tcx().bound_fn_sig(def_id).subst(self.tcx(), substs);
-                p!(print(sig), " {{", print_value_path(def_id, substs), "}}");
+                if NO_QUERIES.with(|q| q.get()) {
+                    p!(print_def_path(def_id, substs));
+                } else {
+                    let sig = self.tcx().fn_sig(def_id).subst(self.tcx(), substs);
+                    p!(print(sig), " {{", print_value_path(def_id, substs), "}}");
+                }
             }
             ty::FnPtr(ref bare_fn) => p!(print(bare_fn)),
             ty::Infer(infer_ty) => {
@@ -698,8 +701,10 @@ pub trait PrettyPrinter<'tcx>:
             ty::Error(_) => p!("[type error]"),
             ty::Param(ref param_ty) => p!(print(param_ty)),
             ty::Bound(debruijn, bound_ty) => match bound_ty.kind {
-                ty::BoundTyKind::Anon => self.pretty_print_bound_var(debruijn, bound_ty.var)?,
-                ty::BoundTyKind::Param(p) => p!(write("{}", p)),
+                ty::BoundTyKind::Anon(bv) => {
+                    self.pretty_print_bound_var(debruijn, ty::BoundVar::from_u32(bv))?
+                }
+                ty::BoundTyKind::Param(_, s) => p!(write("{}", s)),
             },
             ty::Adt(def, substs) => {
                 p!(print_def_path(def.did(), substs));
@@ -730,15 +735,18 @@ pub trait PrettyPrinter<'tcx>:
                     p!(print(data))
                 }
             }
-            ty::Placeholder(placeholder) => p!(write("Placeholder({:?})", placeholder)),
+            ty::Placeholder(placeholder) => match placeholder.name {
+                ty::BoundTyKind::Anon(_) => p!(write("Placeholder({:?})", placeholder)),
+                ty::BoundTyKind::Param(_, name) => p!(write("{}", name)),
+            },
             ty::Alias(ty::Opaque, ty::AliasTy { def_id, substs, .. }) => {
-                // FIXME(eddyb) print this with `print_def_path`.
                 // We use verbose printing in 'NO_QUERIES' mode, to
                 // avoid needing to call `predicates_of`. This should
                 // only affect certain debug messages (e.g. messages printed
                 // from `rustc_middle::ty` during the computation of `tcx.predicates_of`),
                 // and should have no effect on any compiler output.
-                if self.should_print_verbose() || NO_QUERIES.with(|q| q.get()) {
+                if self.should_print_verbose() {
+                    // FIXME(eddyb) print this with `print_def_path`.
                     p!(write("Opaque({:?}, {:?})", def_id, substs));
                     return Ok(self);
                 }
@@ -746,8 +754,10 @@ pub trait PrettyPrinter<'tcx>:
                 let parent = self.tcx().parent(def_id);
                 match self.tcx().def_kind(parent) {
                     DefKind::TyAlias | DefKind::AssocTy => {
+                        // NOTE: I know we should check for NO_QUERIES here, but it's alright.
+                        // `type_of` on a type alias or assoc type should never cause a cycle.
                         if let ty::Alias(ty::Opaque, ty::AliasTy { def_id: d, .. }) =
-                            *self.tcx().type_of(parent).kind()
+                            *self.tcx().type_of(parent).subst_identity().kind()
                         {
                             if d == def_id {
                                 // If the type alias directly starts with the `impl` of the
@@ -760,7 +770,14 @@ pub trait PrettyPrinter<'tcx>:
                         p!(print_def_path(def_id, substs));
                         return Ok(self);
                     }
-                    _ => return self.pretty_print_opaque_impl_type(def_id, substs),
+                    _ => {
+                        if NO_QUERIES.with(|q| q.get()) {
+                            p!(print_def_path(def_id, &[]));
+                            return Ok(self);
+                        } else {
+                            return self.pretty_print_opaque_impl_type(def_id, substs);
+                        }
+                    }
                 }
             }
             ty::Str => p!("str"),
@@ -811,6 +828,28 @@ pub trait PrettyPrinter<'tcx>:
             ty::GeneratorWitness(types) => {
                 p!(in_binder(&types));
             }
+            ty::GeneratorWitnessMIR(did, substs) => {
+                p!(write("["));
+                if !self.tcx().sess.verbose() {
+                    p!("generator witness");
+                    // FIXME(eddyb) should use `def_span`.
+                    if let Some(did) = did.as_local() {
+                        let span = self.tcx().def_span(did);
+                        p!(write(
+                            "@{}",
+                            // This may end up in stderr diagnostics but it may also be emitted
+                            // into MIR. Hence we use the remapped path if available
+                            self.tcx().sess.source_map().span_to_embeddable_string(span)
+                        ));
+                    } else {
+                        p!(write("@"), print_def_path(did, substs));
+                    }
+                } else {
+                    p!(print_def_path(did, substs));
+                }
+
+                p!("]")
+            }
             ty::Closure(did, substs) => {
                 p!(write("["));
                 if !self.should_print_verbose() {
@@ -1062,9 +1101,11 @@ pub trait PrettyPrinter<'tcx>:
             write!(self, "Sized")?;
         }
 
-        for re in lifetimes {
-            write!(self, " + ")?;
-            self = self.print_region(re)?;
+        if !FORCE_TRIMMED_PATH.with(|flag| flag.get()) {
+            for re in lifetimes {
+                write!(self, " + ")?;
+                self = self.print_region(re)?;
+            }
         }
 
         Ok(self)
@@ -1171,7 +1212,7 @@ pub trait PrettyPrinter<'tcx>:
                 // in order to place the projections inside the `<...>`.
                 if !resugared {
                     // Use a type that can't appear in defaults of type parameters.
-                    let dummy_cx = cx.tcx().mk_ty_infer(ty::FreshTy(0));
+                    let dummy_cx = cx.tcx().mk_fresh_ty(0);
                     let principal = principal.with_self_ty(cx.tcx(), dummy_cx);
 
                     let args = cx
@@ -1992,7 +2033,7 @@ impl<'tcx> PrettyPrinter<'tcx> for FmtPrinter<'_, 'tcx> {
 
     fn in_binder<T>(self, value: &ty::Binder<'tcx, T>) -> Result<Self, Self::Error>
     where
-        T: Print<'tcx, Self, Output = Self, Error = Self::Error> + TypeFoldable<'tcx>,
+        T: Print<'tcx, Self, Output = Self, Error = Self::Error> + TypeFoldable<TyCtxt<'tcx>>,
     {
         self.pretty_in_binder(value)
     }
@@ -2003,7 +2044,7 @@ impl<'tcx> PrettyPrinter<'tcx> for FmtPrinter<'_, 'tcx> {
         f: C,
     ) -> Result<Self, Self::Error>
     where
-        T: Print<'tcx, Self, Output = Self, Error = Self::Error> + TypeFoldable<'tcx>,
+        T: Print<'tcx, Self, Output = Self, Error = Self::Error> + TypeFoldable<TyCtxt<'tcx>>,
     {
         self.pretty_wrap_binder(value, f)
     }
@@ -2048,6 +2089,10 @@ impl<'tcx> PrettyPrinter<'tcx> for FmtPrinter<'_, 'tcx> {
             return true;
         }
 
+        if FORCE_TRIMMED_PATH.with(|flag| flag.get()) {
+            return false;
+        }
+
         let identify_regions = self.tcx.sess.opts.unstable_opts.identify_regions;
 
         match *region {
@@ -2071,7 +2116,7 @@ impl<'tcx> PrettyPrinter<'tcx> for FmtPrinter<'_, 'tcx> {
 
             ty::ReVar(_) if identify_regions => true,
 
-            ty::ReVar(_) | ty::ReErased => false,
+            ty::ReVar(_) | ty::ReErased | ty::ReError(_) => false,
 
             ty::ReStatic => true,
         }
@@ -2151,6 +2196,7 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
             }
             ty::ReVar(_) => {}
             ty::ReErased => {}
+            ty::ReError(_) => {}
             ty::ReStatic => {
                 p!("'static");
                 return Ok(self);
@@ -2178,12 +2224,12 @@ struct RegionFolder<'a, 'tcx> {
             ),
 }
 
-impl<'a, 'tcx> ty::TypeFolder<'tcx> for RegionFolder<'a, 'tcx> {
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+impl<'a, 'tcx> ty::TypeFolder<TyCtxt<'tcx>> for RegionFolder<'a, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
-    fn fold_binder<T: TypeFoldable<'tcx>>(
+    fn fold_binder<T: TypeFoldable<TyCtxt<'tcx>>>(
         &mut self,
         t: ty::Binder<'tcx, T>,
     ) -> ty::Binder<'tcx, T> {
@@ -2228,7 +2274,7 @@ impl<'a, 'tcx> ty::TypeFolder<'tcx> for RegionFolder<'a, 'tcx> {
         };
         if let ty::ReLateBound(debruijn1, br) = *region {
             assert_eq!(debruijn1, ty::INNERMOST);
-            self.tcx.mk_region(ty::ReLateBound(self.current_index, br))
+            self.tcx.mk_re_late_bound(self.current_index, br)
         } else {
             region
         }
@@ -2243,7 +2289,7 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
         value: &ty::Binder<'tcx, T>,
     ) -> Result<(Self, T, BTreeMap<ty::BoundRegion, ty::Region<'tcx>>), fmt::Error>
     where
-        T: Print<'tcx, Self, Output = Self, Error = fmt::Error> + TypeFoldable<'tcx>,
+        T: Print<'tcx, Self, Output = Self, Error = fmt::Error> + TypeFoldable<TyCtxt<'tcx>>,
     {
         fn name_by_region_index(
             index: usize,
@@ -2324,6 +2370,7 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
         } else {
             let tcx = self.tcx;
 
+            let trim_path = FORCE_TRIMMED_PATH.with(|flag| flag.get());
             // Closure used in `RegionFolder` to create names for anonymous late-bound
             // regions. We use two `DebruijnIndex`es (one for the currently folded
             // late-bound region and the other for the binder level) to determine
@@ -2339,10 +2386,10 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
                         if let Some(lt_idx) = lifetime_idx {
                             if lt_idx > binder_level_idx {
                                 let kind = ty::BrNamed(CRATE_DEF_ID.to_def_id(), name);
-                                return tcx.mk_region(ty::ReLateBound(
+                                return tcx.mk_re_late_bound(
                                     ty::INNERMOST,
                                     ty::BoundRegion { var: br.var, kind },
-                                ));
+                                );
                             }
                         }
 
@@ -2354,10 +2401,10 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
                         if let Some(lt_idx) = lifetime_idx {
                             if lt_idx > binder_level_idx {
                                 let kind = ty::BrNamed(def_id, name);
-                                return tcx.mk_region(ty::ReLateBound(
+                                return tcx.mk_re_late_bound(
                                     ty::INNERMOST,
                                     ty::BoundRegion { var: br.var, kind },
-                                ));
+                                );
                             }
                         }
 
@@ -2367,10 +2414,10 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
                         if let Some(lt_idx) = lifetime_idx {
                             if lt_idx > binder_level_idx {
                                 let kind = br.kind;
-                                return tcx.mk_region(ty::ReLateBound(
+                                return tcx.mk_re_late_bound(
                                     ty::INNERMOST,
                                     ty::BoundRegion { var: br.var, kind },
-                                ));
+                                );
                             }
                         }
 
@@ -2378,9 +2425,11 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
                     }
                 };
 
-                start_or_continue(&mut self, "for<", ", ");
-                do_continue(&mut self, name);
-                tcx.mk_region(ty::ReLateBound(ty::INNERMOST, ty::BoundRegion { var: br.var, kind }))
+                if !trim_path {
+                    start_or_continue(&mut self, "for<", ", ");
+                    do_continue(&mut self, name);
+                }
+                tcx.mk_re_late_bound(ty::INNERMOST, ty::BoundRegion { var: br.var, kind })
             };
             let mut folder = RegionFolder {
                 tcx,
@@ -2390,7 +2439,9 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
             };
             let new_value = value.clone().skip_binder().fold_with(&mut folder);
             let region_map = folder.region_map;
-            start_or_continue(&mut self, "", "> ");
+            if !trim_path {
+                start_or_continue(&mut self, "", "> ");
+            }
             (new_value, region_map)
         };
 
@@ -2401,7 +2452,7 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
 
     pub fn pretty_in_binder<T>(self, value: &ty::Binder<'tcx, T>) -> Result<Self, fmt::Error>
     where
-        T: Print<'tcx, Self, Output = Self, Error = fmt::Error> + TypeFoldable<'tcx>,
+        T: Print<'tcx, Self, Output = Self, Error = fmt::Error> + TypeFoldable<TyCtxt<'tcx>>,
     {
         let old_region_index = self.region_index;
         let (new, new_value, _) = self.name_all_regions(value)?;
@@ -2417,7 +2468,7 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
         f: C,
     ) -> Result<Self, fmt::Error>
     where
-        T: Print<'tcx, Self, Output = Self, Error = fmt::Error> + TypeFoldable<'tcx>,
+        T: Print<'tcx, Self, Output = Self, Error = fmt::Error> + TypeFoldable<TyCtxt<'tcx>>,
     {
         let old_region_index = self.region_index;
         let (new, new_value, _) = self.name_all_regions(value)?;
@@ -2429,7 +2480,7 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
 
     fn prepare_region_info<T>(&mut self, value: &ty::Binder<'tcx, T>)
     where
-        T: TypeVisitable<'tcx>,
+        T: TypeVisitable<TyCtxt<'tcx>>,
     {
         struct RegionNameCollector<'tcx> {
             used_region_names: FxHashSet<Symbol>,
@@ -2445,7 +2496,7 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
             }
         }
 
-        impl<'tcx> ty::visit::TypeVisitor<'tcx> for RegionNameCollector<'tcx> {
+        impl<'tcx> ty::visit::TypeVisitor<TyCtxt<'tcx>> for RegionNameCollector<'tcx> {
             type BreakTy = ();
 
             fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
@@ -2482,7 +2533,7 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
 
 impl<'tcx, T, P: PrettyPrinter<'tcx>> Print<'tcx, P> for ty::Binder<'tcx, T>
 where
-    T: Print<'tcx, P, Output = P, Error = P::Error> + TypeFoldable<'tcx>,
+    T: Print<'tcx, P, Output = P, Error = P::Error> + TypeFoldable<TyCtxt<'tcx>>,
 {
     type Output = P;
     type Error = P::Error;
@@ -2648,7 +2699,7 @@ define_print_and_forward_display! {
 
     ty::ExistentialTraitRef<'tcx> {
         // Use a type that can't appear in defaults of type parameters.
-        let dummy_self = cx.tcx().mk_ty_infer(ty::FreshTy(0));
+        let dummy_self = cx.tcx().mk_fresh_ty(0);
         let trait_ref = self.with_self_ty(cx.tcx(), dummy_self);
         p!(print(trait_ref.print_only_trait_path()))
     }
@@ -2774,15 +2825,18 @@ define_print_and_forward_display! {
             ty::PredicateKind::Clause(ty::Clause::RegionOutlives(predicate)) => p!(print(predicate)),
             ty::PredicateKind::Clause(ty::Clause::TypeOutlives(predicate)) => p!(print(predicate)),
             ty::PredicateKind::Clause(ty::Clause::Projection(predicate)) => p!(print(predicate)),
+            ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(ct, ty)) => {
+                p!("the constant `", print(ct), "` has type `", print(ty), "`")
+            },
             ty::PredicateKind::WellFormed(arg) => p!(print(arg), " well-formed"),
             ty::PredicateKind::ObjectSafe(trait_def_id) => {
                 p!("the trait `", print_def_path(trait_def_id, &[]), "` is object-safe")
             }
-            ty::PredicateKind::ClosureKind(closure_def_id, _closure_substs, kind) => {
-                p!("the closure `",
+            ty::PredicateKind::ClosureKind(closure_def_id, _closure_substs, kind) => p!(
+                "the closure `",
                 print_value_path(closure_def_id, &[]),
-                write("` implements the trait `{}`", kind))
-            }
+                write("` implements the trait `{}`", kind)
+            ),
             ty::PredicateKind::ConstEvaluatable(ct) => {
                 p!("the constant `", print(ct), "` can be evaluated")
             }
@@ -2793,6 +2847,7 @@ define_print_and_forward_display! {
                 p!("the type `", print(ty), "` is found in the environment")
             }
             ty::PredicateKind::Ambiguous => p!("ambiguous"),
+            ty::PredicateKind::AliasEq(t1, t2) => p!(print(t1), " == ", print(t2)),
         }
     }
 
diff --git a/compiler/rustc_middle/src/ty/query.rs b/compiler/rustc_middle/src/ty/query.rs
index 9d4ee22a7..2bc51baf8 100644
--- a/compiler/rustc_middle/src/ty/query.rs
+++ b/compiler/rustc_middle/src/ty/query.rs
@@ -1,3 +1,5 @@
+#![allow(unused_parens)]
+
 use crate::dep_graph;
 use crate::infer::canonical::{self, Canonical};
 use crate::lint::LintExpectation;
@@ -6,7 +8,7 @@ use crate::middle::codegen_fn_attrs::CodegenFnAttrs;
 use crate::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo};
 use crate::middle::lib_features::LibFeatures;
 use crate::middle::privacy::EffectiveVisibilities;
-use crate::middle::resolve_lifetime::{ObjectLifetimeDefault, Region, ResolveLifetimes};
+use crate::middle::resolve_bound_vars::{ObjectLifetimeDefault, ResolveBoundVars, ResolvedArg};
 use crate::middle::stability::{self, DeprecationEntry};
 use crate::mir;
 use crate::mir::interpret::GlobalId;
@@ -30,11 +32,12 @@ use crate::traits::specialization_graph;
 use crate::traits::{self, ImplSource};
 use crate::ty::context::TyCtxtFeed;
 use crate::ty::fast_reject::SimplifiedType;
-use crate::ty::layout::TyAndLayout;
+use crate::ty::layout::ValidityRequirement;
 use crate::ty::subst::{GenericArg, SubstsRef};
 use crate::ty::util::AlwaysRequiresDrop;
 use crate::ty::GeneratorDiagnosticData;
 use crate::ty::{self, CrateInherentImpls, ParamEnvAnd, Ty, TyCtxt, UnusedGenericParams};
+use rustc_arena::TypedArena;
 use rustc_ast as ast;
 use rustc_ast::expand::allocator::AllocatorKind;
 use rustc_attr as attr;
@@ -42,10 +45,11 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap, FxIndexSet};
 use rustc_data_structures::steal::Steal;
 use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::Lrc;
+use rustc_data_structures::sync::WorkerLocal;
 use rustc_data_structures::unord::UnordSet;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
-use rustc_hir::def::DefKind;
+use rustc_hir::def::{DefKind, DocLinkResMap};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, DefIdSet, LocalDefId};
 use rustc_hir::hir_id::OwnerId;
 use rustc_hir::lang_items::{LangItem, LanguageItems};
@@ -60,6 +64,7 @@ use rustc_span::symbol::Symbol;
 use rustc_span::{Span, DUMMY_SP};
 use rustc_target::abi;
 use rustc_target::spec::PanicStrategy;
+use std::mem;
 use std::ops::Deref;
 use std::path::PathBuf;
 use std::sync::Arc;
@@ -67,6 +72,12 @@ use std::sync::Arc;
 pub(crate) use rustc_query_system::query::QueryJobId;
 use rustc_query_system::query::*;
 
+#[derive(Default)]
+pub struct QuerySystem<'tcx> {
+    pub arenas: QueryArenas<'tcx>,
+    pub caches: QueryCaches<'tcx>,
+}
+
 #[derive(Copy, Clone)]
 pub struct TyCtxtAt<'tcx> {
     pub tcx: TyCtxt<'tcx>,
@@ -106,30 +117,21 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 }
 
-/// Helper for `TyCtxtEnsure` to avoid a closure.
-#[inline(always)]
-fn noop<T>(_: &T) {}
-
-/// Helper to ensure that queries only return `Copy` types.
-#[inline(always)]
-fn copy<T: Copy>(x: &T) -> T {
-    *x
-}
-
 macro_rules! query_helper_param_ty {
     (DefId) => { impl IntoQueryParam<DefId> };
+    (LocalDefId) => { impl IntoQueryParam<LocalDefId> };
     ($K:ty) => { $K };
 }
 
-macro_rules! query_storage {
-    ([][$K:ty, $V:ty]) => {
-        <<$K as Key>::CacheSelector as CacheSelector<'tcx, $V>>::Cache
+macro_rules! query_if_arena {
+    ([] $arena:tt $no_arena:tt) => {
+        $no_arena
     };
-    ([(arena_cache) $($rest:tt)*][$K:ty, $V:ty]) => {
-        <<$K as Key>::CacheSelector as CacheSelector<'tcx, $V>>::ArenaCache
+    ([(arena_cache) $($rest:tt)*] $arena:tt $no_arena:tt) => {
+        $arena
     };
-    ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
-        query_storage!([$($modifiers)*][$($args)*])
+    ([$other:tt $($modifiers:tt)*]$($args:tt)*) => {
+        query_if_arena!([$($modifiers)*]$($args)*)
     };
 }
 
@@ -141,7 +143,7 @@ macro_rules! separate_provide_extern_decl {
         for<'tcx> fn(
             TyCtxt<'tcx>,
             query_keys::$name<'tcx>,
-        ) -> query_values::$name<'tcx>
+        ) -> query_provided::$name<'tcx>
     };
     ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
         separate_provide_extern_decl!([$($modifiers)*][$($args)*])
@@ -199,17 +201,71 @@ macro_rules! define_callbacks {
 
             $(pub type $name<'tcx> = $V;)*
         }
+
+        /// This module specifies the type returned from query providers and the type used for
+        /// decoding. For regular queries this is the declared returned type `V`, but
+        /// `arena_cache` will use `<V as Deref>::Target` instead.
         #[allow(nonstandard_style, unused_lifetimes)]
-        pub mod query_storage {
+        pub mod query_provided {
             use super::*;
 
-            $(pub type $name<'tcx> = query_storage!([$($modifiers)*][$($K)*, $V]);)*
+            $(
+                pub type $name<'tcx> = query_if_arena!([$($modifiers)*] (<$V as Deref>::Target) ($V));
+            )*
         }
+
+        /// This module has a function per query which takes a `query_provided` value and coverts
+        /// it to a regular `V` value by allocating it on an arena if the query has the
+        /// `arena_cache` modifier. This will happen when computing the query using a provider or
+        /// decoding a stored result.
         #[allow(nonstandard_style, unused_lifetimes)]
-        pub mod query_stored {
+        pub mod query_provided_to_value {
             use super::*;
 
-            $(pub type $name<'tcx> = <query_storage::$name<'tcx> as QueryStorage>::Stored;)*
+            $(
+                #[inline(always)]
+                pub fn $name<'tcx>(
+                    _tcx: TyCtxt<'tcx>,
+                    value: query_provided::$name<'tcx>,
+                ) -> query_values::$name<'tcx> {
+                    query_if_arena!([$($modifiers)*]
+                        {
+                            if mem::needs_drop::<query_provided::$name<'tcx>>() {
+                                &*_tcx.query_system.arenas.$name.alloc(value)
+                            } else {
+                                &*_tcx.arena.dropless.alloc(value)
+                            }
+                        }
+                        (value)
+                    )
+                }
+            )*
+        }
+        #[allow(nonstandard_style, unused_lifetimes)]
+        pub mod query_storage {
+            use super::*;
+
+            $(
+                pub type $name<'tcx> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, $V>>::Cache;
+            )*
+        }
+
+        pub struct QueryArenas<'tcx> {
+            $($(#[$attr])* pub $name: query_if_arena!([$($modifiers)*]
+                (WorkerLocal<TypedArena<<$V as Deref>::Target>>)
+                ()
+            ),)*
+        }
+
+        impl Default for QueryArenas<'_> {
+            fn default() -> Self {
+                Self {
+                    $($name: query_if_arena!([$($modifiers)*]
+                        (WorkerLocal::new(|_| Default::default()))
+                        ()
+                    ),)*
+                }
+            }
         }
 
         #[derive(Default)]
@@ -224,14 +280,10 @@ macro_rules! define_callbacks {
                 let key = key.into_query_param();
                 opt_remap_env_constness!([$($modifiers)*][key]);
 
-                let cached = try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key, noop);
-
-                match cached {
-                    Ok(()) => return,
-                    Err(()) => (),
-                }
-
-                self.tcx.queries.$name(self.tcx, DUMMY_SP, key, QueryMode::Ensure);
+                match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
+                    Some(_) => return,
+                    None => self.tcx.queries.$name(self.tcx, DUMMY_SP, key, QueryMode::Ensure),
+                };
             })*
         }
 
@@ -239,7 +291,7 @@ macro_rules! define_callbacks {
             $($(#[$attr])*
             #[inline(always)]
             #[must_use]
-            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<'tcx>
+            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V
             {
                 self.at(DUMMY_SP).$name(key)
             })*
@@ -248,19 +300,15 @@ macro_rules! define_callbacks {
         impl<'tcx> TyCtxtAt<'tcx> {
             $($(#[$attr])*
             #[inline(always)]
-            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<'tcx>
+            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V
             {
                 let key = key.into_query_param();
                 opt_remap_env_constness!([$($modifiers)*][key]);
 
-                let cached = try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key, copy);
-
-                match cached {
-                    Ok(value) => return value,
-                    Err(()) => (),
+                match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
+                    Some(value) => value,
+                    None => self.tcx.queries.$name(self.tcx, self.span, key, QueryMode::Get).unwrap(),
                 }
-
-                self.tcx.queries.$name(self.tcx, self.span, key, QueryMode::Get).unwrap()
             })*
         }
 
@@ -268,7 +316,7 @@ macro_rules! define_callbacks {
             $(pub $name: for<'tcx> fn(
                 TyCtxt<'tcx>,
                 query_keys::$name<'tcx>,
-            ) -> query_values::$name<'tcx>,)*
+            ) -> query_provided::$name<'tcx>,)*
         }
 
         pub struct ExternProviders {
@@ -281,8 +329,9 @@ macro_rules! define_callbacks {
 
                 Providers {
                     $($name: |_, key| bug!(
-                        "`tcx.{}({:?})` is not supported for {} crate;\n
-                        hint: Queries can be either made to the local crate, or the external crate. This error means you tried to use it for one that's not supported.\n
+                        "`tcx.{}({:?})` is not supported for {} crate;\n\
+                        hint: Queries can be either made to the local crate, or the external crate. \
+                        This error means you tried to use it for one that's not supported.\n\
                         If that's not the case, {} was likely never assigned to a provider function.\n",
                         stringify!($name),
                         key,
@@ -323,7 +372,7 @@ macro_rules! define_callbacks {
                 span: Span,
                 key: query_keys::$name<'tcx>,
                 mode: QueryMode,
-            ) -> Option<query_stored::$name<'tcx>>;)*
+            ) -> Option<$V>;)*
         }
     };
 }
@@ -345,34 +394,34 @@ macro_rules! define_feedable {
         $(impl<'tcx, K: IntoQueryParam<$($K)*> + Copy> TyCtxtFeed<'tcx, K> {
             $(#[$attr])*
             #[inline(always)]
-            pub fn $name(self, value: $V) -> query_stored::$name<'tcx> {
+            pub fn $name(self, value: query_provided::$name<'tcx>) -> $V {
                 let key = self.key().into_query_param();
                 opt_remap_env_constness!([$($modifiers)*][key]);
 
                 let tcx = self.tcx;
-                let cache = &tcx.query_caches.$name;
-
-                let cached = try_get_cached(tcx, cache, &key, copy);
+                let value = query_provided_to_value::$name(tcx, value);
+                let cache = &tcx.query_system.caches.$name;
 
-                match cached {
-                    Ok(old) => {
+                match try_get_cached(tcx, cache, &key) {
+                    Some(old) => {
                         bug!(
                             "Trying to feed an already recorded value for query {} key={key:?}:\nold value: {old:?}\nnew value: {value:?}",
                             stringify!($name),
+                        )
+                    }
+                    None => {
+                        let dep_node = dep_graph::DepNode::construct(tcx, dep_graph::DepKind::$name, &key);
+                        let dep_node_index = tcx.dep_graph.with_feed_task(
+                            dep_node,
+                            tcx,
+                            key,
+                            &value,
+                            hash_result!([$($modifiers)*]),
                         );
+                        cache.complete(key, value, dep_node_index);
+                        value
                     }
-                    Err(()) => (),
                 }
-
-                let dep_node = dep_graph::DepNode::construct(tcx, dep_graph::DepKind::$name, &key);
-                let dep_node_index = tcx.dep_graph.with_feed_task(
-                    dep_node,
-                    tcx,
-                    key,
-                    &value,
-                    hash_result!([$($modifiers)*]),
-                );
-                cache.complete(key, value, dep_node_index)
             }
         })*
     }
@@ -418,6 +467,13 @@ mod sealed {
         }
     }
 
+    impl IntoQueryParam<LocalDefId> for OwnerId {
+        #[inline(always)]
+        fn into_query_param(self) -> LocalDefId {
+            self.def_id
+        }
+    }
+
     impl IntoQueryParam<DefId> for LocalDefId {
         #[inline(always)]
         fn into_query_param(self) -> DefId {
diff --git a/compiler/rustc_middle/src/ty/relate.rs b/compiler/rustc_middle/src/ty/relate.rs
index 65fd8d975..3fc5f5bed 100644
--- a/compiler/rustc_middle/src/ty/relate.rs
+++ b/compiler/rustc_middle/src/ty/relate.rs
@@ -9,7 +9,6 @@ use crate::ty::{self, Expr, ImplSubject, Term, TermKind, Ty, TyCtxt, TypeFoldabl
 use crate::ty::{GenericArg, GenericArgKind, SubstsRef};
 use rustc_hir as ast;
 use rustc_hir::def_id::DefId;
-use rustc_span::DUMMY_SP;
 use rustc_target::spec::abi;
 use std::iter;
 
@@ -106,7 +105,7 @@ pub trait TypeRelation<'tcx>: Sized {
         T: Relate<'tcx>;
 }
 
-pub trait Relate<'tcx>: TypeFoldable<'tcx> + PartialEq + Copy {
+pub trait Relate<'tcx>: TypeFoldable<TyCtxt<'tcx>> + PartialEq + Copy {
     fn relate<R: TypeRelation<'tcx>>(
         relation: &mut R,
         a: Self,
@@ -145,7 +144,7 @@ pub fn relate_substs<'tcx, R: TypeRelation<'tcx>>(
     a_subst: SubstsRef<'tcx>,
     b_subst: SubstsRef<'tcx>,
 ) -> RelateResult<'tcx, SubstsRef<'tcx>> {
-    relation.tcx().mk_substs(iter::zip(a_subst, b_subst).map(|(a, b)| {
+    relation.tcx().mk_substs_from_iter(iter::zip(a_subst, b_subst).map(|(a, b)| {
         relation.relate_with_variance(ty::Invariant, ty::VarianceDiagInfo::default(), a, b)
     }))
 }
@@ -164,8 +163,7 @@ pub fn relate_substs_with_variances<'tcx, R: TypeRelation<'tcx>>(
     let params = iter::zip(a_subst, b_subst).enumerate().map(|(i, (a, b))| {
         let variance = variances[i];
         let variance_info = if variance == ty::Invariant && fetch_ty_for_diag {
-            let ty =
-                *cached_ty.get_or_insert_with(|| tcx.bound_type_of(ty_def_id).subst(tcx, a_subst));
+            let ty = *cached_ty.get_or_insert_with(|| tcx.type_of(ty_def_id).subst(tcx, a_subst));
             ty::VarianceDiagInfo::Invariant { ty, param_index: i.try_into().unwrap() }
         } else {
             ty::VarianceDiagInfo::default()
@@ -173,7 +171,7 @@ pub fn relate_substs_with_variances<'tcx, R: TypeRelation<'tcx>>(
         relation.relate_with_variance(variance, variance_info, a, b)
     });
 
-    tcx.mk_substs(params)
+    tcx.mk_substs_from_iter(params)
 }
 
 impl<'tcx> Relate<'tcx> for ty::FnSig<'tcx> {
@@ -224,7 +222,7 @@ impl<'tcx> Relate<'tcx> for ty::FnSig<'tcx> {
                 r => r,
             });
         Ok(ty::FnSig {
-            inputs_and_output: tcx.mk_type_list(inputs_and_output)?,
+            inputs_and_output: tcx.mk_type_list_from_iter(inputs_and_output)?,
             c_variadic: a.c_variadic,
             unsafety,
             abi,
@@ -354,7 +352,8 @@ impl<'tcx> Relate<'tcx> for GeneratorWitness<'tcx> {
     ) -> RelateResult<'tcx, GeneratorWitness<'tcx>> {
         assert_eq!(a.0.len(), b.0.len());
         let tcx = relation.tcx();
-        let types = tcx.mk_type_list(iter::zip(a.0, b.0).map(|(a, b)| relation.relate(a, b)))?;
+        let types =
+            tcx.mk_type_list_from_iter(iter::zip(a.0, b.0).map(|(a, b)| relation.relate(a, b)))?;
         Ok(GeneratorWitness(types))
     }
 }
@@ -414,7 +413,7 @@ pub fn super_relate_tys<'tcx, R: TypeRelation<'tcx>>(
             bug!("bound types encountered in super_relate_tys")
         }
 
-        (&ty::Error(guar), _) | (_, &ty::Error(guar)) => Ok(tcx.ty_error_with_guaranteed(guar)),
+        (&ty::Error(guar), _) | (_, &ty::Error(guar)) => Ok(tcx.ty_error(guar)),
 
         (&ty::Never, _)
         | (&ty::Char, _)
@@ -443,12 +442,7 @@ pub fn super_relate_tys<'tcx, R: TypeRelation<'tcx>>(
             if a_repr == b_repr =>
         {
             let region_bound = relation.with_cause(Cause::ExistentialRegionBound, |relation| {
-                relation.relate_with_variance(
-                    ty::Contravariant,
-                    ty::VarianceDiagInfo::default(),
-                    a_region,
-                    b_region,
-                )
+                relation.relate(a_region, b_region)
             })?;
             Ok(tcx.mk_dynamic(relation.relate(a_obj, b_obj)?, region_bound, a_repr))
         }
@@ -473,6 +467,16 @@ pub fn super_relate_tys<'tcx, R: TypeRelation<'tcx>>(
             Ok(tcx.mk_generator_witness(types))
         }
 
+        (&ty::GeneratorWitnessMIR(a_id, a_substs), &ty::GeneratorWitnessMIR(b_id, b_substs))
+            if a_id == b_id =>
+        {
+            // All GeneratorWitness types with the same id represent
+            // the (anonymous) type of the same generator expression. So
+            // all of their regions should be equated.
+            let substs = relation.relate(a_substs, b_substs)?;
+            Ok(tcx.mk_generator_witness_mir(a_id, substs))
+        }
+
         (&ty::Closure(a_id, a_substs), &ty::Closure(b_id, b_substs)) if a_id == b_id => {
             // All Closure types with the same id represent
             // the (anonymous) type of the same closure expression. So
@@ -487,12 +491,7 @@ pub fn super_relate_tys<'tcx, R: TypeRelation<'tcx>>(
         }
 
         (&ty::Ref(a_r, a_ty, a_mutbl), &ty::Ref(b_r, b_ty, b_mutbl)) => {
-            let r = relation.relate_with_variance(
-                ty::Contravariant,
-                ty::VarianceDiagInfo::default(),
-                a_r,
-                b_r,
-            )?;
+            let r = relation.relate(a_r, b_r)?;
             let a_mt = ty::TypeAndMut { ty: a_ty, mutbl: a_mutbl };
             let b_mt = ty::TypeAndMut { ty: b_ty, mutbl: b_mutbl };
             let mt = relate_type_and_mut(relation, a_mt, b_mt, a)?;
@@ -502,7 +501,7 @@ pub fn super_relate_tys<'tcx, R: TypeRelation<'tcx>>(
         (&ty::Array(a_t, sz_a), &ty::Array(b_t, sz_b)) => {
             let t = relation.relate(a_t, b_t)?;
             match relation.relate(sz_a, sz_b) {
-                Ok(sz) => Ok(tcx.mk_ty(ty::Array(t, sz))),
+                Ok(sz) => Ok(tcx.mk_array_with_const_len(t, sz)),
                 Err(err) => {
                     // Check whether the lengths are both concrete/known values,
                     // but are unequal, for better diagnostics.
@@ -511,8 +510,8 @@ pub fn super_relate_tys<'tcx, R: TypeRelation<'tcx>>(
                     // we however cannot end up with errors in `Relate` during both
                     // `type_of` and `predicates_of`. This means that evaluating the
                     // constants should not cause cycle errors here.
-                    let sz_a = sz_a.try_eval_usize(tcx, relation.param_env());
-                    let sz_b = sz_b.try_eval_usize(tcx, relation.param_env());
+                    let sz_a = sz_a.try_eval_target_usize(tcx, relation.param_env());
+                    let sz_b = sz_b.try_eval_target_usize(tcx, relation.param_env());
                     match (sz_a, sz_b) {
                         (Some(sz_a_val), Some(sz_b_val)) if sz_a_val != sz_b_val => Err(
                             TypeError::FixedArraySize(expected_found(relation, sz_a_val, sz_b_val)),
@@ -530,7 +529,7 @@ pub fn super_relate_tys<'tcx, R: TypeRelation<'tcx>>(
 
         (&ty::Tuple(as_), &ty::Tuple(bs)) => {
             if as_.len() == bs.len() {
-                Ok(tcx.mk_tup(iter::zip(as_, bs).map(|(a, b)| relation.relate(a, b)))?)
+                Ok(tcx.mk_tup_from_iter(iter::zip(as_, bs).map(|(a, b)| relation.relate(a, b)))?)
             } else if !(as_.is_empty() || bs.is_empty()) {
                 Err(TypeError::TupleSize(expected_found(relation, as_.len(), bs.len())))
             } else {
@@ -594,25 +593,6 @@ pub fn super_relate_consts<'tcx, R: TypeRelation<'tcx>>(
     debug!("{}.super_relate_consts(a = {:?}, b = {:?})", relation.tag(), a, b);
     let tcx = relation.tcx();
 
-    let a_ty;
-    let b_ty;
-    if relation.tcx().features().adt_const_params {
-        a_ty = tcx.normalize_erasing_regions(relation.param_env(), a.ty());
-        b_ty = tcx.normalize_erasing_regions(relation.param_env(), b.ty());
-    } else {
-        a_ty = tcx.erase_regions(a.ty());
-        b_ty = tcx.erase_regions(b.ty());
-    }
-    if a_ty != b_ty {
-        relation.tcx().sess.delay_span_bug(
-            DUMMY_SP,
-            &format!(
-                "cannot relate constants ({:?}, {:?}) of different types: {} != {}",
-                a, b, a_ty, b_ty
-            ),
-        );
-    }
-
     // HACK(const_generics): We still need to eagerly evaluate consts when
     // relating them because during `normalize_param_env_or_error`,
     // we may relate an evaluated constant in a obligation against
@@ -629,6 +609,8 @@ pub fn super_relate_consts<'tcx, R: TypeRelation<'tcx>>(
         b = tcx.expand_abstract_consts(b);
     }
 
+    debug!("{}.super_relate_consts(normed_a = {:?}, normed_b = {:?})", relation.tag(), a, b);
+
     // Currently, the values that can be unified are primitive types,
     // and those that derive both `PartialEq` and `Eq`, corresponding
     // to structural-match types.
@@ -665,36 +647,34 @@ pub fn super_relate_consts<'tcx, R: TypeRelation<'tcx>>(
 
             // FIXME(generic_const_exprs): is it possible to relate two consts which are not identical
             // exprs? Should we care about that?
+            // FIXME(generic_const_exprs): relating the `ty()`s is a little weird since it is supposed to
+            // ICE If they mismatch. Unfortunately `ConstKind::Expr` is a little special and can be thought
+            // of as being generic over the argument types, however this is implicit so these types don't get
+            // related when we relate the substs of the item this const arg is for.
             let expr = match (ae, be) {
-                (Expr::Binop(a_op, al, ar), Expr::Binop(b_op, bl, br))
-                    if a_op == b_op && al.ty() == bl.ty() && ar.ty() == br.ty() =>
-                {
+                (Expr::Binop(a_op, al, ar), Expr::Binop(b_op, bl, br)) if a_op == b_op => {
+                    r.relate(al.ty(), bl.ty())?;
+                    r.relate(ar.ty(), br.ty())?;
                     Expr::Binop(a_op, r.consts(al, bl)?, r.consts(ar, br)?)
                 }
-                (Expr::UnOp(a_op, av), Expr::UnOp(b_op, bv))
-                    if a_op == b_op && av.ty() == bv.ty() =>
-                {
+                (Expr::UnOp(a_op, av), Expr::UnOp(b_op, bv)) if a_op == b_op => {
+                    r.relate(av.ty(), bv.ty())?;
                     Expr::UnOp(a_op, r.consts(av, bv)?)
                 }
-                (Expr::Cast(ak, av, at), Expr::Cast(bk, bv, bt))
-                    if ak == bk && av.ty() == bv.ty() =>
-                {
+                (Expr::Cast(ak, av, at), Expr::Cast(bk, bv, bt)) if ak == bk => {
+                    r.relate(av.ty(), bv.ty())?;
                     Expr::Cast(ak, r.consts(av, bv)?, r.tys(at, bt)?)
                 }
                 (Expr::FunctionCall(af, aa), Expr::FunctionCall(bf, ba))
-                    if aa.len() == ba.len()
-                        && af.ty() == bf.ty()
-                        && aa
-                            .iter()
-                            .zip(ba.iter())
-                            .all(|(a_arg, b_arg)| a_arg.ty() == b_arg.ty()) =>
+                    if aa.len() == ba.len() =>
                 {
+                    r.relate(af.ty(), bf.ty())?;
                     let func = r.consts(af, bf)?;
                     let mut related_args = Vec::with_capacity(aa.len());
                     for (a_arg, b_arg) in aa.iter().zip(ba.iter()) {
                         related_args.push(r.consts(a_arg, b_arg)?);
                     }
-                    let related_args = tcx.mk_const_list(related_args.iter());
+                    let related_args = tcx.mk_const_list(&related_args);
                     Expr::FunctionCall(func, related_args)
                 }
                 _ => return Err(TypeError::ConstMismatch(expected_found(r, a, b))),
@@ -741,7 +721,7 @@ impl<'tcx> Relate<'tcx> for &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>> {
                 _ => Err(TypeError::ExistentialMismatch(expected_found(relation, a, b))),
             }
         });
-        tcx.mk_poly_existential_predicates(v)
+        tcx.mk_poly_existential_predicates_from_iter(v)
     }
 }
 
diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs
index 7d4d35b7f..ef643531b 100644
--- a/compiler/rustc_middle/src/ty/structural_impls.rs
+++ b/compiler/rustc_middle/src/ty/structural_impls.rs
@@ -1,6 +1,7 @@
-//! This module contains implements of the `Lift` and `TypeFoldable`
-//! traits for various types in the Rust compiler. Most are written by
-//! hand, though we've recently added some macros and proc-macros to help with the tedium.
+//! This module contains implementations of the `Lift`, `TypeFoldable` and
+//! `TypeVisitable` traits for various types in the Rust compiler. Most are
+//! written by hand, though we've recently added some macros and proc-macros
+//! to help with the tedium.
 
 use crate::mir::interpret;
 use crate::mir::{Field, ProjectionKind};
@@ -8,12 +9,11 @@ use crate::ty::fold::{FallibleTypeFolder, TypeFoldable, TypeSuperFoldable};
 use crate::ty::print::{with_no_trimmed_paths, FmtPrinter, Printer};
 use crate::ty::visit::{TypeSuperVisitable, TypeVisitable, TypeVisitor};
 use crate::ty::{self, AliasTy, InferConst, Lift, Term, TermKind, Ty, TyCtxt};
-use rustc_data_structures::functor::IdFunctor;
 use rustc_hir::def::Namespace;
 use rustc_index::vec::{Idx, IndexVec};
+use rustc_target::abi::TyAndLayout;
 
 use std::fmt;
-use std::mem::ManuallyDrop;
 use std::ops::ControlFlow;
 use std::rc::Rc;
 use std::sync::Arc;
@@ -147,6 +147,7 @@ impl<'tcx> fmt::Debug for ty::Predicate<'tcx> {
 impl<'tcx> fmt::Debug for ty::Clause<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match *self {
+            ty::Clause::ConstArgHasType(ct, ty) => write!(f, "ConstArgHasType({ct:?}, {ty:?})"),
             ty::Clause::Trait(ref a) => a.fmt(f),
             ty::Clause::RegionOutlives(ref pair) => pair.fmt(f),
             ty::Clause::TypeOutlives(ref pair) => pair.fmt(f),
@@ -176,6 +177,7 @@ impl<'tcx> fmt::Debug for ty::PredicateKind<'tcx> {
                 write!(f, "TypeWellFormedFromEnv({:?})", ty)
             }
             ty::PredicateKind::Ambiguous => write!(f, "Ambiguous"),
+            ty::PredicateKind::AliasEq(t1, t2) => write!(f, "AliasEq({t1:?}, {t2:?})"),
         }
     }
 }
@@ -193,16 +195,27 @@ impl<'tcx> fmt::Debug for AliasTy<'tcx> {
 // Atomic structs
 //
 // For things that don't carry any arena-allocated data (and are
-// copy...), just add them to this list.
+// copy...), just add them to one of these lists as appropriat.
 
-TrivialTypeTraversalAndLiftImpls! {
+// For things for which the type library provides traversal implementations
+// for all Interners, we only need to provide a Lift implementation:
+CloneLiftImpls! {
     (),
     bool,
     usize,
-    ::rustc_target::abi::VariantIdx,
+    u16,
     u32,
     u64,
     String,
+    rustc_type_ir::DebruijnIndex,
+}
+
+// For things about which the type library does not know, or does not
+// provide any traversal implementations, we need to provide both a Lift
+// implementation and traversal implementations (the latter only for
+// TyCtxt<'_> interners).
+TrivialTypeTraversalAndLiftImpls! {
+    ::rustc_target::abi::VariantIdx,
     crate::middle::region::Scope,
     crate::ty::FloatTy,
     ::rustc_ast::InlineAsmOptions,
@@ -238,6 +251,7 @@ TrivialTypeTraversalAndLiftImpls! {
     crate::ty::AssocKind,
     crate::ty::AliasKind,
     crate::ty::Placeholder<crate::ty::BoundRegionKind>,
+    crate::ty::Placeholder<crate::ty::BoundTyKind>,
     crate::ty::ClosureKind,
     crate::ty::FreeRegion,
     crate::ty::InferTy,
@@ -249,11 +263,11 @@ TrivialTypeTraversalAndLiftImpls! {
     crate::ty::UniverseIndex,
     crate::ty::Variance,
     ::rustc_span::Span,
+    ::rustc_span::symbol::Ident,
     ::rustc_errors::ErrorGuaranteed,
     Field,
     interpret::Scalar,
     rustc_target::abi::Size,
-    rustc_type_ir::DebruijnIndex,
     ty::BoundVar,
     ty::Placeholder<ty::BoundVar>,
 }
@@ -356,281 +370,106 @@ impl<'a, 'tcx> Lift<'tcx> for ty::ParamEnv<'a> {
 }
 
 ///////////////////////////////////////////////////////////////////////////
-// TypeFoldable implementations.
+// Traversal implementations.
 
 /// AdtDefs are basically the same as a DefId.
-impl<'tcx> TypeFoldable<'tcx> for ty::AdtDef<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, _folder: &mut F) -> Result<Self, F::Error> {
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for ty::AdtDef<'tcx> {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
+        _folder: &mut F,
+    ) -> Result<Self, F::Error> {
         Ok(self)
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx> for ty::AdtDef<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, _visitor: &mut V) -> ControlFlow<V::BreakTy> {
+impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for ty::AdtDef<'tcx> {
+    fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(
+        &self,
+        _visitor: &mut V,
+    ) -> ControlFlow<V::BreakTy> {
         ControlFlow::Continue(())
     }
 }
 
-impl<'tcx, T: TypeFoldable<'tcx>, U: TypeFoldable<'tcx>> TypeFoldable<'tcx> for (T, U) {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(
+impl<'tcx, T: TypeFoldable<TyCtxt<'tcx>>> TypeFoldable<TyCtxt<'tcx>> for ty::Binder<'tcx, T> {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
         self,
         folder: &mut F,
-    ) -> Result<(T, U), F::Error> {
-        Ok((self.0.try_fold_with(folder)?, self.1.try_fold_with(folder)?))
+    ) -> Result<Self, F::Error> {
+        folder.try_fold_binder(self)
     }
 }
 
-impl<'tcx, T: TypeVisitable<'tcx>, U: TypeVisitable<'tcx>> TypeVisitable<'tcx> for (T, U) {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.0.visit_with(visitor)?;
-        self.1.visit_with(visitor)
+impl<'tcx, T: TypeVisitable<TyCtxt<'tcx>>> TypeVisitable<TyCtxt<'tcx>> for ty::Binder<'tcx, T> {
+    fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+        visitor.visit_binder(self)
     }
 }
 
-impl<'tcx, A: TypeFoldable<'tcx>, B: TypeFoldable<'tcx>, C: TypeFoldable<'tcx>> TypeFoldable<'tcx>
-    for (A, B, C)
-{
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(
+impl<'tcx, T: TypeFoldable<TyCtxt<'tcx>>> TypeSuperFoldable<TyCtxt<'tcx>> for ty::Binder<'tcx, T> {
+    fn try_super_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
         self,
         folder: &mut F,
-    ) -> Result<(A, B, C), F::Error> {
-        Ok((
-            self.0.try_fold_with(folder)?,
-            self.1.try_fold_with(folder)?,
-            self.2.try_fold_with(folder)?,
-        ))
+    ) -> Result<Self, F::Error> {
+        self.try_map_bound(|ty| ty.try_fold_with(folder))
     }
 }
 
-impl<'tcx, A: TypeVisitable<'tcx>, B: TypeVisitable<'tcx>, C: TypeVisitable<'tcx>>
-    TypeVisitable<'tcx> for (A, B, C)
+impl<'tcx, T: TypeVisitable<TyCtxt<'tcx>>> TypeSuperVisitable<TyCtxt<'tcx>>
+    for ty::Binder<'tcx, T>
 {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.0.visit_with(visitor)?;
-        self.1.visit_with(visitor)?;
-        self.2.visit_with(visitor)
+    fn super_visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(
+        &self,
+        visitor: &mut V,
+    ) -> ControlFlow<V::BreakTy> {
+        self.as_ref().skip_binder().visit_with(visitor)
     }
 }
 
-EnumTypeTraversalImpl! {
-    impl<'tcx, T> TypeFoldable<'tcx> for Option<T> {
-        (Some)(a),
-        (None),
-    } where T: TypeFoldable<'tcx>
-}
-EnumTypeTraversalImpl! {
-    impl<'tcx, T> TypeVisitable<'tcx> for Option<T> {
-        (Some)(a),
-        (None),
-    } where T: TypeVisitable<'tcx>
-}
-
-EnumTypeTraversalImpl! {
-    impl<'tcx, T, E> TypeFoldable<'tcx> for Result<T, E> {
-        (Ok)(a),
-        (Err)(a),
-    } where T: TypeFoldable<'tcx>, E: TypeFoldable<'tcx>,
-}
-EnumTypeTraversalImpl! {
-    impl<'tcx, T, E> TypeVisitable<'tcx> for Result<T, E> {
-        (Ok)(a),
-        (Err)(a),
-    } where T: TypeVisitable<'tcx>, E: TypeVisitable<'tcx>,
-}
-
-impl<'tcx, T: TypeFoldable<'tcx>> TypeFoldable<'tcx> for Rc<T> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(
-        mut self,
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>> {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
         folder: &mut F,
     ) -> Result<Self, F::Error> {
-        // We merely want to replace the contained `T`, if at all possible,
-        // so that we don't needlessly allocate a new `Rc` or indeed clone
-        // the contained type.
-        unsafe {
-            // First step is to ensure that we have a unique reference to
-            // the contained type, which `Rc::make_mut` will accomplish (by
-            // allocating a new `Rc` and cloning the `T` only if required).
-            // This is done *before* casting to `Rc<ManuallyDrop<T>>` so that
-            // panicking during `make_mut` does not leak the `T`.
-            Rc::make_mut(&mut self);
-
-            // Casting to `Rc<ManuallyDrop<T>>` is safe because `ManuallyDrop`
-            // is `repr(transparent)`.
-            let ptr = Rc::into_raw(self).cast::<ManuallyDrop<T>>();
-            let mut unique = Rc::from_raw(ptr);
-
-            // Call to `Rc::make_mut` above guarantees that `unique` is the
-            // sole reference to the contained value, so we can avoid doing
-            // a checked `get_mut` here.
-            let slot = Rc::get_mut_unchecked(&mut unique);
-
-            // Semantically move the contained type out from `unique`, fold
-            // it, then move the folded value back into `unique`. Should
-            // folding fail, `ManuallyDrop` ensures that the "moved-out"
-            // value is not re-dropped.
-            let owned = ManuallyDrop::take(slot);
-            let folded = owned.try_fold_with(folder)?;
-            *slot = ManuallyDrop::new(folded);
-
-            // Cast back to `Rc<T>`.
-            Ok(Rc::from_raw(Rc::into_raw(unique).cast()))
-        }
+        ty::util::fold_list(self, folder, |tcx, v| tcx.mk_poly_existential_predicates(v))
     }
 }
 
-impl<'tcx, T: TypeVisitable<'tcx>> TypeVisitable<'tcx> for Rc<T> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        (**self).visit_with(visitor)
-    }
-}
-
-impl<'tcx, T: TypeFoldable<'tcx>> TypeFoldable<'tcx> for Arc<T> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(
-        mut self,
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx ty::List<ty::Const<'tcx>> {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
         folder: &mut F,
     ) -> Result<Self, F::Error> {
-        // We merely want to replace the contained `T`, if at all possible,
-        // so that we don't needlessly allocate a new `Arc` or indeed clone
-        // the contained type.
-        unsafe {
-            // First step is to ensure that we have a unique reference to
-            // the contained type, which `Arc::make_mut` will accomplish (by
-            // allocating a new `Arc` and cloning the `T` only if required).
-            // This is done *before* casting to `Arc<ManuallyDrop<T>>` so that
-            // panicking during `make_mut` does not leak the `T`.
-            Arc::make_mut(&mut self);
-
-            // Casting to `Arc<ManuallyDrop<T>>` is safe because `ManuallyDrop`
-            // is `repr(transparent)`.
-            let ptr = Arc::into_raw(self).cast::<ManuallyDrop<T>>();
-            let mut unique = Arc::from_raw(ptr);
-
-            // Call to `Arc::make_mut` above guarantees that `unique` is the
-            // sole reference to the contained value, so we can avoid doing
-            // a checked `get_mut` here.
-            let slot = Arc::get_mut_unchecked(&mut unique);
-
-            // Semantically move the contained type out from `unique`, fold
-            // it, then move the folded value back into `unique`. Should
-            // folding fail, `ManuallyDrop` ensures that the "moved-out"
-            // value is not re-dropped.
-            let owned = ManuallyDrop::take(slot);
-            let folded = owned.try_fold_with(folder)?;
-            *slot = ManuallyDrop::new(folded);
-
-            // Cast back to `Arc<T>`.
-            Ok(Arc::from_raw(Arc::into_raw(unique).cast()))
-        }
-    }
-}
-
-impl<'tcx, T: TypeVisitable<'tcx>> TypeVisitable<'tcx> for Arc<T> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        (**self).visit_with(visitor)
-    }
-}
-
-impl<'tcx, T: TypeFoldable<'tcx>> TypeFoldable<'tcx> for Box<T> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        self.try_map_id(|value| value.try_fold_with(folder))
-    }
-}
-
-impl<'tcx, T: TypeVisitable<'tcx>> TypeVisitable<'tcx> for Box<T> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        (**self).visit_with(visitor)
-    }
-}
-
-impl<'tcx, T: TypeFoldable<'tcx>> TypeFoldable<'tcx> for Vec<T> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        self.try_map_id(|t| t.try_fold_with(folder))
-    }
-}
-
-impl<'tcx, T: TypeVisitable<'tcx>> TypeVisitable<'tcx> for Vec<T> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.iter().try_for_each(|t| t.visit_with(visitor))
-    }
-}
-
-impl<'tcx, T: TypeVisitable<'tcx>> TypeVisitable<'tcx> for &[T] {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.iter().try_for_each(|t| t.visit_with(visitor))
-    }
-}
-
-impl<'tcx, T: TypeFoldable<'tcx>> TypeFoldable<'tcx> for Box<[T]> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        self.try_map_id(|t| t.try_fold_with(folder))
-    }
-}
-
-impl<'tcx, T: TypeVisitable<'tcx>> TypeVisitable<'tcx> for Box<[T]> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.iter().try_for_each(|t| t.visit_with(visitor))
+        ty::util::fold_list(self, folder, |tcx, v| tcx.mk_const_list(v))
     }
 }
 
-impl<'tcx, T: TypeFoldable<'tcx>> TypeFoldable<'tcx> for ty::Binder<'tcx, T> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        folder.try_fold_binder(self)
-    }
-}
-
-impl<'tcx, T: TypeVisitable<'tcx>> TypeVisitable<'tcx> for ty::Binder<'tcx, T> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        visitor.visit_binder(self)
-    }
-}
-
-impl<'tcx, T: TypeFoldable<'tcx>> TypeSuperFoldable<'tcx> for ty::Binder<'tcx, T> {
-    fn try_super_fold_with<F: FallibleTypeFolder<'tcx>>(
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx ty::List<ProjectionKind> {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
         self,
         folder: &mut F,
     ) -> Result<Self, F::Error> {
-        self.try_map_bound(|ty| ty.try_fold_with(folder))
+        ty::util::fold_list(self, folder, |tcx, v| tcx.mk_projs(v))
     }
 }
 
-impl<'tcx, T: TypeVisitable<'tcx>> TypeSuperVisitable<'tcx> for ty::Binder<'tcx, T> {
-    fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.as_ref().skip_binder().visit_with(visitor)
-    }
-}
-
-impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        ty::util::fold_list(self, folder, |tcx, v| tcx.intern_poly_existential_predicates(v))
-    }
-}
-
-impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<ty::Const<'tcx>> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        ty::util::fold_list(self, folder, |tcx, v| tcx.mk_const_list(v.iter()))
-    }
-}
-
-impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<ProjectionKind> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        ty::util::fold_list(self, folder, |tcx, v| tcx.intern_projs(v))
-    }
-}
-
-impl<'tcx> TypeFoldable<'tcx> for Ty<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for Ty<'tcx> {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self, F::Error> {
         folder.try_fold_ty(self)
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx> for Ty<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for Ty<'tcx> {
+    fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
         visitor.visit_ty(*self)
     }
 }
 
-impl<'tcx> TypeSuperFoldable<'tcx> for Ty<'tcx> {
-    fn try_super_fold_with<F: FallibleTypeFolder<'tcx>>(
+impl<'tcx> TypeSuperFoldable<TyCtxt<'tcx>> for Ty<'tcx> {
+    fn try_super_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
         self,
         folder: &mut F,
     ) -> Result<Self, F::Error> {
@@ -654,6 +493,9 @@ impl<'tcx> TypeSuperFoldable<'tcx> for Ty<'tcx> {
                 ty::Generator(did, substs.try_fold_with(folder)?, movability)
             }
             ty::GeneratorWitness(types) => ty::GeneratorWitness(types.try_fold_with(folder)?),
+            ty::GeneratorWitnessMIR(did, substs) => {
+                ty::GeneratorWitnessMIR(did, substs.try_fold_with(folder)?)
+            }
             ty::Closure(did, substs) => ty::Closure(did, substs.try_fold_with(folder)?),
             ty::Alias(kind, data) => ty::Alias(kind, data.try_fold_with(folder)?),
 
@@ -672,12 +514,15 @@ impl<'tcx> TypeSuperFoldable<'tcx> for Ty<'tcx> {
             | ty::Foreign(..) => return Ok(self),
         };
 
-        Ok(if *self.kind() == kind { self } else { folder.tcx().mk_ty(kind) })
+        Ok(if *self.kind() == kind { self } else { folder.interner().mk_ty_from_kind(kind) })
     }
 }
 
-impl<'tcx> TypeSuperVisitable<'tcx> for Ty<'tcx> {
-    fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+impl<'tcx> TypeSuperVisitable<TyCtxt<'tcx>> for Ty<'tcx> {
+    fn super_visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(
+        &self,
+        visitor: &mut V,
+    ) -> ControlFlow<V::BreakTy> {
         match self.kind() {
             ty::RawPtr(ref tm) => tm.visit_with(visitor),
             ty::Array(typ, sz) => {
@@ -699,6 +544,7 @@ impl<'tcx> TypeSuperVisitable<'tcx> for Ty<'tcx> {
             }
             ty::Generator(_did, ref substs, _) => substs.visit_with(visitor),
             ty::GeneratorWitness(ref types) => types.visit_with(visitor),
+            ty::GeneratorWitnessMIR(_did, ref substs) => substs.visit_with(visitor),
             ty::Closure(_did, ref substs) => substs.visit_with(visitor),
             ty::Alias(_, ref data) => data.visit_with(visitor),
 
@@ -719,20 +565,23 @@ impl<'tcx> TypeSuperVisitable<'tcx> for Ty<'tcx> {
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for ty::Region<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for ty::Region<'tcx> {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self, F::Error> {
         folder.try_fold_region(self)
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx> for ty::Region<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for ty::Region<'tcx> {
+    fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
         visitor.visit_region(*self)
     }
 }
 
-impl<'tcx> TypeSuperFoldable<'tcx> for ty::Region<'tcx> {
-    fn try_super_fold_with<F: FallibleTypeFolder<'tcx>>(
+impl<'tcx> TypeSuperFoldable<TyCtxt<'tcx>> for ty::Region<'tcx> {
+    fn try_super_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
         self,
         _folder: &mut F,
     ) -> Result<Self, F::Error> {
@@ -740,116 +589,127 @@ impl<'tcx> TypeSuperFoldable<'tcx> for ty::Region<'tcx> {
     }
 }
 
-impl<'tcx> TypeSuperVisitable<'tcx> for ty::Region<'tcx> {
-    fn super_visit_with<V: TypeVisitor<'tcx>>(&self, _visitor: &mut V) -> ControlFlow<V::BreakTy> {
+impl<'tcx> TypeSuperVisitable<TyCtxt<'tcx>> for ty::Region<'tcx> {
+    fn super_visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(
+        &self,
+        _visitor: &mut V,
+    ) -> ControlFlow<V::BreakTy> {
         ControlFlow::Continue(())
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for ty::Predicate<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for ty::Predicate<'tcx> {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self, F::Error> {
         folder.try_fold_predicate(self)
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx> for ty::Predicate<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for ty::Predicate<'tcx> {
+    fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
         visitor.visit_predicate(*self)
     }
-
-    #[inline]
-    fn has_vars_bound_at_or_above(&self, binder: ty::DebruijnIndex) -> bool {
-        self.outer_exclusive_binder() > binder
-    }
-
-    #[inline]
-    fn has_type_flags(&self, flags: ty::TypeFlags) -> bool {
-        self.flags().intersects(flags)
-    }
 }
 
-impl<'tcx> TypeSuperFoldable<'tcx> for ty::Predicate<'tcx> {
-    fn try_super_fold_with<F: FallibleTypeFolder<'tcx>>(
+impl<'tcx> TypeSuperFoldable<TyCtxt<'tcx>> for ty::Predicate<'tcx> {
+    fn try_super_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
         self,
         folder: &mut F,
     ) -> Result<Self, F::Error> {
         let new = self.kind().try_fold_with(folder)?;
-        Ok(folder.tcx().reuse_or_mk_predicate(self, new))
+        Ok(folder.interner().reuse_or_mk_predicate(self, new))
     }
 }
 
-impl<'tcx> TypeSuperVisitable<'tcx> for ty::Predicate<'tcx> {
-    fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+impl<'tcx> TypeSuperVisitable<TyCtxt<'tcx>> for ty::Predicate<'tcx> {
+    fn super_visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(
+        &self,
+        visitor: &mut V,
+    ) -> ControlFlow<V::BreakTy> {
         self.kind().visit_with(visitor)
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<ty::Predicate<'tcx>> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        ty::util::fold_list(self, folder, |tcx, v| tcx.intern_predicates(v))
-    }
-}
-
-impl<'tcx, T: TypeFoldable<'tcx>, I: Idx> TypeFoldable<'tcx> for IndexVec<I, T> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        self.try_map_id(|x| x.try_fold_with(folder))
-    }
-}
-
-impl<'tcx, T: TypeVisitable<'tcx>, I: Idx> TypeVisitable<'tcx> for IndexVec<I, T> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.iter().try_for_each(|t| t.visit_with(visitor))
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx ty::List<ty::Predicate<'tcx>> {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self, F::Error> {
+        ty::util::fold_list(self, folder, |tcx, v| tcx.mk_predicates(v))
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for ty::Const<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for ty::Const<'tcx> {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self, F::Error> {
         folder.try_fold_const(self)
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx> for ty::Const<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for ty::Const<'tcx> {
+    fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
         visitor.visit_const(*self)
     }
 }
 
-impl<'tcx> TypeSuperFoldable<'tcx> for ty::Const<'tcx> {
-    fn try_super_fold_with<F: FallibleTypeFolder<'tcx>>(
+impl<'tcx> TypeSuperFoldable<TyCtxt<'tcx>> for ty::Const<'tcx> {
+    fn try_super_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
         self,
         folder: &mut F,
     ) -> Result<Self, F::Error> {
         let ty = self.ty().try_fold_with(folder)?;
         let kind = self.kind().try_fold_with(folder)?;
         if ty != self.ty() || kind != self.kind() {
-            Ok(folder.tcx().mk_const(kind, ty))
+            Ok(folder.interner().mk_const(kind, ty))
         } else {
             Ok(self)
         }
     }
 }
 
-impl<'tcx> TypeSuperVisitable<'tcx> for ty::Const<'tcx> {
-    fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+impl<'tcx> TypeSuperVisitable<TyCtxt<'tcx>> for ty::Const<'tcx> {
+    fn super_visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(
+        &self,
+        visitor: &mut V,
+    ) -> ControlFlow<V::BreakTy> {
         self.ty().visit_with(visitor)?;
         self.kind().visit_with(visitor)
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for InferConst<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, _folder: &mut F) -> Result<Self, F::Error> {
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for InferConst<'tcx> {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
+        _folder: &mut F,
+    ) -> Result<Self, F::Error> {
         Ok(self)
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx> for InferConst<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, _visitor: &mut V) -> ControlFlow<V::BreakTy> {
+impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for InferConst<'tcx> {
+    fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(
+        &self,
+        _visitor: &mut V,
+    ) -> ControlFlow<V::BreakTy> {
         ControlFlow::Continue(())
     }
 }
 
-impl<'tcx> TypeSuperVisitable<'tcx> for ty::UnevaluatedConst<'tcx> {
-    fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+impl<'tcx> TypeSuperVisitable<TyCtxt<'tcx>> for ty::UnevaluatedConst<'tcx> {
+    fn super_visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(
+        &self,
+        visitor: &mut V,
+    ) -> ControlFlow<V::BreakTy> {
         self.substs.visit_with(visitor)
     }
 }
+
+impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for TyAndLayout<'tcx, Ty<'tcx>> {
+    fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+        visitor.visit_ty(self.ty)
+    }
+}
diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs
index 91a7d5d38..e6a73e8bb 100644
--- a/compiler/rustc_middle/src/ty/sty.rs
+++ b/compiler/rustc_middle/src/ty/sty.rs
@@ -8,7 +8,7 @@ use crate::ty::visit::ValidateBoundVars;
 use crate::ty::InferTy::*;
 use crate::ty::{
     self, AdtDef, DefIdTree, Discr, FallibleTypeFolder, Term, Ty, TyCtxt, TypeFlags, TypeFoldable,
-    TypeSuperFoldable, TypeSuperVisitable, TypeVisitable, TypeVisitor,
+    TypeSuperFoldable, TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor,
 };
 use crate::ty::{List, ParamEnv};
 use hir::def::DefKind;
@@ -107,6 +107,15 @@ impl BoundRegionKind {
             _ => None,
         }
     }
+
+    pub fn expect_anon(&self) -> u32 {
+        match *self {
+            BoundRegionKind::BrNamed(_, _) | BoundRegionKind::BrEnv => {
+                bug!("expected anon region: {self:?}")
+            }
+            BoundRegionKind::BrAnon(idx, _) => idx,
+        }
+    }
 }
 
 pub trait Article {
@@ -250,7 +259,7 @@ impl<'tcx> ClosureSubsts<'tcx> {
         parts: ClosureSubstsParts<'tcx, Ty<'tcx>>,
     ) -> ClosureSubsts<'tcx> {
         ClosureSubsts {
-            substs: tcx.mk_substs(
+            substs: tcx.mk_substs_from_iter(
                 parts.parent_substs.iter().copied().chain(
                     [parts.closure_kind_ty, parts.closure_sig_as_fn_ptr_ty, parts.tupled_upvars_ty]
                         .iter()
@@ -377,7 +386,7 @@ impl<'tcx> GeneratorSubsts<'tcx> {
         parts: GeneratorSubstsParts<'tcx, Ty<'tcx>>,
     ) -> GeneratorSubsts<'tcx> {
         GeneratorSubsts {
-            substs: tcx.mk_substs(
+            substs: tcx.mk_substs_from_iter(
                 parts.parent_substs.iter().copied().chain(
                     [
                         parts.resume_ty,
@@ -568,12 +577,12 @@ impl<'tcx> GeneratorSubsts<'tcx> {
         self,
         def_id: DefId,
         tcx: TyCtxt<'tcx>,
-    ) -> impl Iterator<Item = impl Iterator<Item = Ty<'tcx>> + Captures<'tcx>> {
+    ) -> impl Iterator<Item: Iterator<Item = Ty<'tcx>> + Captures<'tcx>> {
         let layout = tcx.generator_layout(def_id).unwrap();
         layout.variant_fields.iter().map(move |variant| {
-            variant
-                .iter()
-                .map(move |field| ty::EarlyBinder(layout.field_tys[*field]).subst(tcx, self.substs))
+            variant.iter().map(move |field| {
+                ty::EarlyBinder(layout.field_tys[*field].ty).subst(tcx, self.substs)
+            })
         })
     }
 
@@ -655,7 +664,7 @@ impl<'tcx> InlineConstSubsts<'tcx> {
         parts: InlineConstSubstsParts<'tcx, Ty<'tcx>>,
     ) -> InlineConstSubsts<'tcx> {
         InlineConstSubsts {
-            substs: tcx.mk_substs(
+            substs: tcx.mk_substs_from_iter(
                 parts.parent_substs.iter().copied().chain(std::iter::once(parts.ty.into())),
             ),
         }
@@ -853,7 +862,7 @@ impl<'tcx> TraitRef<'tcx> {
         substs: SubstsRef<'tcx>,
     ) -> ty::TraitRef<'tcx> {
         let defs = tcx.generics_of(trait_id);
-        tcx.mk_trait_ref(trait_id, tcx.intern_substs(&substs[..defs.params.len()]))
+        tcx.mk_trait_ref(trait_id, tcx.mk_substs(&substs[..defs.params.len()]))
     }
 }
 
@@ -899,7 +908,7 @@ impl<'tcx> ExistentialTraitRef<'tcx> {
 
         ty::ExistentialTraitRef {
             def_id: trait_ref.def_id,
-            substs: tcx.intern_substs(&trait_ref.substs[1..]),
+            substs: tcx.mk_substs(&trait_ref.substs[1..]),
         }
     }
 
@@ -983,7 +992,7 @@ pub struct Binder<'tcx, T>(T, &'tcx List<BoundVariableKind>);
 
 impl<'tcx, T> Binder<'tcx, T>
 where
-    T: TypeVisitable<'tcx>,
+    T: TypeVisitable<TyCtxt<'tcx>>,
 {
     /// Wraps `value` in a binder, asserting that `value` does not
     /// contain any bound vars that would be bound by the
@@ -1051,14 +1060,14 @@ impl<'tcx, T> Binder<'tcx, T> {
         Binder(value, self.1)
     }
 
-    pub fn map_bound_ref<F, U: TypeVisitable<'tcx>>(&self, f: F) -> Binder<'tcx, U>
+    pub fn map_bound_ref<F, U: TypeVisitable<TyCtxt<'tcx>>>(&self, f: F) -> Binder<'tcx, U>
     where
         F: FnOnce(&T) -> U,
     {
         self.as_ref().map_bound(f)
     }
 
-    pub fn map_bound<F, U: TypeVisitable<'tcx>>(self, f: F) -> Binder<'tcx, U>
+    pub fn map_bound<F, U: TypeVisitable<TyCtxt<'tcx>>>(self, f: F) -> Binder<'tcx, U>
     where
         F: FnOnce(T) -> U,
     {
@@ -1070,7 +1079,10 @@ impl<'tcx, T> Binder<'tcx, T> {
         Binder(value, self.1)
     }
 
-    pub fn try_map_bound<F, U: TypeVisitable<'tcx>, E>(self, f: F) -> Result<Binder<'tcx, U>, E>
+    pub fn try_map_bound<F, U: TypeVisitable<TyCtxt<'tcx>>, E>(
+        self,
+        f: F,
+    ) -> Result<Binder<'tcx, U>, E>
     where
         F: FnOnce(T) -> Result<U, E>,
     {
@@ -1093,7 +1105,7 @@ impl<'tcx, T> Binder<'tcx, T> {
     /// in `bind`. This may be (debug) asserted in the future.
     pub fn rebind<U>(&self, value: U) -> Binder<'tcx, U>
     where
-        U: TypeVisitable<'tcx>,
+        U: TypeVisitable<TyCtxt<'tcx>>,
     {
         if cfg!(debug_assertions) {
             let mut validator = ValidateBoundVars::new(self.bound_vars());
@@ -1114,7 +1126,7 @@ impl<'tcx, T> Binder<'tcx, T> {
     /// would not be that useful.)
     pub fn no_bound_vars(self) -> Option<T>
     where
-        T: TypeVisitable<'tcx>,
+        T: TypeVisitable<TyCtxt<'tcx>>,
     {
         if self.0.has_escaping_bound_vars() { None } else { Some(self.skip_binder()) }
     }
@@ -1153,16 +1165,16 @@ struct SkipBindersAt<'tcx> {
     index: ty::DebruijnIndex,
 }
 
-impl<'tcx> FallibleTypeFolder<'tcx> for SkipBindersAt<'tcx> {
+impl<'tcx> FallibleTypeFolder<TyCtxt<'tcx>> for SkipBindersAt<'tcx> {
     type Error = ();
 
-    fn tcx(&self) -> TyCtxt<'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
     fn try_fold_binder<T>(&mut self, t: Binder<'tcx, T>) -> Result<Binder<'tcx, T>, Self::Error>
     where
-        T: ty::TypeFoldable<'tcx>,
+        T: ty::TypeFoldable<TyCtxt<'tcx>>,
     {
         self.index.shift_in(1);
         let value = t.try_map_bound(|t| t.try_fold_with(self));
@@ -1177,7 +1189,7 @@ impl<'tcx> FallibleTypeFolder<'tcx> for SkipBindersAt<'tcx> {
             if index == self.index {
                 Err(())
             } else {
-                Ok(self.tcx().mk_ty(ty::Bound(index.shifted_out(1), bv)))
+                Ok(self.interner().mk_bound(index.shifted_out(1), bv))
             }
         } else {
             ty.try_super_fold_with(self)
@@ -1191,7 +1203,7 @@ impl<'tcx> FallibleTypeFolder<'tcx> for SkipBindersAt<'tcx> {
             if index == self.index {
                 Err(())
             } else {
-                Ok(self.tcx().mk_region(ty::ReLateBound(index.shifted_out(1), bv)))
+                Ok(self.interner().mk_re_late_bound(index.shifted_out(1), bv))
             }
         } else {
             r.try_super_fold_with(self)
@@ -1205,7 +1217,7 @@ impl<'tcx> FallibleTypeFolder<'tcx> for SkipBindersAt<'tcx> {
             if index == self.index {
                 Err(())
             } else {
-                Ok(self.tcx().mk_const(
+                Ok(self.interner().mk_const(
                     ty::ConstKind::Bound(index.shifted_out(1), bv),
                     ct.ty().try_fold_with(self)?,
                 ))
@@ -1266,7 +1278,7 @@ impl<'tcx> AliasTy<'tcx> {
     }
 
     pub fn to_ty(self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
-        tcx.mk_ty(ty::Alias(self.kind(tcx), self))
+        tcx.mk_alias(self.kind(tcx), self)
     }
 }
 
@@ -1510,13 +1522,22 @@ pub struct BoundTy {
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, TyEncodable, TyDecodable)]
 #[derive(HashStable)]
 pub enum BoundTyKind {
-    Anon,
-    Param(Symbol),
+    Anon(u32),
+    Param(DefId, Symbol),
+}
+
+impl BoundTyKind {
+    pub fn expect_anon(self) -> u32 {
+        match self {
+            BoundTyKind::Anon(i) => i,
+            _ => bug!(),
+        }
+    }
 }
 
 impl From<BoundVar> for BoundTy {
     fn from(var: BoundVar) -> Self {
-        BoundTy { var, kind: BoundTyKind::Anon }
+        BoundTy { var, kind: BoundTyKind::Anon(var.as_u32()) }
     }
 }
 
@@ -1539,7 +1560,7 @@ impl<'tcx> ExistentialProjection<'tcx> {
     pub fn trait_ref(&self, tcx: TyCtxt<'tcx>) -> ty::ExistentialTraitRef<'tcx> {
         let def_id = tcx.parent(self.def_id);
         let subst_count = tcx.generics_of(def_id).count() - 1;
-        let substs = tcx.intern_substs(&self.substs[..subst_count]);
+        let substs = tcx.mk_substs(&self.substs[..subst_count]);
         ty::ExistentialTraitRef { def_id, substs }
     }
 
@@ -1567,7 +1588,7 @@ impl<'tcx> ExistentialProjection<'tcx> {
 
         Self {
             def_id: projection_predicate.projection_ty.def_id,
-            substs: tcx.intern_substs(&projection_predicate.projection_ty.substs[1..]),
+            substs: tcx.mk_substs(&projection_predicate.projection_ty.substs[1..]),
             term: projection_predicate.term,
         }
     }
@@ -1620,10 +1641,16 @@ impl<'tcx> Region<'tcx> {
             ty::ReVar(..) => false,
             ty::RePlaceholder(placeholder) => placeholder.name.is_named(),
             ty::ReErased => false,
+            ty::ReError(_) => false,
         }
     }
 
     #[inline]
+    pub fn is_error(self) -> bool {
+        matches!(*self, ty::ReError(_))
+    }
+
+    #[inline]
     pub fn is_static(self) -> bool {
         matches!(*self, ty::ReStatic)
     }
@@ -1683,6 +1710,7 @@ impl<'tcx> Region<'tcx> {
             ty::ReErased => {
                 flags = flags | TypeFlags::HAS_RE_ERASED;
             }
+            ty::ReError(_) => {}
         }
 
         debug!("type_flags({:?}) = {:?}", self, flags);
@@ -1733,6 +1761,13 @@ impl<'tcx> Region<'tcx> {
     pub fn is_var(self) -> bool {
         matches!(self.kind(), ty::ReVar(_))
     }
+
+    pub fn as_var(self) -> Option<RegionVid> {
+        match self.kind() {
+            ty::ReVar(vid) => Some(vid),
+            _ => None,
+        }
+    }
 }
 
 /// Type utilities
@@ -1867,7 +1902,7 @@ impl<'tcx> Ty<'tcx> {
                         // The way we evaluate the `N` in `[T; N]` here only works since we use
                         // `simd_size_and_type` post-monomorphization. It will probably start to ICE
                         // if we use it in generic code. See the `simd-array-trait` ui test.
-                        (f0_len.eval_usize(tcx, ParamEnv::empty()) as u64, *f0_elem_ty)
+                        (f0_len.eval_target_usize(tcx, ParamEnv::empty()) as u64, *f0_elem_ty)
                     }
                     // Otherwise, the fields of this Adt are the SIMD components (and we assume they
                     // all have the same type).
@@ -2028,7 +2063,7 @@ impl<'tcx> Ty<'tcx> {
     pub fn contains(self, other: Ty<'tcx>) -> bool {
         struct ContainsTyVisitor<'tcx>(Ty<'tcx>);
 
-        impl<'tcx> TypeVisitor<'tcx> for ContainsTyVisitor<'tcx> {
+        impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for ContainsTyVisitor<'tcx> {
             type BreakTy = ();
 
             fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
@@ -2040,6 +2075,28 @@ impl<'tcx> Ty<'tcx> {
         cf.is_break()
     }
 
+    /// Checks whether a type recursively contains any closure
+    ///
+    /// Example: `Option<[closure@file.rs:4:20]>` returns true
+    pub fn contains_closure(self) -> bool {
+        struct ContainsClosureVisitor;
+
+        impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for ContainsClosureVisitor {
+            type BreakTy = ();
+
+            fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
+                if let ty::Closure(_, _) = t.kind() {
+                    ControlFlow::Break(())
+                } else {
+                    t.super_visit_with(self)
+                }
+            }
+        }
+
+        let cf = self.visit_with(&mut ContainsClosureVisitor);
+        cf.is_break()
+    }
+
     /// Returns the type and mutability of `*ty`.
     ///
     /// The parameter `explicit` indicates if this is an *explicit* dereference.
@@ -2065,7 +2122,7 @@ impl<'tcx> Ty<'tcx> {
 
     pub fn fn_sig(self, tcx: TyCtxt<'tcx>) -> PolyFnSig<'tcx> {
         match self.kind() {
-            FnDef(def_id, substs) => tcx.bound_fn_sig(*def_id).subst(tcx, substs),
+            FnDef(def_id, substs) => tcx.fn_sig(*def_id).subst(tcx, substs),
             FnPtr(f) => *f,
             Error(_) => {
                 // ignore errors (#54954)
@@ -2161,7 +2218,7 @@ impl<'tcx> Ty<'tcx> {
                 let assoc_items = tcx.associated_item_def_ids(
                     tcx.require_lang_item(hir::LangItem::DiscriminantKind, None),
                 );
-                tcx.mk_projection(assoc_items[0], tcx.intern_substs(&[self.into()]))
+                tcx.mk_projection(assoc_items[0], tcx.mk_substs(&[self.into()]))
             }
 
             ty::Bool
@@ -2181,6 +2238,7 @@ impl<'tcx> Ty<'tcx> {
             | ty::Dynamic(..)
             | ty::Closure(..)
             | ty::GeneratorWitness(..)
+            | ty::GeneratorWitnessMIR(..)
             | ty::Never
             | ty::Tuple(_)
             | ty::Error(_)
@@ -2216,6 +2274,7 @@ impl<'tcx> Ty<'tcx> {
             | ty::Ref(..)
             | ty::Generator(..)
             | ty::GeneratorWitness(..)
+            | ty::GeneratorWitnessMIR(..)
             | ty::Array(..)
             | ty::Closure(..)
             | ty::Never
@@ -2232,7 +2291,7 @@ impl<'tcx> Ty<'tcx> {
             ty::Str | ty::Slice(_) => (tcx.types.usize, false),
             ty::Dynamic(..) => {
                 let dyn_metadata = tcx.require_lang_item(LangItem::DynMetadata, None);
-                (tcx.bound_type_of(dyn_metadata).subst(tcx, &[tail.into()]), false)
+                (tcx.type_of(dyn_metadata).subst(tcx, &[tail.into()]), false)
             },
 
             // type parameters only have unit metadata if they're sized, so return true
@@ -2302,6 +2361,7 @@ impl<'tcx> Ty<'tcx> {
             | ty::Ref(..)
             | ty::Generator(..)
             | ty::GeneratorWitness(..)
+            | ty::GeneratorWitnessMIR(..)
             | ty::Array(..)
             | ty::Closure(..)
             | ty::Never
@@ -2366,7 +2426,7 @@ impl<'tcx> Ty<'tcx> {
             // anything with custom metadata it might be more complicated.
             ty::Ref(_, _, hir::Mutability::Not) | ty::RawPtr(..) => false,
 
-            ty::Generator(..) | ty::GeneratorWitness(..) => false,
+            ty::Generator(..) | ty::GeneratorWitness(..) | ty::GeneratorWitnessMIR(..) => false,
 
             // Might be, but not "trivial" so just giving the safe answer.
             ty::Adt(..) | ty::Closure(..) => false,
diff --git a/compiler/rustc_middle/src/ty/subst.rs b/compiler/rustc_middle/src/ty/subst.rs
index a07582fc8..b090bd9d8 100644
--- a/compiler/rustc_middle/src/ty/subst.rs
+++ b/compiler/rustc_middle/src/ty/subst.rs
@@ -3,7 +3,7 @@
 use crate::ty::codec::{TyDecoder, TyEncoder};
 use crate::ty::fold::{FallibleTypeFolder, TypeFoldable, TypeFolder, TypeSuperFoldable};
 use crate::ty::sty::{ClosureSubsts, GeneratorSubsts, InlineConstSubsts};
-use crate::ty::visit::{TypeVisitable, TypeVisitor};
+use crate::ty::visit::{TypeVisitable, TypeVisitableExt, TypeVisitor};
 use crate::ty::{self, Lift, List, ParamConst, Ty, TyCtxt};
 
 use rustc_data_structures::intern::Interned;
@@ -71,7 +71,7 @@ impl<'tcx> List<Ty<'tcx>> {
     /// Allows to freely switch between `List<Ty<'tcx>>` and `List<GenericArg<'tcx>>`.
     ///
     /// As lists are interned, `List<Ty<'tcx>>` and `List<GenericArg<'tcx>>` have
-    /// be interned together, see `intern_type_list` for more details.
+    /// be interned together, see `mk_type_list` for more details.
     #[inline]
     pub fn as_substs(&'tcx self) -> SubstsRef<'tcx> {
         assert_eq!(TYPE_TAG, 0);
@@ -227,8 +227,11 @@ impl<'a, 'tcx> Lift<'tcx> for GenericArg<'a> {
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for GenericArg<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for GenericArg<'tcx> {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self, F::Error> {
         match self.unpack() {
             GenericArgKind::Lifetime(lt) => lt.try_fold_with(folder).map(Into::into),
             GenericArgKind::Type(ty) => ty.try_fold_with(folder).map(Into::into),
@@ -237,8 +240,8 @@ impl<'tcx> TypeFoldable<'tcx> for GenericArg<'tcx> {
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx> for GenericArg<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for GenericArg<'tcx> {
+    fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
         match self.unpack() {
             GenericArgKind::Lifetime(lt) => lt.visit_with(visitor),
             GenericArgKind::Type(ty) => ty.visit_with(visitor),
@@ -267,13 +270,11 @@ pub type SubstsRef<'tcx> = &'tcx InternalSubsts<'tcx>;
 impl<'tcx> InternalSubsts<'tcx> {
     /// Checks whether all elements of this list are types, if so, transmute.
     pub fn try_as_type_list(&'tcx self) -> Option<&'tcx List<Ty<'tcx>>> {
-        if self.iter().all(|arg| matches!(arg.unpack(), GenericArgKind::Type(_))) {
+        self.iter().all(|arg| matches!(arg.unpack(), GenericArgKind::Type(_))).then(|| {
             assert_eq!(TYPE_TAG, 0);
             // SAFETY: All elements are types, see `List<Ty<'tcx>>::as_substs`.
-            Some(unsafe { &*(self as *const List<GenericArg<'tcx>> as *const List<Ty<'tcx>>) })
-        } else {
-            None
-        }
+            unsafe { &*(self as *const List<GenericArg<'tcx>> as *const List<Ty<'tcx>>) }
+        })
     }
 
     /// Interpret these substitutions as the substitutions of a closure type.
@@ -318,7 +319,7 @@ impl<'tcx> InternalSubsts<'tcx> {
         let count = defs.count();
         let mut substs = SmallVec::with_capacity(count);
         Self::fill_item(&mut substs, tcx, defs, &mut mk_kind);
-        tcx.intern_substs(&substs)
+        tcx.mk_substs(&substs)
     }
 
     pub fn extend_to<F>(&self, tcx: TyCtxt<'tcx>, def_id: DefId, mut mk_kind: F) -> SubstsRef<'tcx>
@@ -467,26 +468,33 @@ impl<'tcx> InternalSubsts<'tcx> {
         target_substs: SubstsRef<'tcx>,
     ) -> SubstsRef<'tcx> {
         let defs = tcx.generics_of(source_ancestor);
-        tcx.mk_substs(target_substs.iter().chain(self.iter().skip(defs.params.len())))
+        tcx.mk_substs_from_iter(target_substs.iter().chain(self.iter().skip(defs.params.len())))
     }
 
     pub fn truncate_to(&self, tcx: TyCtxt<'tcx>, generics: &ty::Generics) -> SubstsRef<'tcx> {
-        tcx.mk_substs(self.iter().take(generics.count()))
+        tcx.mk_substs_from_iter(self.iter().take(generics.count()))
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for SubstsRef<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for SubstsRef<'tcx> {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self, F::Error> {
         // This code is hot enough that it's worth specializing for the most
         // common length lists, to avoid the overhead of `SmallVec` creation.
         // The match arms are in order of frequency. The 1, 2, and 0 cases are
         // typically hit in 90--99.99% of cases. When folding doesn't change
         // the substs, it's faster to reuse the existing substs rather than
-        // calling `intern_substs`.
+        // calling `mk_substs`.
         match self.len() {
             1 => {
                 let param0 = self[0].try_fold_with(folder)?;
-                if param0 == self[0] { Ok(self) } else { Ok(folder.tcx().intern_substs(&[param0])) }
+                if param0 == self[0] {
+                    Ok(self)
+                } else {
+                    Ok(folder.interner().mk_substs(&[param0]))
+                }
             }
             2 => {
                 let param0 = self[0].try_fold_with(folder)?;
@@ -494,17 +502,20 @@ impl<'tcx> TypeFoldable<'tcx> for SubstsRef<'tcx> {
                 if param0 == self[0] && param1 == self[1] {
                     Ok(self)
                 } else {
-                    Ok(folder.tcx().intern_substs(&[param0, param1]))
+                    Ok(folder.interner().mk_substs(&[param0, param1]))
                 }
             }
             0 => Ok(self),
-            _ => ty::util::fold_list(self, folder, |tcx, v| tcx.intern_substs(v)),
+            _ => ty::util::fold_list(self, folder, |tcx, v| tcx.mk_substs(v)),
         }
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<Ty<'tcx>> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx ty::List<Ty<'tcx>> {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self, F::Error> {
         // This code is fairly hot, though not as hot as `SubstsRef`.
         //
         // When compiling stage 2, I get the following results:
@@ -527,17 +538,17 @@ impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<Ty<'tcx>> {
                 if param0 == self[0] && param1 == self[1] {
                     Ok(self)
                 } else {
-                    Ok(folder.tcx().intern_type_list(&[param0, param1]))
+                    Ok(folder.interner().mk_type_list(&[param0, param1]))
                 }
             }
-            _ => ty::util::fold_list(self, folder, |tcx, v| tcx.intern_type_list(v)),
+            _ => ty::util::fold_list(self, folder, |tcx, v| tcx.mk_type_list(v)),
         }
     }
 }
 
-impl<'tcx, T: TypeVisitable<'tcx>> TypeVisitable<'tcx> for &'tcx ty::List<T> {
+impl<'tcx, T: TypeVisitable<TyCtxt<'tcx>>> TypeVisitable<TyCtxt<'tcx>> for &'tcx ty::List<T> {
     #[inline]
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+    fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
         self.iter().try_for_each(|t| t.visit_with(visitor))
     }
 }
@@ -553,8 +564,8 @@ impl<'tcx, T: TypeVisitable<'tcx>> TypeVisitable<'tcx> for &'tcx ty::List<T> {
 pub struct EarlyBinder<T>(pub T);
 
 /// For early binders, you should first call `subst` before using any visitors.
-impl<'tcx, T> !TypeFoldable<'tcx> for ty::EarlyBinder<T> {}
-impl<'tcx, T> !TypeVisitable<'tcx> for ty::EarlyBinder<T> {}
+impl<'tcx, T> !TypeFoldable<TyCtxt<'tcx>> for ty::EarlyBinder<T> {}
+impl<'tcx, T> !TypeVisitable<TyCtxt<'tcx>> for ty::EarlyBinder<T> {}
 
 impl<T> EarlyBinder<T> {
     pub fn as_ref(&self) -> EarlyBinder<&T> {
@@ -615,7 +626,7 @@ impl<T, U> EarlyBinder<(T, U)> {
 
 impl<'tcx, 's, I: IntoIterator> EarlyBinder<I>
 where
-    I::Item: TypeFoldable<'tcx>,
+    I::Item: TypeFoldable<TyCtxt<'tcx>>,
 {
     pub fn subst_iter(
         self,
@@ -634,7 +645,7 @@ pub struct SubstIter<'s, 'tcx, I: IntoIterator> {
 
 impl<'tcx, I: IntoIterator> Iterator for SubstIter<'_, 'tcx, I>
 where
-    I::Item: TypeFoldable<'tcx>,
+    I::Item: TypeFoldable<TyCtxt<'tcx>>,
 {
     type Item = I::Item;
 
@@ -650,7 +661,7 @@ where
 impl<'tcx, I: IntoIterator> DoubleEndedIterator for SubstIter<'_, 'tcx, I>
 where
     I::IntoIter: DoubleEndedIterator,
-    I::Item: TypeFoldable<'tcx>,
+    I::Item: TypeFoldable<TyCtxt<'tcx>>,
 {
     fn next_back(&mut self) -> Option<Self::Item> {
         Some(EarlyBinder(self.it.next_back()?).subst(self.tcx, self.substs))
@@ -660,14 +671,14 @@ where
 impl<'tcx, I: IntoIterator> ExactSizeIterator for SubstIter<'_, 'tcx, I>
 where
     I::IntoIter: ExactSizeIterator,
-    I::Item: TypeFoldable<'tcx>,
+    I::Item: TypeFoldable<TyCtxt<'tcx>>,
 {
 }
 
 impl<'tcx, 's, I: IntoIterator> EarlyBinder<I>
 where
     I::Item: Deref,
-    <I::Item as Deref>::Target: Copy + TypeFoldable<'tcx>,
+    <I::Item as Deref>::Target: Copy + TypeFoldable<TyCtxt<'tcx>>,
 {
     pub fn subst_iter_copied(
         self,
@@ -687,7 +698,7 @@ pub struct SubstIterCopied<'a, 'tcx, I: IntoIterator> {
 impl<'tcx, I: IntoIterator> Iterator for SubstIterCopied<'_, 'tcx, I>
 where
     I::Item: Deref,
-    <I::Item as Deref>::Target: Copy + TypeFoldable<'tcx>,
+    <I::Item as Deref>::Target: Copy + TypeFoldable<TyCtxt<'tcx>>,
 {
     type Item = <I::Item as Deref>::Target;
 
@@ -704,7 +715,7 @@ impl<'tcx, I: IntoIterator> DoubleEndedIterator for SubstIterCopied<'_, 'tcx, I>
 where
     I::IntoIter: DoubleEndedIterator,
     I::Item: Deref,
-    <I::Item as Deref>::Target: Copy + TypeFoldable<'tcx>,
+    <I::Item as Deref>::Target: Copy + TypeFoldable<TyCtxt<'tcx>>,
 {
     fn next_back(&mut self) -> Option<Self::Item> {
         Some(EarlyBinder(*self.it.next_back()?).subst(self.tcx, self.substs))
@@ -715,7 +726,7 @@ impl<'tcx, I: IntoIterator> ExactSizeIterator for SubstIterCopied<'_, 'tcx, I>
 where
     I::IntoIter: ExactSizeIterator,
     I::Item: Deref,
-    <I::Item as Deref>::Target: Copy + TypeFoldable<'tcx>,
+    <I::Item as Deref>::Target: Copy + TypeFoldable<TyCtxt<'tcx>>,
 {
 }
 
@@ -741,7 +752,7 @@ impl<T: Iterator> Iterator for EarlyBinderIter<T> {
     }
 }
 
-impl<'tcx, T: TypeFoldable<'tcx>> ty::EarlyBinder<T> {
+impl<'tcx, T: TypeFoldable<TyCtxt<'tcx>>> ty::EarlyBinder<T> {
     pub fn subst(self, tcx: TyCtxt<'tcx>, substs: &[GenericArg<'tcx>]) -> T {
         let mut folder = SubstFolder { tcx, substs, binders_passed: 0 };
         self.0.fold_with(&mut folder)
@@ -758,6 +769,11 @@ impl<'tcx, T: TypeFoldable<'tcx>> ty::EarlyBinder<T> {
     pub fn subst_identity(self) -> T {
         self.0
     }
+
+    /// Returns the inner value, but only if it contains no bound vars.
+    pub fn no_bound_vars(self) -> Option<T> {
+        if !self.0.needs_subst() { Some(self.0) } else { None }
+    }
 }
 
 ///////////////////////////////////////////////////////////////////////////
@@ -771,13 +787,13 @@ struct SubstFolder<'a, 'tcx> {
     binders_passed: u32,
 }
 
-impl<'a, 'tcx> TypeFolder<'tcx> for SubstFolder<'a, 'tcx> {
+impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for SubstFolder<'a, 'tcx> {
     #[inline]
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
-    fn fold_binder<T: TypeFoldable<'tcx>>(
+    fn fold_binder<T: TypeFoldable<TyCtxt<'tcx>>>(
         &mut self,
         t: ty::Binder<'tcx, T>,
     ) -> ty::Binder<'tcx, T> {
@@ -970,7 +986,7 @@ impl<'a, 'tcx> SubstFolder<'a, 'tcx> {
     /// As indicated in the diagram, here the same type `&'a i32` is substituted once, but in the
     /// first case we do not increase the De Bruijn index and in the second case we do. The reason
     /// is that only in the second case have we passed through a fn binder.
-    fn shift_vars_through_binders<T: TypeFoldable<'tcx>>(&self, val: T) -> T {
+    fn shift_vars_through_binders<T: TypeFoldable<TyCtxt<'tcx>>>(&self, val: T) -> T {
         debug!(
             "shift_vars(val={:?}, binders_passed={:?}, has_escaping_bound_vars={:?})",
             val,
@@ -982,7 +998,7 @@ impl<'a, 'tcx> SubstFolder<'a, 'tcx> {
             return val;
         }
 
-        let result = ty::fold::shift_vars(TypeFolder::tcx(self), val, self.binders_passed);
+        let result = ty::fold::shift_vars(TypeFolder::interner(self), val, self.binders_passed);
         debug!("shift_vars: shifted result = {:?}", result);
 
         result
diff --git a/compiler/rustc_middle/src/ty/trait_def.rs b/compiler/rustc_middle/src/ty/trait_def.rs
index b38a5fbf2..233c0df2d 100644
--- a/compiler/rustc_middle/src/ty/trait_def.rs
+++ b/compiler/rustc_middle/src/ty/trait_def.rs
@@ -1,6 +1,6 @@
 use crate::traits::specialization_graph;
 use crate::ty::fast_reject::{self, SimplifiedType, TreatParams};
-use crate::ty::visit::TypeVisitable;
+use crate::ty::visit::TypeVisitableExt;
 use crate::ty::{Ident, Ty, TyCtxt};
 use hir::def_id::LOCAL_CRATE;
 use rustc_hir as hir;
@@ -31,6 +31,15 @@ pub struct TraitDef {
     /// and thus `impl`s of it are allowed to overlap.
     pub is_marker: bool,
 
+    /// If `true`, then this trait has to `#[rustc_coinductive]` attribute or
+    /// is an auto trait. This indicates that trait solver cycles involving an
+    /// `X: ThisTrait` goal are accepted.
+    ///
+    /// In the future all traits should be coinductive, but we need a better
+    /// formal understanding of what exactly that means and should probably
+    /// also have already switched to the new trait solver.
+    pub is_coinductive: bool,
+
     /// If `true`, then this trait has the `#[rustc_skip_array_during_method_dispatch]`
     /// attribute, indicating that editions before 2021 should not consider this trait
     /// during method dispatch if the receiver is an array.
@@ -81,28 +90,6 @@ impl TraitImpls {
 }
 
 impl<'tcx> TraitDef {
-    pub fn new(
-        def_id: DefId,
-        unsafety: hir::Unsafety,
-        paren_sugar: bool,
-        has_auto_impl: bool,
-        is_marker: bool,
-        skip_array_during_method_dispatch: bool,
-        specialization_kind: TraitSpecializationKind,
-        must_implement_one_of: Option<Box<[Ident]>>,
-    ) -> TraitDef {
-        TraitDef {
-            def_id,
-            unsafety,
-            paren_sugar,
-            has_auto_impl,
-            is_marker,
-            skip_array_during_method_dispatch,
-            specialization_kind,
-            must_implement_one_of,
-        }
-    }
-
     pub fn ancestors(
         &self,
         tcx: TyCtxt<'tcx>,
@@ -238,7 +225,7 @@ pub(super) fn trait_impls_of_provider(tcx: TyCtxt<'_>, trait_id: DefId) -> Trait
     for &impl_def_id in tcx.hir().trait_impls(trait_id) {
         let impl_def_id = impl_def_id.to_def_id();
 
-        let impl_self_ty = tcx.type_of(impl_def_id);
+        let impl_self_ty = tcx.type_of(impl_def_id).subst_identity();
         if impl_self_ty.references_error() {
             continue;
         }
diff --git a/compiler/rustc_middle/src/ty/typeck_results.rs b/compiler/rustc_middle/src/ty/typeck_results.rs
index 2902c6dc5..586958247 100644
--- a/compiler/rustc_middle/src/ty/typeck_results.rs
+++ b/compiler/rustc_middle/src/ty/typeck_results.rs
@@ -1,6 +1,7 @@
 use crate::{
     hir::place::Place as HirPlace,
     infer::canonical::Canonical,
+    traits::ObligationCause,
     ty::{
         self, tls, BindingMode, BoundVar, CanonicalPolyFnSig, ClosureSizeProfileData,
         GenericArgKind, InternalSubsts, SubstsRef, Ty, UserSubsts,
@@ -193,6 +194,11 @@ pub struct TypeckResults<'tcx> {
     /// that are live across the yield of this generator (if a generator).
     pub generator_interior_types: ty::Binder<'tcx, Vec<GeneratorInteriorTypeCause<'tcx>>>,
 
+    /// Stores the predicates that apply on generator witness types.
+    /// formatting modified file tests/ui/generator/retain-resume-ref.rs
+    pub generator_interior_predicates:
+        FxHashMap<LocalDefId, Vec<(ty::Predicate<'tcx>, ObligationCause<'tcx>)>>,
+
     /// We sometimes treat byte string literals (which are of type `&[u8; N]`)
     /// as `&[u8]`, depending on the pattern in which they are used.
     /// This hashset records all instances where we behave
@@ -271,6 +277,7 @@ impl<'tcx> TypeckResults<'tcx> {
             closure_fake_reads: Default::default(),
             rvalue_scopes: Default::default(),
             generator_interior_types: ty::Binder::dummy(Default::default()),
+            generator_interior_predicates: Default::default(),
             treat_byte_string_as_slice: Default::default(),
             closure_size_eval: Default::default(),
         }
@@ -365,7 +372,7 @@ impl<'tcx> TypeckResults<'tcx> {
 
     pub fn node_type(&self, id: hir::HirId) -> Ty<'tcx> {
         self.node_type_opt(id).unwrap_or_else(|| {
-            bug!("node_type: no type for node `{}`", tls::with(|tcx| tcx.hir().node_to_string(id)))
+            bug!("node_type: no type for node {}", tls::with(|tcx| tcx.hir().node_to_string(id)))
         })
     }
 
@@ -544,9 +551,8 @@ fn validate_hir_id_for_typeck_results(hir_owner: OwnerId, hir_id: hir::HirId) {
 fn invalid_hir_id_for_typeck_results(hir_owner: OwnerId, hir_id: hir::HirId) {
     ty::tls::with(|tcx| {
         bug!(
-            "node {} with HirId::owner {:?} cannot be placed in TypeckResults with hir_owner {:?}",
+            "node {} cannot be placed in TypeckResults with hir_owner {:?}",
             tcx.hir().node_to_string(hir_id),
-            hir_id.owner,
             hir_owner
         )
     });
@@ -563,7 +569,7 @@ impl<'a, V> LocalTableInContext<'a, V> {
         self.data.contains_key(&id.local_id)
     }
 
-    pub fn get(&self, id: hir::HirId) -> Option<&V> {
+    pub fn get(&self, id: hir::HirId) -> Option<&'a V> {
         validate_hir_id_for_typeck_results(self.hir_owner, id);
         self.data.get(&id.local_id)
     }
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index d0d1dcc58..90270e0ee 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -3,10 +3,9 @@
 use crate::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use crate::mir;
 use crate::ty::layout::IntegerExt;
-use crate::ty::query::TyCtxtAt;
 use crate::ty::{
-    self, DefIdTree, FallibleTypeFolder, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeSuperFoldable,
-    TypeVisitable,
+    self, DefIdTree, FallibleTypeFolder, ToPredicate, Ty, TyCtxt, TypeFoldable, TypeFolder,
+    TypeSuperFoldable, TypeVisitableExt,
 };
 use crate::ty::{GenericArgKind, SubstsRef};
 use rustc_apfloat::Float as _;
@@ -168,7 +167,7 @@ impl<'tcx> TyCtxt<'tcx> {
                 | DefKind::Fn
                 | DefKind::AssocFn
                 | DefKind::AssocConst
-                | DefKind::Impl,
+                | DefKind::Impl { .. },
                 def_id,
             ) => Some(def_id),
             Res::Err => None,
@@ -363,7 +362,7 @@ impl<'tcx> TyCtxt<'tcx> {
         let drop_trait = self.lang_items().drop_trait()?;
         self.ensure().coherent_trait(drop_trait);
 
-        let ty = self.type_of(adt_did);
+        let ty = self.type_of(adt_did).subst_identity();
         let (did, constness) = self.find_map_relevant_impl(drop_trait, ty, |impl_did| {
             if let Some(item_id) = self.associated_item_def_ids(impl_did).first() {
                 if validate(self, impl_did).is_ok() {
@@ -416,12 +415,12 @@ impl<'tcx> TyCtxt<'tcx> {
         // <P1, P2, P0>, and then look up which of the impl substs refer to
         // parameters marked as pure.
 
-        let impl_substs = match *self.type_of(impl_def_id).kind() {
+        let impl_substs = match *self.type_of(impl_def_id).subst_identity().kind() {
             ty::Adt(def_, substs) if def_ == def => substs,
             _ => bug!(),
         };
 
-        let item_substs = match *self.type_of(def.did()).kind() {
+        let item_substs = match *self.type_of(def.did()).subst_identity().kind() {
             ty::Adt(def_, substs) if def_ == def => substs,
             _ => bug!(),
         };
@@ -565,14 +564,14 @@ impl<'tcx> TyCtxt<'tcx> {
         self,
         closure_def_id: DefId,
         closure_substs: SubstsRef<'tcx>,
-        env_region: ty::RegionKind<'tcx>,
+        env_region: ty::Region<'tcx>,
     ) -> Option<Ty<'tcx>> {
         let closure_ty = self.mk_closure(closure_def_id, closure_substs);
         let closure_kind_ty = closure_substs.as_closure().kind_ty();
         let closure_kind = closure_kind_ty.to_opt_closure_kind()?;
         let env_ty = match closure_kind {
-            ty::ClosureKind::Fn => self.mk_imm_ref(self.mk_region(env_region), closure_ty),
-            ty::ClosureKind::FnMut => self.mk_mut_ref(self.mk_region(env_region), closure_ty),
+            ty::ClosureKind::Fn => self.mk_imm_ref(env_region, closure_ty),
+            ty::ClosureKind::FnMut => self.mk_mut_ref(env_region, closure_ty),
             ty::ClosureKind::FnOnce => closure_ty,
         };
         Some(env_ty)
@@ -603,7 +602,10 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Get the type of the pointer to the static that we use in MIR.
     pub fn static_ptr_ty(self, def_id: DefId) -> Ty<'tcx> {
         // Make sure that any constants in the static's type are evaluated.
-        let static_ty = self.normalize_erasing_regions(ty::ParamEnv::empty(), self.type_of(def_id));
+        let static_ty = self.normalize_erasing_regions(
+            ty::ParamEnv::empty(),
+            self.type_of(def_id).subst_identity(),
+        );
 
         // Make sure that accesses to unsafe statics end up using raw pointers.
         // For thread-locals, this needs to be kept in sync with `Rvalue::ty`.
@@ -616,6 +618,36 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
+    /// Return the set of types that should be taken into accound when checking
+    /// trait bounds on a generator's internal state.
+    pub fn generator_hidden_types(
+        self,
+        def_id: DefId,
+    ) -> impl Iterator<Item = ty::EarlyBinder<Ty<'tcx>>> {
+        let generator_layout = &self.mir_generator_witnesses(def_id);
+        generator_layout
+            .field_tys
+            .iter()
+            .filter(|decl| !decl.ignore_for_traits)
+            .map(|decl| ty::EarlyBinder(decl.ty))
+    }
+
+    /// Normalizes all opaque types in the given value, replacing them
+    /// with their underlying types.
+    pub fn expand_opaque_types(self, val: Ty<'tcx>) -> Ty<'tcx> {
+        let mut visitor = OpaqueTypeExpander {
+            seen_opaque_tys: FxHashSet::default(),
+            expanded_cache: FxHashMap::default(),
+            primary_def_id: None,
+            found_recursion: false,
+            found_any_recursion: false,
+            check_recursion: false,
+            expand_generators: false,
+            tcx: self,
+        };
+        val.fold_with(&mut visitor)
+    }
+
     /// Expands the given impl trait type, stopping if the type is recursive.
     #[instrument(skip(self), level = "debug", ret)]
     pub fn try_expand_impl_trait_type(
@@ -630,6 +662,7 @@ impl<'tcx> TyCtxt<'tcx> {
             found_recursion: false,
             found_any_recursion: false,
             check_recursion: true,
+            expand_generators: true,
             tcx: self,
         };
 
@@ -637,10 +670,6 @@ impl<'tcx> TyCtxt<'tcx> {
         if visitor.found_recursion { Err(expanded_type) } else { Ok(expanded_type) }
     }
 
-    pub fn bound_type_of(self, def_id: DefId) -> ty::EarlyBinder<Ty<'tcx>> {
-        ty::EarlyBinder(self.type_of(def_id))
-    }
-
     pub fn bound_return_position_impl_trait_in_trait_tys(
         self,
         def_id: DefId,
@@ -648,10 +677,6 @@ impl<'tcx> TyCtxt<'tcx> {
         ty::EarlyBinder(self.collect_return_position_impl_trait_in_trait_tys(def_id))
     }
 
-    pub fn bound_fn_sig(self, def_id: DefId) -> ty::EarlyBinder<ty::PolyFnSig<'tcx>> {
-        ty::EarlyBinder(self.fn_sig(def_id))
-    }
-
     pub fn bound_explicit_item_bounds(
         self,
         def_id: DefId,
@@ -736,11 +761,39 @@ impl<'tcx> TyCtxt<'tcx> {
         }
         (generator_layout, generator_saved_local_names)
     }
-}
 
-impl<'tcx> TyCtxtAt<'tcx> {
-    pub fn bound_type_of(self, def_id: DefId) -> ty::EarlyBinder<Ty<'tcx>> {
-        ty::EarlyBinder(self.type_of(def_id))
+    /// Query and get an English description for the item's kind.
+    pub fn def_descr(self, def_id: DefId) -> &'static str {
+        self.def_kind_descr(self.def_kind(def_id), def_id)
+    }
+
+    /// Get an English description for the item's kind.
+    pub fn def_kind_descr(self, def_kind: DefKind, def_id: DefId) -> &'static str {
+        match def_kind {
+            DefKind::AssocFn if self.associated_item(def_id).fn_has_self_parameter => "method",
+            DefKind::Generator => match self.generator_kind(def_id).unwrap() {
+                rustc_hir::GeneratorKind::Async(..) => "async closure",
+                rustc_hir::GeneratorKind::Gen => "generator",
+            },
+            _ => def_kind.descr(def_id),
+        }
+    }
+
+    /// Gets an English article for the [`TyCtxt::def_descr`].
+    pub fn def_descr_article(self, def_id: DefId) -> &'static str {
+        self.def_kind_descr_article(self.def_kind(def_id), def_id)
+    }
+
+    /// Gets an English article for the [`TyCtxt::def_kind_descr`].
+    pub fn def_kind_descr_article(self, def_kind: DefKind, def_id: DefId) -> &'static str {
+        match def_kind {
+            DefKind::AssocFn if self.associated_item(def_id).fn_has_self_parameter => "a",
+            DefKind::Generator => match self.generator_kind(def_id).unwrap() {
+                rustc_hir::GeneratorKind::Async(..) => "an",
+                rustc_hir::GeneratorKind::Gen => "a",
+            },
+            _ => def_kind.article(),
+        }
     }
 }
 
@@ -756,6 +809,7 @@ struct OpaqueTypeExpander<'tcx> {
     primary_def_id: Option<DefId>,
     found_recursion: bool,
     found_any_recursion: bool,
+    expand_generators: bool,
     /// Whether or not to check for recursive opaque types.
     /// This is `true` when we're explicitly checking for opaque type
     /// recursion, and 'false' otherwise to avoid unnecessary work.
@@ -773,7 +827,7 @@ impl<'tcx> OpaqueTypeExpander<'tcx> {
             let expanded_ty = match self.expanded_cache.get(&(def_id, substs)) {
                 Some(expanded_ty) => *expanded_ty,
                 None => {
-                    let generic_ty = self.tcx.bound_type_of(def_id);
+                    let generic_ty = self.tcx.type_of(def_id);
                     let concrete_ty = generic_ty.subst(self.tcx, substs);
                     let expanded_ty = self.fold_ty(concrete_ty);
                     self.expanded_cache.insert((def_id, substs), expanded_ty);
@@ -792,20 +846,77 @@ impl<'tcx> OpaqueTypeExpander<'tcx> {
             None
         }
     }
+
+    fn expand_generator(&mut self, def_id: DefId, substs: SubstsRef<'tcx>) -> Option<Ty<'tcx>> {
+        if self.found_any_recursion {
+            return None;
+        }
+        let substs = substs.fold_with(self);
+        if !self.check_recursion || self.seen_opaque_tys.insert(def_id) {
+            let expanded_ty = match self.expanded_cache.get(&(def_id, substs)) {
+                Some(expanded_ty) => *expanded_ty,
+                None => {
+                    for bty in self.tcx.generator_hidden_types(def_id) {
+                        let hidden_ty = bty.subst(self.tcx, substs);
+                        self.fold_ty(hidden_ty);
+                    }
+                    let expanded_ty = self.tcx.mk_generator_witness_mir(def_id, substs);
+                    self.expanded_cache.insert((def_id, substs), expanded_ty);
+                    expanded_ty
+                }
+            };
+            if self.check_recursion {
+                self.seen_opaque_tys.remove(&def_id);
+            }
+            Some(expanded_ty)
+        } else {
+            // If another opaque type that we contain is recursive, then it
+            // will report the error, so we don't have to.
+            self.found_any_recursion = true;
+            self.found_recursion = def_id == *self.primary_def_id.as_ref().unwrap();
+            None
+        }
+    }
 }
 
-impl<'tcx> TypeFolder<'tcx> for OpaqueTypeExpander<'tcx> {
-    fn tcx(&self) -> TyCtxt<'tcx> {
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for OpaqueTypeExpander<'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
     fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
-        if let ty::Alias(ty::Opaque, ty::AliasTy { def_id, substs, .. }) = *t.kind() {
+        let mut t = if let ty::Alias(ty::Opaque, ty::AliasTy { def_id, substs, .. }) = *t.kind() {
             self.expand_opaque_ty(def_id, substs).unwrap_or(t)
-        } else if t.has_opaque_types() {
+        } else if t.has_opaque_types() || t.has_generators() {
             t.super_fold_with(self)
         } else {
             t
+        };
+        if self.expand_generators {
+            if let ty::GeneratorWitnessMIR(def_id, substs) = *t.kind() {
+                t = self.expand_generator(def_id, substs).unwrap_or(t);
+            }
+        }
+        t
+    }
+
+    fn fold_predicate(&mut self, p: ty::Predicate<'tcx>) -> ty::Predicate<'tcx> {
+        if let ty::PredicateKind::Clause(clause) = p.kind().skip_binder()
+            && let ty::Clause::Projection(projection_pred) = clause
+        {
+            p.kind()
+                .rebind(ty::ProjectionPredicate {
+                    projection_ty: projection_pred.projection_ty.fold_with(self),
+                    // Don't fold the term on the RHS of the projection predicate.
+                    // This is because for default trait methods with RPITITs, we
+                    // install a `NormalizesTo(Projection(RPITIT) -> Opaque(RPITIT))`
+                    // predicate, which would trivially cause a cycle when we do
+                    // anything that requires `ParamEnv::with_reveal_all_normalized`.
+                    term: projection_pred.term,
+                })
+                .to_predicate(self.tcx)
+        } else {
+            p.super_fold_with(self)
         }
     }
 }
@@ -911,6 +1022,7 @@ impl<'tcx> Ty<'tcx> {
             | ty::Foreign(_)
             | ty::Generator(..)
             | ty::GeneratorWitness(_)
+            | ty::GeneratorWitnessMIR(..)
             | ty::Infer(_)
             | ty::Alias(..)
             | ty::Param(_)
@@ -950,6 +1062,7 @@ impl<'tcx> Ty<'tcx> {
             | ty::Foreign(_)
             | ty::Generator(..)
             | ty::GeneratorWitness(_)
+            | ty::GeneratorWitnessMIR(..)
             | ty::Infer(_)
             | ty::Alias(..)
             | ty::Param(_)
@@ -1077,7 +1190,10 @@ impl<'tcx> Ty<'tcx> {
                 false
             }
 
-            ty::Foreign(_) | ty::GeneratorWitness(..) | ty::Error(_) => false,
+            ty::Foreign(_)
+            | ty::GeneratorWitness(..)
+            | ty::GeneratorWitnessMIR(..)
+            | ty::Error(_) => false,
         }
     }
 
@@ -1173,6 +1289,7 @@ pub fn needs_drop_components<'tcx>(
         | ty::FnPtr(_)
         | ty::Char
         | ty::GeneratorWitness(..)
+        | ty::GeneratorWitnessMIR(..)
         | ty::RawPtr(_)
         | ty::Ref(..)
         | ty::Str => Ok(SmallVec::new()),
@@ -1243,7 +1360,11 @@ pub fn is_trivially_const_drop(ty: Ty<'_>) -> bool {
 
         // Not trivial because they have components, and instead of looking inside,
         // we'll just perform trait selection.
-        ty::Closure(..) | ty::Generator(..) | ty::GeneratorWitness(_) | ty::Adt(..) => false,
+        ty::Closure(..)
+        | ty::Generator(..)
+        | ty::GeneratorWitness(_)
+        | ty::GeneratorWitnessMIR(..)
+        | ty::Adt(..) => false,
 
         ty::Array(ty, _) | ty::Slice(ty) => is_trivially_const_drop(ty),
 
@@ -1262,8 +1383,8 @@ pub fn fold_list<'tcx, F, T>(
     intern: impl FnOnce(TyCtxt<'tcx>, &[T]) -> &'tcx ty::List<T>,
 ) -> Result<&'tcx ty::List<T>, F::Error>
 where
-    F: FallibleTypeFolder<'tcx>,
-    T: TypeFoldable<'tcx> + PartialEq + Copy,
+    F: FallibleTypeFolder<TyCtxt<'tcx>>,
+    T: TypeFoldable<TyCtxt<'tcx>> + PartialEq + Copy,
 {
     let mut iter = list.iter();
     // Look for the first element that changed
@@ -1279,7 +1400,7 @@ where
             for t in iter {
                 new_list.push(t.try_fold_with(folder)?)
             }
-            Ok(intern(folder.tcx(), &new_list))
+            Ok(intern(folder.interner(), &new_list))
         }
         Some((_, Err(err))) => {
             return Err(err);
@@ -1304,13 +1425,15 @@ pub fn reveal_opaque_types_in_bounds<'tcx>(
         found_recursion: false,
         found_any_recursion: false,
         check_recursion: false,
+        expand_generators: false,
         tcx,
     };
     val.fold_with(&mut visitor)
 }
 
 /// Determines whether an item is annotated with `doc(hidden)`.
-pub fn is_doc_hidden(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
+fn is_doc_hidden(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
+    assert!(def_id.is_local());
     tcx.get_attrs(def_id, sym::doc)
         .filter_map(|attr| attr.meta_item_list())
         .any(|items| items.iter().any(|item| item.has_name(sym::hidden)))
@@ -1325,7 +1448,7 @@ pub fn is_doc_notable_trait(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
 
 /// Determines whether an item is an intrinsic by Abi.
 pub fn is_intrinsic(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
-    matches!(tcx.fn_sig(def_id).abi(), Abi::RustIntrinsic | Abi::PlatformIntrinsic)
+    matches!(tcx.fn_sig(def_id).skip_binder().abi(), Abi::RustIntrinsic | Abi::PlatformIntrinsic)
 }
 
 pub fn provide(providers: &mut ty::query::Providers) {
diff --git a/compiler/rustc_middle/src/ty/visit.rs b/compiler/rustc_middle/src/ty/visit.rs
index bee3cc4d7..6814cadb9 100644
--- a/compiler/rustc_middle/src/ty/visit.rs
+++ b/compiler/rustc_middle/src/ty/visit.rs
@@ -1,69 +1,13 @@
-//! A visiting traversal mechanism for complex data structures that contain type
-//! information.
-//!
-//! This is a read-only traversal of the data structure.
-//!
-//! This traversal has limited flexibility. Only a small number of "types of
-//! interest" within the complex data structures can receive custom
-//! visitation. These are the ones containing the most important type-related
-//! information, such as `Ty`, `Predicate`, `Region`, and `Const`.
-//!
-//! There are three groups of traits involved in each traversal.
-//! - `TypeVisitable`. This is implemented once for many types, including:
-//!   - Types of interest, for which the methods delegate to the visitor.
-//!   - All other types, including generic containers like `Vec` and `Option`.
-//!     It defines a "skeleton" of how they should be visited.
-//! - `TypeSuperVisitable`. This is implemented only for each type of interest,
-//!   and defines the visiting "skeleton" for these types.
-//! - `TypeVisitor`. This is implemented for each visitor. This defines how
-//!   types of interest are visited.
-//!
-//! This means each visit is a mixture of (a) generic visiting operations, and (b)
-//! custom visit operations that are specific to the visitor.
-//! - The `TypeVisitable` impls handle most of the traversal, and call into
-//!   `TypeVisitor` when they encounter a type of interest.
-//! - A `TypeVisitor` may call into another `TypeVisitable` impl, because some of
-//!   the types of interest are recursive and can contain other types of interest.
-//! - A `TypeVisitor` may also call into a `TypeSuperVisitable` impl, because each
-//!   visitor might provide custom handling only for some types of interest, or
-//!   only for some variants of each type of interest, and then use default
-//!   traversal for the remaining cases.
-//!
-//! For example, if you have `struct S(Ty, U)` where `S: TypeVisitable` and `U:
-//! TypeVisitable`, and an instance `s = S(ty, u)`, it would be visited like so:
-//! ```text
-//! s.visit_with(visitor) calls
-//! - ty.visit_with(visitor) calls
-//!   - visitor.visit_ty(ty) may call
-//!     - ty.super_visit_with(visitor)
-//! - u.visit_with(visitor)
-//! ```
 use crate::ty::{self, flags::FlagComputation, Binder, Ty, TyCtxt, TypeFlags};
 use rustc_errors::ErrorGuaranteed;
 
 use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::sso::SsoHashSet;
-use std::fmt;
 use std::ops::ControlFlow;
 
-/// This trait is implemented for every type that can be visited,
-/// providing the skeleton of the traversal.
-///
-/// To implement this conveniently, use the derive macro located in
-/// `rustc_macros`.
-pub trait TypeVisitable<'tcx>: fmt::Debug + Clone {
-    /// The entry point for visiting. To visit a value `t` with a visitor `v`
-    /// call: `t.visit_with(v)`.
-    ///
-    /// For most types, this just traverses the value, calling `visit_with` on
-    /// each field/element.
-    ///
-    /// For types of interest (such as `Ty`), the implementation of this method
-    /// that calls a visitor method specifically for that type (such as
-    /// `V::visit_ty`). This is where control transfers from `TypeFoldable` to
-    /// `TypeVisitor`.
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy>;
+pub use rustc_type_ir::visit::{TypeSuperVisitable, TypeVisitable, TypeVisitor};
 
+pub trait TypeVisitableExt<'tcx>: TypeVisitable<TyCtxt<'tcx>> {
     /// Returns `true` if `self` has any late-bound regions that are either
     /// bound by `binder` or bound by some binder outside of `binder`.
     /// If `binder` is `ty::INNERMOST`, this indicates whether
@@ -100,6 +44,9 @@ pub trait TypeVisitable<'tcx>: fmt::Debug + Clone {
     fn has_opaque_types(&self) -> bool {
         self.has_type_flags(TypeFlags::HAS_TY_OPAQUE)
     }
+    fn has_generators(&self) -> bool {
+        self.has_type_flags(TypeFlags::HAS_TY_GENERATOR)
+    }
     fn references_error(&self) -> bool {
         self.has_type_flags(TypeFlags::HAS_ERROR)
     }
@@ -182,45 +129,7 @@ pub trait TypeVisitable<'tcx>: fmt::Debug + Clone {
     }
 }
 
-pub trait TypeSuperVisitable<'tcx>: TypeVisitable<'tcx> {
-    /// Provides a default visit for a type of interest. This should only be
-    /// called within `TypeVisitor` methods, when a non-custom traversal is
-    /// desired for the value of the type of interest passed to that method.
-    /// For example, in `MyVisitor::visit_ty(ty)`, it is valid to call
-    /// `ty.super_visit_with(self)`, but any other visiting should be done
-    /// with `xyz.visit_with(self)`.
-    fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy>;
-}
-
-/// This trait is implemented for every visiting traversal. There is a visit
-/// method defined for every type of interest. Each such method has a default
-/// that recurses into the type's fields in a non-custom fashion.
-pub trait TypeVisitor<'tcx>: Sized {
-    type BreakTy = !;
-
-    fn visit_binder<T: TypeVisitable<'tcx>>(
-        &mut self,
-        t: &Binder<'tcx, T>,
-    ) -> ControlFlow<Self::BreakTy> {
-        t.super_visit_with(self)
-    }
-
-    fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
-        t.super_visit_with(self)
-    }
-
-    fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
-        r.super_visit_with(self)
-    }
-
-    fn visit_const(&mut self, c: ty::Const<'tcx>) -> ControlFlow<Self::BreakTy> {
-        c.super_visit_with(self)
-    }
-
-    fn visit_predicate(&mut self, p: ty::Predicate<'tcx>) -> ControlFlow<Self::BreakTy> {
-        p.super_visit_with(self)
-    }
-}
+impl<'tcx, T: TypeVisitable<TyCtxt<'tcx>>> TypeVisitableExt<'tcx> for T {}
 
 ///////////////////////////////////////////////////////////////////////////
 // Region folder
@@ -229,7 +138,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Invoke `callback` on every region appearing free in `value`.
     pub fn for_each_free_region(
         self,
-        value: &impl TypeVisitable<'tcx>,
+        value: &impl TypeVisitable<TyCtxt<'tcx>>,
         mut callback: impl FnMut(ty::Region<'tcx>),
     ) {
         self.any_free_region_meets(value, |r| {
@@ -241,7 +150,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Returns `true` if `callback` returns true for every region appearing free in `value`.
     pub fn all_free_regions_meet(
         self,
-        value: &impl TypeVisitable<'tcx>,
+        value: &impl TypeVisitable<TyCtxt<'tcx>>,
         mut callback: impl FnMut(ty::Region<'tcx>) -> bool,
     ) -> bool {
         !self.any_free_region_meets(value, |r| !callback(r))
@@ -250,7 +159,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Returns `true` if `callback` returns true for some region appearing free in `value`.
     pub fn any_free_region_meets(
         self,
-        value: &impl TypeVisitable<'tcx>,
+        value: &impl TypeVisitable<TyCtxt<'tcx>>,
         callback: impl FnMut(ty::Region<'tcx>) -> bool,
     ) -> bool {
         struct RegionVisitor<F> {
@@ -275,13 +184,13 @@ impl<'tcx> TyCtxt<'tcx> {
             callback: F,
         }
 
-        impl<'tcx, F> TypeVisitor<'tcx> for RegionVisitor<F>
+        impl<'tcx, F> TypeVisitor<TyCtxt<'tcx>> for RegionVisitor<F>
         where
             F: FnMut(ty::Region<'tcx>) -> bool,
         {
             type BreakTy = ();
 
-            fn visit_binder<T: TypeVisitable<'tcx>>(
+            fn visit_binder<T: TypeVisitable<TyCtxt<'tcx>>>(
                 &mut self,
                 t: &Binder<'tcx, T>,
             ) -> ControlFlow<Self::BreakTy> {
@@ -328,7 +237,7 @@ impl<'tcx> TyCtxt<'tcx> {
         value: &Binder<'tcx, T>,
     ) -> FxHashSet<ty::BoundRegionKind>
     where
-        T: TypeVisitable<'tcx>,
+        T: TypeVisitable<TyCtxt<'tcx>>,
     {
         self.collect_late_bound_regions(value, true)
     }
@@ -339,7 +248,7 @@ impl<'tcx> TyCtxt<'tcx> {
         value: &Binder<'tcx, T>,
     ) -> FxHashSet<ty::BoundRegionKind>
     where
-        T: TypeVisitable<'tcx>,
+        T: TypeVisitable<TyCtxt<'tcx>>,
     {
         self.collect_late_bound_regions(value, false)
     }
@@ -350,7 +259,7 @@ impl<'tcx> TyCtxt<'tcx> {
         just_constraint: bool,
     ) -> FxHashSet<ty::BoundRegionKind>
     where
-        T: TypeVisitable<'tcx>,
+        T: TypeVisitable<TyCtxt<'tcx>>,
     {
         let mut collector = LateBoundRegionsCollector::new(just_constraint);
         let result = value.as_ref().skip_binder().visit_with(&mut collector);
@@ -377,10 +286,10 @@ impl<'tcx> ValidateBoundVars<'tcx> {
     }
 }
 
-impl<'tcx> TypeVisitor<'tcx> for ValidateBoundVars<'tcx> {
+impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for ValidateBoundVars<'tcx> {
     type BreakTy = ();
 
-    fn visit_binder<T: TypeVisitable<'tcx>>(
+    fn visit_binder<T: TypeVisitable<TyCtxt<'tcx>>>(
         &mut self,
         t: &Binder<'tcx, T>,
     ) -> ControlFlow<Self::BreakTy> {
@@ -489,10 +398,10 @@ struct HasEscapingVarsVisitor {
     outer_index: ty::DebruijnIndex,
 }
 
-impl<'tcx> TypeVisitor<'tcx> for HasEscapingVarsVisitor {
+impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for HasEscapingVarsVisitor {
     type BreakTy = FoundEscapingVars;
 
-    fn visit_binder<T: TypeVisitable<'tcx>>(
+    fn visit_binder<T: TypeVisitable<TyCtxt<'tcx>>>(
         &mut self,
         t: &Binder<'tcx, T>,
     ) -> ControlFlow<Self::BreakTy> {
@@ -566,7 +475,7 @@ impl std::fmt::Debug for HasTypeFlagsVisitor {
     }
 }
 
-impl<'tcx> TypeVisitor<'tcx> for HasTypeFlagsVisitor {
+impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for HasTypeFlagsVisitor {
     type BreakTy = FoundFlags;
 
     #[inline]
@@ -636,8 +545,8 @@ impl LateBoundRegionsCollector {
     }
 }
 
-impl<'tcx> TypeVisitor<'tcx> for LateBoundRegionsCollector {
-    fn visit_binder<T: TypeVisitable<'tcx>>(
+impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for LateBoundRegionsCollector {
+    fn visit_binder<T: TypeVisitable<TyCtxt<'tcx>>>(
         &mut self,
         t: &Binder<'tcx, T>,
     ) -> ControlFlow<Self::BreakTy> {
@@ -698,7 +607,7 @@ impl MaxUniverse {
     }
 }
 
-impl<'tcx> TypeVisitor<'tcx> for MaxUniverse {
+impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for MaxUniverse {
     fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
         if let ty::Placeholder(placeholder) = t.kind() {
             self.max_universe = ty::UniverseIndex::from_u32(
diff --git a/compiler/rustc_middle/src/ty/vtable.rs b/compiler/rustc_middle/src/ty/vtable.rs
index f77bd9f0c..b9b1cd73a 100644
--- a/compiler/rustc_middle/src/ty/vtable.rs
+++ b/compiler/rustc_middle/src/ty/vtable.rs
@@ -112,5 +112,5 @@ pub(super) fn vtable_allocation_provider<'tcx>(
     }
 
     vtable.mutability = Mutability::Not;
-    tcx.create_memory_alloc(tcx.intern_const_alloc(vtable))
+    tcx.create_memory_alloc(tcx.mk_const_alloc(vtable))
 }
diff --git a/compiler/rustc_middle/src/ty/walk.rs b/compiler/rustc_middle/src/ty/walk.rs
index 708a5e4d0..182945b9c 100644
--- a/compiler/rustc_middle/src/ty/walk.rs
+++ b/compiler/rustc_middle/src/ty/walk.rs
@@ -190,6 +190,7 @@ fn push_inner<'tcx>(stack: &mut TypeWalkerStack<'tcx>, parent: GenericArg<'tcx>)
             ty::Adt(_, substs)
             | ty::Closure(_, substs)
             | ty::Generator(_, substs, _)
+            | ty::GeneratorWitnessMIR(_, substs)
             | ty::FnDef(_, substs) => {
                 stack.extend(substs.iter().rev());
             }
diff --git a/compiler/rustc_middle/src/values.rs b/compiler/rustc_middle/src/values.rs
index 34e8a5597..c4f526dbd 100644
--- a/compiler/rustc_middle/src/values.rs
+++ b/compiler/rustc_middle/src/values.rs
@@ -2,7 +2,7 @@ use crate::dep_graph::DepKind;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::{pluralize, struct_span_err, Applicability, MultiSpan};
 use rustc_hir as hir;
-use rustc_hir::def::DefKind;
+use rustc_hir::def::{DefKind, Res};
 use rustc_middle::ty::Representability;
 use rustc_middle::ty::{self, DefIdTree, Ty, TyCtxt};
 use rustc_query_system::query::QueryInfo;
@@ -16,7 +16,7 @@ impl<'tcx> Value<TyCtxt<'tcx>, DepKind> for Ty<'_> {
     fn from_cycle_error(tcx: TyCtxt<'tcx>, _: &[QueryInfo<DepKind>]) -> Self {
         // SAFETY: This is never called when `Self` is not `Ty<'tcx>`.
         // FIXME: Represent the above fact in the trait system somehow.
-        unsafe { std::mem::transmute::<Ty<'tcx>, Ty<'_>>(tcx.ty_error()) }
+        unsafe { std::mem::transmute::<Ty<'tcx>, Ty<'_>>(tcx.ty_error_misc()) }
     }
 }
 
@@ -34,7 +34,7 @@ impl<'tcx> Value<TyCtxt<'tcx>, DepKind> for ty::SymbolName<'_> {
 
 impl<'tcx> Value<TyCtxt<'tcx>, DepKind> for ty::Binder<'_, ty::FnSig<'_>> {
     fn from_cycle_error(tcx: TyCtxt<'tcx>, stack: &[QueryInfo<DepKind>]) -> Self {
-        let err = tcx.ty_error();
+        let err = tcx.ty_error_misc();
 
         let arity = if let Some(frame) = stack.get(0)
             && frame.query.dep_kind == DepKind::fn_sig
@@ -199,7 +199,8 @@ fn find_item_ty_spans(
 ) {
     match ty.kind {
         hir::TyKind::Path(hir::QPath::Resolved(_, path)) => {
-            if let Some(def_id) = path.res.opt_def_id() {
+            if let Res::Def(kind, def_id) = path.res
+                && kind != DefKind::TyAlias {
                 let check_params = def_id.as_local().map_or(true, |def_id| {
                     if def_id == needle {
                         spans.push(ty.span);
diff --git a/compiler/rustc_mir_build/Cargo.toml b/compiler/rustc_mir_build/Cargo.toml
index 4ad3343d3..f24b165d7 100644
--- a/compiler/rustc_mir_build/Cargo.toml
+++ b/compiler/rustc_mir_build/Cargo.toml
@@ -11,7 +11,6 @@ tracing = "0.1"
 either = "1"
 rustc_middle = { path = "../rustc_middle" }
 rustc_apfloat = { path = "../rustc_apfloat" }
-rustc_attr = { path = "../rustc_attr" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_index = { path = "../rustc_index" }
 rustc_errors = { path = "../rustc_errors" }
diff --git a/compiler/rustc_error_messages/locales/en-US/mir_build.ftl b/compiler/rustc_mir_build/locales/en-US.ftl
index 224855fff..93e7fb330 100644
--- a/compiler/rustc_error_messages/locales/en-US/mir_build.ftl
+++ b/compiler/rustc_mir_build/locales/en-US.ftl
@@ -299,10 +299,18 @@ mir_build_borrow_of_moved_value = borrow of moved value
     .suggestion = borrow this binding in the pattern to avoid moving the value
 
 mir_build_multiple_mut_borrows = cannot borrow value as mutable more than once at a time
-    .label = first mutable borrow, by `{$name}`, occurs here
-    .mutable_borrow = another mutable borrow, by `{$name_mut}`, occurs here
-    .immutable_borrow = also borrowed as immutable, by `{$name_immut}`, here
-    .moved = also moved into `{$name_moved}` here
+
+mir_build_already_borrowed = cannot borrow value as mutable because it is also borrowed as immutable
+
+mir_build_already_mut_borrowed = cannot borrow value as immutable because it is also borrowed as mutable
+
+mir_build_moved_while_borrowed = cannot move out of value because it is borrowed
+
+mir_build_mutable_borrow = value is mutably borrowed by `{$name}` here
+
+mir_build_borrow = value is borrowed by `{$name}` here
+
+mir_build_moved = value is moved into `{$name}` here
 
 mir_build_union_pattern = cannot use unions in constant patterns
 
@@ -366,3 +374,9 @@ mir_build_suggest_let_else = you might want to use `let else` to handle the {$co
     } matched
 
 mir_build_suggest_attempted_int_lit = alternatively, you could prepend the pattern with an underscore to define a new named variable; identifiers cannot begin with digits
+
+
+mir_build_rustc_box_attribute_error = `#[rustc_box]` attribute used incorrectly
+    .attributes = no other attributes may be applied
+    .not_box = `#[rustc_box]` may only be applied to a `Box::new()` call
+    .missing_box = `#[rustc_box]` requires the `owned_box` lang item
diff --git a/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs b/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs
index 0bca02589..dbba529ae 100644
--- a/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs
+++ b/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs
@@ -18,6 +18,9 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
             @call("mir_storage_dead", args) => {
                 Ok(StatementKind::StorageDead(self.parse_local(args[0])?))
             },
+            @call("mir_deinit", args) => {
+                Ok(StatementKind::Deinit(Box::new(self.parse_place(args[0])?)))
+            },
             @call("mir_retag", args) => {
                 Ok(StatementKind::Retag(RetagKind::Default, Box::new(self.parse_place(args[0])?)))
             },
@@ -141,12 +144,29 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
     fn parse_rvalue(&self, expr_id: ExprId) -> PResult<Rvalue<'tcx>> {
         parse_by_kind!(self, expr_id, _, "rvalue",
             @call("mir_discriminant", args) => self.parse_place(args[0]).map(Rvalue::Discriminant),
+            @call("mir_checked", args) => {
+                parse_by_kind!(self, args[0], _, "binary op",
+                    ExprKind::Binary { op, lhs, rhs } => Ok(Rvalue::CheckedBinaryOp(
+                        *op, Box::new((self.parse_operand(*lhs)?, self.parse_operand(*rhs)?))
+                    )),
+                )
+            },
+            @call("mir_len", args) => Ok(Rvalue::Len(self.parse_place(args[0])?)),
             ExprKind::Borrow { borrow_kind, arg } => Ok(
                 Rvalue::Ref(self.tcx.lifetimes.re_erased, *borrow_kind, self.parse_place(*arg)?)
             ),
             ExprKind::AddressOf { mutability, arg } => Ok(
                 Rvalue::AddressOf(*mutability, self.parse_place(*arg)?)
             ),
+            ExprKind::Binary { op, lhs, rhs } =>  Ok(
+                Rvalue::BinaryOp(*op, Box::new((self.parse_operand(*lhs)?, self.parse_operand(*rhs)?)))
+            ),
+            ExprKind::Unary { op, arg } => Ok(
+                Rvalue::UnaryOp(*op, self.parse_operand(*arg)?)
+            ),
+            ExprKind::Repeat { value, count } => Ok(
+                Rvalue::Repeat(self.parse_operand(*value)?, *count)
+            ),
             _ => self.parse_operand(expr_id).map(Rvalue::Use),
         )
     }
diff --git a/compiler/rustc_mir_build/src/build/expr/as_constant.rs b/compiler/rustc_mir_build/src/build/expr/as_constant.rs
index 1d96893c7..cfacb5ea3 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_constant.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_constant.rs
@@ -55,7 +55,7 @@ pub fn as_constant_inner<'tcx>(
                         ConstantKind::Ty(tcx.const_error_with_guaranteed(ty, guar))
                     }
                     Err(LitToConstError::TypeError) => {
-                        bug!("encountered type error in `lit_to_mir_constant")
+                        bug!("encountered type error in `lit_to_mir_constant`")
                     }
                 };
 
@@ -132,14 +132,14 @@ pub(crate) fn lit_to_mir_constant<'tcx>(
         (ast::LitKind::Str(s, _), ty::Ref(_, inner_ty, _)) if inner_ty.is_str() => {
             let s = s.as_str();
             let allocation = Allocation::from_bytes_byte_aligned_immutable(s.as_bytes());
-            let allocation = tcx.intern_const_alloc(allocation);
+            let allocation = tcx.mk_const_alloc(allocation);
             ConstValue::Slice { data: allocation, start: 0, end: s.len() }
         }
         (ast::LitKind::ByteStr(data, _), ty::Ref(_, inner_ty, _))
             if matches!(inner_ty.kind(), ty::Slice(_)) =>
         {
             let allocation = Allocation::from_bytes_byte_aligned_immutable(data as &[u8]);
-            let allocation = tcx.intern_const_alloc(allocation);
+            let allocation = tcx.mk_const_alloc(allocation);
             ConstValue::Slice { data: allocation, start: 0, end: data.len() }
         }
         (ast::LitKind::ByteStr(data, _), ty::Ref(_, inner_ty, _)) if inner_ty.is_array() => {
diff --git a/compiler/rustc_mir_build/src/build/expr/as_operand.rs b/compiler/rustc_mir_build/src/build/expr/as_operand.rs
index c621efb3b..ff3198847 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_operand.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_operand.rs
@@ -170,7 +170,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     // Return the operand *tmp0 to be used as the call argument
                     let place = Place {
                         local: operand,
-                        projection: tcx.intern_place_elems(&[PlaceElem::Deref]),
+                        projection: tcx.mk_place_elems(&[PlaceElem::Deref]),
                     };
 
                     return block.and(Operand::Move(place));
diff --git a/compiler/rustc_mir_build/src/build/expr/as_place.rs b/compiler/rustc_mir_build/src/build/expr/as_place.rs
index e22fa6365..33200b80a 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_place.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_place.rs
@@ -11,7 +11,7 @@ use rustc_middle::mir::AssertKind::BoundsCheck;
 use rustc_middle::mir::*;
 use rustc_middle::thir::*;
 use rustc_middle::ty::AdtDef;
-use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, Ty, TyCtxt, Variance};
+use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, Ty, Variance};
 use rustc_span::Span;
 use rustc_target::abi::VariantIdx;
 
@@ -183,7 +183,7 @@ fn to_upvars_resolved_place_builder<'tcx>(
             &projection,
         ) else {
         let closure_span = cx.tcx.def_span(closure_def_id);
-        if !enable_precise_capture(cx.tcx, closure_span) {
+        if !enable_precise_capture(closure_span) {
             bug!(
                 "No associated capture found for {:?}[{:#?}] even though \
                     capture_disjoint_fields isn't enabled",
@@ -263,7 +263,7 @@ impl<'tcx> PlaceBuilder<'tcx> {
         let resolved = self.resolve_upvar(cx);
         let builder = resolved.as_ref().unwrap_or(self);
         let PlaceBase::Local(local) = builder.base else { return None };
-        let projection = cx.tcx.intern_place_elems(&builder.projection);
+        let projection = cx.tcx.mk_place_elems(&builder.projection);
         Some(Place { local, projection })
     }
 
@@ -692,7 +692,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             tcx.mk_imm_ref(tcx.lifetimes.re_erased, fake_borrow_deref_ty);
                         let fake_borrow_temp =
                             self.local_decls.push(LocalDecl::new(fake_borrow_ty, expr_span));
-                        let projection = tcx.intern_place_elems(&base_place.projection[..idx]);
+                        let projection = tcx.mk_place_elems(&base_place.projection[..idx]);
                         self.cfg.push_assign(
                             block,
                             source_info,
@@ -745,8 +745,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     }
 }
 
-/// Precise capture is enabled if the feature gate `capture_disjoint_fields` is enabled or if
-/// user is using Rust Edition 2021 or higher.
-fn enable_precise_capture(tcx: TyCtxt<'_>, closure_span: Span) -> bool {
-    tcx.features().capture_disjoint_fields || closure_span.rust_2021()
+/// Precise capture is enabled if user is using Rust Edition 2021 or higher.
+fn enable_precise_capture(closure_span: Span) -> bool {
+    closure_span.rust_2021()
 }
diff --git a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
index c7b3eb44d..a4e48c154 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
@@ -55,7 +55,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 })
             }
             ExprKind::Repeat { value, count } => {
-                if Some(0) == count.try_eval_usize(this.tcx, this.param_env) {
+                if Some(0) == count.try_eval_target_usize(this.tcx, this.param_env) {
                     this.build_zero_repeat(block, value, scope, source_info)
                 } else {
                     let value_operand = unpack!(
@@ -439,10 +439,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         // We implicitly set the discriminant to 0. See
                         // librustc_mir/transform/deaggregator.rs for details.
                         let movability = movability.unwrap();
-                        Box::new(AggregateKind::Generator(closure_id, substs, movability))
+                        Box::new(AggregateKind::Generator(
+                            closure_id.to_def_id(),
+                            substs,
+                            movability,
+                        ))
                     }
                     UpvarSubsts::Closure(substs) => {
-                        Box::new(AggregateKind::Closure(closure_id, substs))
+                        Box::new(AggregateKind::Closure(closure_id.to_def_id(), substs))
                     }
                 };
                 block.and(Rvalue::Aggregate(result, operands))
@@ -516,7 +520,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let source_info = self.source_info(span);
         let bool_ty = self.tcx.types.bool;
         if self.check_overflow && op.is_checkable() && ty.is_integral() {
-            let result_tup = self.tcx.intern_tup(&[ty, bool_ty]);
+            let result_tup = self.tcx.mk_tup(&[ty, bool_ty]);
             let result_value = self.temp(result_tup, span);
 
             self.cfg.push_assign(
diff --git a/compiler/rustc_mir_build/src/build/expr/as_temp.rs b/compiler/rustc_mir_build/src/build/expr/as_temp.rs
index 0ca4e3745..3d3cf7555 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_temp.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_temp.rs
@@ -44,7 +44,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let expr_ty = expr.ty;
         let temp = {
             let mut local_decl = LocalDecl::new(expr_ty, expr_span);
-            if mutability == Mutability::Not {
+            if mutability.is_not() {
                 local_decl = local_decl.immutable();
             }
 
diff --git a/compiler/rustc_mir_build/src/build/expr/into.rs b/compiler/rustc_mir_build/src/build/expr/into.rs
index 38b1fa91d..dac9bf0a8 100644
--- a/compiler/rustc_mir_build/src/build/expr/into.rs
+++ b/compiler/rustc_mir_build/src/build/expr/into.rs
@@ -319,7 +319,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 // See the notes for `ExprKind::Array` in `as_rvalue` and for
                 // `ExprKind::Borrow` above.
                 let is_union = adt_def.is_union();
-                let active_field_index = if is_union { Some(fields[0].name.index()) } else { None };
+                let active_field_index = is_union.then(|| fields[0].name.index());
 
                 let scope = this.local_scope();
 
diff --git a/compiler/rustc_mir_build/src/build/matches/mod.rs b/compiler/rustc_mir_build/src/build/matches/mod.rs
index 0961ce11e..de2851a1a 100644
--- a/compiler/rustc_mir_build/src/build/matches/mod.rs
+++ b/compiler/rustc_mir_build/src/build/matches/mod.rs
@@ -1206,7 +1206,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                     fake_borrows.insert(Place {
                         local: source.local,
-                        projection: self.tcx.intern_place_elems(proj_base),
+                        projection: self.tcx.mk_place_elems(proj_base),
                     });
                 }
             }
@@ -1743,12 +1743,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             .map(|matched_place_ref| {
                 let matched_place = Place {
                     local: matched_place_ref.local,
-                    projection: tcx.intern_place_elems(matched_place_ref.projection),
+                    projection: tcx.mk_place_elems(matched_place_ref.projection),
                 };
                 let fake_borrow_deref_ty = matched_place.ty(&self.local_decls, tcx).ty;
                 let fake_borrow_ty = tcx.mk_imm_ref(tcx.lifetimes.re_erased, fake_borrow_deref_ty);
-                let fake_borrow_temp =
-                    self.local_decls.push(LocalDecl::new(fake_borrow_ty, temp_span));
+                let mut fake_borrow_temp = LocalDecl::new(fake_borrow_ty, temp_span);
+                fake_borrow_temp.internal = self.local_decls[matched_place.local].internal;
+                fake_borrow_temp.local_info = Some(Box::new(LocalInfo::FakeBorrow));
+                let fake_borrow_temp = self.local_decls.push(fake_borrow_temp);
 
                 (matched_place, fake_borrow_temp)
             })
diff --git a/compiler/rustc_mir_build/src/build/matches/test.rs b/compiler/rustc_mir_build/src/build/matches/test.rs
index ad7a568a2..2de89f67d 100644
--- a/compiler/rustc_mir_build/src/build/matches/test.rs
+++ b/compiler/rustc_mir_build/src/build/matches/test.rs
@@ -563,14 +563,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 let not_contained =
                     self.values_not_contained_in_range(&*range, options).unwrap_or(false);
 
-                if not_contained {
+                not_contained.then(|| {
                     // No switch values are contained in the pattern range,
                     // so the pattern can be matched only if this test fails.
-                    let otherwise = options.len();
-                    Some(otherwise)
-                } else {
-                    None
-                }
+                    options.len()
+                })
             }
 
             (&TestKind::SwitchInt { .. }, _) => None,
@@ -835,7 +832,7 @@ fn trait_method<'tcx>(
     tcx: TyCtxt<'tcx>,
     trait_def_id: DefId,
     method_name: Symbol,
-    substs: impl IntoIterator<Item = impl Into<GenericArg<'tcx>>>,
+    substs: impl IntoIterator<Item: Into<GenericArg<'tcx>>>,
 ) -> ConstantKind<'tcx> {
     // The unhygienic comparison here is acceptable because this is only
     // used on known traits.
diff --git a/compiler/rustc_mir_build/src/build/matches/util.rs b/compiler/rustc_mir_build/src/build/matches/util.rs
index cbd494862..c34105174 100644
--- a/compiler/rustc_mir_build/src/build/matches/util.rs
+++ b/compiler/rustc_mir_build/src/build/matches/util.rs
@@ -5,7 +5,7 @@ use crate::build::Builder;
 use rustc_middle::mir::*;
 use rustc_middle::thir::*;
 use rustc_middle::ty;
-use rustc_middle::ty::TypeVisitable;
+use rustc_middle::ty::TypeVisitableExt;
 use smallvec::SmallVec;
 
 impl<'a, 'tcx> Builder<'a, 'tcx> {
@@ -35,7 +35,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let tcx = self.tcx;
         let (min_length, exact_size) = if let Some(place_resolved) = place.try_to_place(self) {
             match place_resolved.ty(&self.local_decls, tcx).ty.kind() {
-                ty::Array(_, length) => (length.eval_usize(tcx, self.param_env), true),
+                ty::Array(_, length) => (length.eval_target_usize(tcx, self.param_env), true),
                 _ => ((prefix.len() + suffix.len()).try_into().unwrap(), false),
             }
         } else {
diff --git a/compiler/rustc_mir_build/src/build/mod.rs b/compiler/rustc_mir_build/src/build/mod.rs
index 9daf68a15..b3f9d8282 100644
--- a/compiler/rustc_mir_build/src/build/mod.rs
+++ b/compiler/rustc_mir_build/src/build/mod.rs
@@ -20,7 +20,7 @@ use rustc_middle::mir::*;
 use rustc_middle::thir::{
     self, BindingMode, Expr, ExprId, LintLevel, LocalVarId, Param, ParamId, PatKind, Thir,
 };
-use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitable, TypeckResults};
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitableExt};
 use rustc_span::symbol::sym;
 use rustc_span::Span;
 use rustc_span::Symbol;
@@ -47,8 +47,6 @@ pub(crate) fn mir_built(
 
 /// Construct the MIR for a given `DefId`.
 fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_> {
-    let body_owner_kind = tcx.hir().body_owner_kind(def.did);
-
     // Ensure unsafeck and abstract const building is ran before we steal the THIR.
     // We can't use `ensure()` for `thir_abstract_const` as it doesn't compute the query
     // if inputs are green. This can cause ICEs when calling `thir_abstract_const` after
@@ -65,16 +63,15 @@ fn mir_build(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_
     }
 
     let body = match tcx.thir_body(def) {
-        Err(error_reported) => construct_error(tcx, def.did, body_owner_kind, error_reported),
+        Err(error_reported) => construct_error(tcx, def.did, error_reported),
         Ok((thir, expr)) => {
             // We ran all queries that depended on THIR at the beginning
             // of `mir_build`, so now we can steal it
             let thir = thir.steal();
 
-            if body_owner_kind.is_fn_or_closure() {
-                construct_fn(tcx, def, &thir, expr)
-            } else {
-                construct_const(tcx, def, &thir, expr)
+            match thir.body_type {
+                thir::BodyTy::Fn(fn_sig) => construct_fn(tcx, def, &thir, expr, fn_sig),
+                thir::BodyTy::Const(ty) => construct_const(tcx, def, &thir, expr, ty),
             }
         }
     };
@@ -158,13 +155,13 @@ struct BlockContext(Vec<BlockFrame>);
 struct Builder<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
     infcx: InferCtxt<'tcx>,
-    typeck_results: &'tcx TypeckResults<'tcx>,
     region_scope_tree: &'tcx region::ScopeTree,
     param_env: ty::ParamEnv<'tcx>,
 
     thir: &'a Thir<'tcx>,
     cfg: CFG<'tcx>,
 
+    def: ty::WithOptConstParam<LocalDefId>,
     def_id: DefId,
     hir_id: hir::HirId,
     parent_module: DefId,
@@ -434,11 +431,16 @@ fn construct_fn<'tcx>(
     fn_def: ty::WithOptConstParam<LocalDefId>,
     thir: &Thir<'tcx>,
     expr: ExprId,
+    fn_sig: ty::FnSig<'tcx>,
 ) -> Body<'tcx> {
     let span = tcx.def_span(fn_def.did);
     let fn_id = tcx.hir().local_def_id_to_hir_id(fn_def.did);
     let generator_kind = tcx.generator_kind(fn_def.did);
 
+    // The representation of thir for `-Zunpretty=thir-tree` relies on
+    // the entry expression being the last element of `thir.exprs`.
+    assert_eq!(expr.as_usize(), thir.exprs.len() - 1);
+
     // Figure out what primary body this item has.
     let body_id = tcx.hir().body_owned_by(fn_def.did);
     let span_with_body = tcx.hir().span_with_body(fn_id);
@@ -449,11 +451,6 @@ fn construct_fn<'tcx>(
         .output
         .span();
 
-    // fetch the fully liberated fn signature (that is, all bound
-    // types/lifetimes replaced)
-    let typeck_results = tcx.typeck_opt_const_arg(fn_def);
-    let fn_sig = typeck_results.liberated_fn_sigs()[fn_id];
-
     let safety = match fn_sig.unsafety {
         hir::Unsafety::Normal => Safety::Safe,
         hir::Unsafety::Unsafe => Safety::FnUnsafe,
@@ -525,13 +522,7 @@ fn construct_fn<'tcx>(
         let return_block =
             unpack!(builder.in_breakable_scope(None, Place::return_place(), fn_end, |builder| {
                 Some(builder.in_scope(arg_scope_s, LintLevel::Inherited, |builder| {
-                    builder.args_and_body(
-                        START_BLOCK,
-                        fn_def.did,
-                        arguments,
-                        arg_scope,
-                        &thir[expr],
-                    )
+                    builder.args_and_body(START_BLOCK, arguments, arg_scope, &thir[expr])
                 }))
             }));
         let source_info = builder.source_info(fn_end);
@@ -559,6 +550,7 @@ fn construct_const<'a, 'tcx>(
     def: ty::WithOptConstParam<LocalDefId>,
     thir: &'a Thir<'tcx>,
     expr: ExprId,
+    const_ty: Ty<'tcx>,
 ) -> Body<'tcx> {
     let hir_id = tcx.hir().local_def_id_to_hir_id(def.did);
 
@@ -582,20 +574,6 @@ fn construct_const<'a, 'tcx>(
         _ => span_bug!(tcx.def_span(def.did), "can't build MIR for {:?}", def.did),
     };
 
-    // Get the revealed type of this const. This is *not* the adjusted
-    // type of its body, which may be a subtype of this type. For
-    // example:
-    //
-    // fn foo(_: &()) {}
-    // static X: fn(&'static ()) = foo;
-    //
-    // The adjusted type of the body of X is `for<'a> fn(&'a ())` which
-    // is not the same as the type of X. We need the type of the return
-    // place to be the type of the constant because NLL typeck will
-    // equate them.
-    let typeck_results = tcx.typeck_opt_const_arg(def);
-    let const_ty = typeck_results.node_type(hir_id);
-
     let infcx = tcx.infer_ctxt().build();
     let mut builder = Builder::new(
         thir,
@@ -625,21 +603,17 @@ fn construct_const<'a, 'tcx>(
 ///
 /// This is required because we may still want to run MIR passes on an item
 /// with type errors, but normal MIR construction can't handle that in general.
-fn construct_error(
-    tcx: TyCtxt<'_>,
-    def: LocalDefId,
-    body_owner_kind: hir::BodyOwnerKind,
-    err: ErrorGuaranteed,
-) -> Body<'_> {
+fn construct_error(tcx: TyCtxt<'_>, def: LocalDefId, err: ErrorGuaranteed) -> Body<'_> {
     let span = tcx.def_span(def);
     let hir_id = tcx.hir().local_def_id_to_hir_id(def);
     let generator_kind = tcx.generator_kind(def);
+    let body_owner_kind = tcx.hir().body_owner_kind(def);
 
-    let ty = tcx.ty_error();
+    let ty = tcx.ty_error(err);
     let num_params = match body_owner_kind {
-        hir::BodyOwnerKind::Fn => tcx.fn_sig(def).inputs().skip_binder().len(),
+        hir::BodyOwnerKind::Fn => tcx.fn_sig(def).skip_binder().inputs().skip_binder().len(),
         hir::BodyOwnerKind::Closure => {
-            let ty = tcx.type_of(def);
+            let ty = tcx.type_of(def).subst_identity();
             match ty.kind() {
                 ty::Closure(_, substs) => {
                     1 + substs.as_closure().sig().inputs().skip_binder().len()
@@ -724,9 +698,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             thir,
             tcx,
             infcx,
-            typeck_results: tcx.typeck_opt_const_arg(def),
             region_scope_tree: tcx.region_scope_tree(def.did),
             param_env,
+            def,
             def_id: def.did.to_def_id(),
             hir_id,
             parent_module: tcx.parent_module(hir_id).to_def_id(),
@@ -776,14 +750,78 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             self.var_debug_info,
             self.fn_span,
             self.generator_kind,
-            self.typeck_results.tainted_by_errors,
+            None,
         )
     }
 
+    fn insert_upvar_arg(&mut self) {
+        let Some(closure_arg) = self.local_decls.get(ty::CAPTURE_STRUCT_LOCAL) else { return };
+
+        let mut closure_ty = closure_arg.ty;
+        let mut closure_env_projs = vec![];
+        if let ty::Ref(_, ty, _) = closure_ty.kind() {
+            closure_env_projs.push(ProjectionElem::Deref);
+            closure_ty = *ty;
+        }
+
+        let upvar_substs = match closure_ty.kind() {
+            ty::Closure(_, substs) => ty::UpvarSubsts::Closure(substs),
+            ty::Generator(_, substs, _) => ty::UpvarSubsts::Generator(substs),
+            _ => return,
+        };
+
+        // In analyze_closure() in upvar.rs we gathered a list of upvars used by an
+        // indexed closure and we stored in a map called closure_min_captures in TypeckResults
+        // with the closure's DefId. Here, we run through that vec of UpvarIds for
+        // the given closure and use the necessary information to create upvar
+        // debuginfo and to fill `self.upvars`.
+        let capture_tys = upvar_substs.upvar_tys();
+
+        let tcx = self.tcx;
+        self.upvars = tcx
+            .closure_captures(self.def.did)
+            .iter()
+            .zip(capture_tys)
+            .enumerate()
+            .map(|(i, (captured_place, ty))| {
+                let name = captured_place.to_symbol();
+
+                let capture = captured_place.info.capture_kind;
+                let var_id = match captured_place.place.base {
+                    HirPlaceBase::Upvar(upvar_id) => upvar_id.var_path.hir_id,
+                    _ => bug!("Expected an upvar"),
+                };
+
+                let mutability = captured_place.mutability;
+
+                let mut projs = closure_env_projs.clone();
+                projs.push(ProjectionElem::Field(Field::new(i), ty));
+                match capture {
+                    ty::UpvarCapture::ByValue => {}
+                    ty::UpvarCapture::ByRef(..) => {
+                        projs.push(ProjectionElem::Deref);
+                    }
+                };
+
+                let use_place = Place {
+                    local: ty::CAPTURE_STRUCT_LOCAL,
+                    projection: tcx.mk_place_elems(&projs),
+                };
+                self.var_debug_info.push(VarDebugInfo {
+                    name,
+                    source_info: SourceInfo::outermost(captured_place.var_ident.span),
+                    value: VarDebugInfoContents::Place(use_place),
+                });
+
+                let capture = Capture { captured_place, use_place, mutability };
+                (var_id, capture)
+            })
+            .collect();
+    }
+
     fn args_and_body(
         &mut self,
         mut block: BasicBlock,
-        fn_def_id: LocalDefId,
         arguments: &IndexVec<ParamId, Param<'tcx>>,
         argument_scope: region::Scope,
         expr: &Expr<'tcx>,
@@ -805,69 +843,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
         }
 
-        let tcx = self.tcx;
-        let tcx_hir = tcx.hir();
-        let hir_typeck_results = self.typeck_results;
-
-        // In analyze_closure() in upvar.rs we gathered a list of upvars used by an
-        // indexed closure and we stored in a map called closure_min_captures in TypeckResults
-        // with the closure's DefId. Here, we run through that vec of UpvarIds for
-        // the given closure and use the necessary information to create upvar
-        // debuginfo and to fill `self.upvars`.
-        if hir_typeck_results.closure_min_captures.get(&fn_def_id).is_some() {
-            let mut closure_env_projs = vec![];
-            let mut closure_ty = self.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty;
-            if let ty::Ref(_, ty, _) = closure_ty.kind() {
-                closure_env_projs.push(ProjectionElem::Deref);
-                closure_ty = *ty;
-            }
-            let upvar_substs = match closure_ty.kind() {
-                ty::Closure(_, substs) => ty::UpvarSubsts::Closure(substs),
-                ty::Generator(_, substs, _) => ty::UpvarSubsts::Generator(substs),
-                _ => span_bug!(self.fn_span, "upvars with non-closure env ty {:?}", closure_ty),
-            };
-            let def_id = self.def_id.as_local().unwrap();
-            let capture_syms = tcx.symbols_for_closure_captures((def_id, fn_def_id));
-            let capture_tys = upvar_substs.upvar_tys();
-            let captures_with_tys = hir_typeck_results
-                .closure_min_captures_flattened(fn_def_id)
-                .zip(capture_tys.zip(capture_syms));
-
-            self.upvars = captures_with_tys
-                .enumerate()
-                .map(|(i, (captured_place, (ty, sym)))| {
-                    let capture = captured_place.info.capture_kind;
-                    let var_id = match captured_place.place.base {
-                        HirPlaceBase::Upvar(upvar_id) => upvar_id.var_path.hir_id,
-                        _ => bug!("Expected an upvar"),
-                    };
-
-                    let mutability = captured_place.mutability;
-
-                    let mut projs = closure_env_projs.clone();
-                    projs.push(ProjectionElem::Field(Field::new(i), ty));
-                    match capture {
-                        ty::UpvarCapture::ByValue => {}
-                        ty::UpvarCapture::ByRef(..) => {
-                            projs.push(ProjectionElem::Deref);
-                        }
-                    };
-
-                    let use_place = Place {
-                        local: ty::CAPTURE_STRUCT_LOCAL,
-                        projection: tcx.intern_place_elems(&projs),
-                    };
-                    self.var_debug_info.push(VarDebugInfo {
-                        name: *sym,
-                        source_info: SourceInfo::outermost(tcx_hir.span(var_id)),
-                        value: VarDebugInfoContents::Place(use_place),
-                    });
-
-                    let capture = Capture { captured_place, use_place, mutability };
-                    (var_id, capture)
-                })
-                .collect();
-        }
+        self.insert_upvar_arg();
 
         let mut scope = None;
         // Bind the argument patterns
diff --git a/compiler/rustc_mir_build/src/errors.rs b/compiler/rustc_mir_build/src/errors.rs
index 7f81aef1c..dc4d2276e 100644
--- a/compiler/rustc_mir_build/src/errors.rs
+++ b/compiler/rustc_mir_build/src/errors.rs
@@ -1,9 +1,10 @@
-use crate::thir::pattern::deconstruct_pat::DeconstructedPat;
-use crate::thir::pattern::MatchCheckCtxt;
-use rustc_errors::Handler;
+use crate::{
+    fluent_generated as fluent,
+    thir::pattern::{deconstruct_pat::DeconstructedPat, MatchCheckCtxt},
+};
 use rustc_errors::{
     error_code, AddToDiagnostic, Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed,
-    IntoDiagnostic, MultiSpan, SubdiagnosticMessage,
+    Handler, IntoDiagnostic, MultiSpan, SubdiagnosticMessage,
 };
 use rustc_hir::def::Res;
 use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
@@ -358,7 +359,7 @@ impl<'a> IntoDiagnostic<'a> for NonExhaustivePatternsTypeNotEmpty<'_, '_, '_> {
     fn into_diagnostic(self, handler: &'a Handler) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
         let mut diag = handler.struct_span_err_with_code(
             self.span,
-            rustc_errors::fluent::mir_build_non_exhaustive_patterns_type_not_empty,
+            fluent::mir_build_non_exhaustive_patterns_type_not_empty,
             error_code!(E0004),
         );
 
@@ -380,7 +381,7 @@ impl<'a> IntoDiagnostic<'a> for NonExhaustivePatternsTypeNotEmpty<'_, '_, '_> {
             let mut span: MultiSpan = def_span.into();
             span.push_span_label(def_span, "");
 
-            diag.span_note(span, rustc_errors::fluent::def_note);
+            diag.span_note(span, fluent::mir_build_def_note);
         }
 
         let is_variant_list_non_exhaustive = match self.ty.kind() {
@@ -391,14 +392,14 @@ impl<'a> IntoDiagnostic<'a> for NonExhaustivePatternsTypeNotEmpty<'_, '_, '_> {
         };
 
         if is_variant_list_non_exhaustive {
-            diag.note(rustc_errors::fluent::non_exhaustive_type_note);
+            diag.note(fluent::mir_build_non_exhaustive_type_note);
         } else {
-            diag.note(rustc_errors::fluent::type_note);
+            diag.note(fluent::mir_build_type_note);
         }
 
         if let ty::Ref(_, sub_ty, _) = self.ty.kind() {
             if !sub_ty.is_inhabited_from(self.cx.tcx, self.cx.module, self.cx.param_env) {
-                diag.note(rustc_errors::fluent::reference_note);
+                diag.note(fluent::mir_build_reference_note);
             }
         }
 
@@ -424,12 +425,12 @@ impl<'a> IntoDiagnostic<'a> for NonExhaustivePatternsTypeNotEmpty<'_, '_, '_> {
         if let Some((span, sugg)) = suggestion {
             diag.span_suggestion_verbose(
                 span,
-                rustc_errors::fluent::suggestion,
+                fluent::mir_build_suggestion,
                 sugg,
                 Applicability::HasPlaceholders,
             );
         } else {
-            diag.help(rustc_errors::fluent::help);
+            diag.help(fluent::mir_build_help);
         }
 
         diag
@@ -469,7 +470,7 @@ pub struct NonConstPath {
 pub struct UnreachablePattern {
     #[label]
     pub span: Option<Span>,
-    #[label(catchall_label)]
+    #[label(mir_build_catchall_label)]
     pub catchall: Option<Span>,
 }
 
@@ -493,7 +494,7 @@ pub struct LowerRangeBoundMustBeLessThanOrEqualToUpper {
     #[primary_span]
     #[label]
     pub span: Span,
-    #[note(teach_note)]
+    #[note(mir_build_teach_note)]
     pub teach: Option<()>,
 }
 
@@ -585,9 +586,9 @@ pub struct BorrowOfMovedValue<'tcx> {
     #[primary_span]
     pub span: Span,
     #[label]
-    #[label(occurs_because_label)]
+    #[label(mir_build_occurs_because_label)]
     pub binding_span: Span,
-    #[label(value_borrowed_label)]
+    #[label(mir_build_value_borrowed_label)]
     pub conflicts_ref: Vec<Span>,
     pub name: Ident,
     pub ty: Ty<'tcx>,
@@ -600,32 +601,56 @@ pub struct BorrowOfMovedValue<'tcx> {
 pub struct MultipleMutBorrows {
     #[primary_span]
     pub span: Span,
-    #[label]
-    pub binding_span: Span,
     #[subdiagnostic]
-    pub occurences: Vec<MultipleMutBorrowOccurence>,
-    pub name: Ident,
+    pub occurences: Vec<Conflict>,
+}
+
+#[derive(Diagnostic)]
+#[diag(mir_build_already_borrowed)]
+pub struct AlreadyBorrowed {
+    #[primary_span]
+    pub span: Span,
+    #[subdiagnostic]
+    pub occurences: Vec<Conflict>,
+}
+
+#[derive(Diagnostic)]
+#[diag(mir_build_already_mut_borrowed)]
+pub struct AlreadyMutBorrowed {
+    #[primary_span]
+    pub span: Span,
+    #[subdiagnostic]
+    pub occurences: Vec<Conflict>,
+}
+
+#[derive(Diagnostic)]
+#[diag(mir_build_moved_while_borrowed)]
+pub struct MovedWhileBorrowed {
+    #[primary_span]
+    pub span: Span,
+    #[subdiagnostic]
+    pub occurences: Vec<Conflict>,
 }
 
 #[derive(Subdiagnostic)]
-pub enum MultipleMutBorrowOccurence {
-    #[label(mutable_borrow)]
-    Mutable {
+pub enum Conflict {
+    #[label(mir_build_mutable_borrow)]
+    Mut {
         #[primary_span]
         span: Span,
-        name_mut: Ident,
+        name: Ident,
     },
-    #[label(immutable_borrow)]
-    Immutable {
+    #[label(mir_build_borrow)]
+    Ref {
         #[primary_span]
         span: Span,
-        name_immut: Ident,
+        name: Ident,
     },
-    #[label(moved)]
+    #[label(mir_build_moved)]
     Moved {
         #[primary_span]
         span: Span,
-        name_moved: Ident,
+        name: Ident,
     },
 }
 
@@ -684,7 +709,7 @@ pub struct NontrivialStructuralMatch<'tcx> {
 #[diag(mir_build_overlapping_range_endpoints)]
 #[note]
 pub struct OverlappingRangeEndpoints<'tcx> {
-    #[label(range)]
+    #[label(mir_build_range)]
     pub range: Span,
     #[subdiagnostic]
     pub overlap: Vec<Overlap<'tcx>>,
@@ -764,7 +789,7 @@ pub(crate) struct PatternNotCovered<'s, 'tcx> {
     pub interpreted_as_const: Option<InterpretedAsConst>,
     #[subdiagnostic]
     pub adt_defined_here: Option<AdtDefinedHere<'tcx>>,
-    #[note(pattern_ty)]
+    #[note(mir_build_pattern_ty)]
     pub _p: (),
     pub pattern_ty: Ty<'tcx>,
     #[subdiagnostic]
@@ -799,10 +824,10 @@ impl<'tcx> AddToDiagnostic for AdtDefinedHere<'tcx> {
         let mut spans = MultiSpan::from(self.adt_def_span);
 
         for Variant { span } in self.variants {
-            spans.push_span_label(span, rustc_errors::fluent::mir_build_variant_defined_here);
+            spans.push_span_label(span, fluent::mir_build_variant_defined_here);
         }
 
-        diag.span_note(spans, rustc_errors::fluent::mir_build_adt_defined_here);
+        diag.span_note(spans, fluent::mir_build_adt_defined_here);
     }
 }
 
@@ -863,3 +888,22 @@ pub enum MiscPatternSuggestion {
         start_span: Span,
     },
 }
+
+#[derive(Diagnostic)]
+#[diag(mir_build_rustc_box_attribute_error)]
+pub struct RustcBoxAttributeError {
+    #[primary_span]
+    pub span: Span,
+    #[subdiagnostic]
+    pub reason: RustcBoxAttrReason,
+}
+
+#[derive(Subdiagnostic)]
+pub enum RustcBoxAttrReason {
+    #[note(mir_build_attributes)]
+    Attributes,
+    #[note(mir_build_not_box)]
+    NotBoxNew,
+    #[note(mir_build_missing_box)]
+    MissingBox,
+}
diff --git a/compiler/rustc_mir_build/src/lib.rs b/compiler/rustc_mir_build/src/lib.rs
index a428180a4..e10a264d3 100644
--- a/compiler/rustc_mir_build/src/lib.rs
+++ b/compiler/rustc_mir_build/src/lib.rs
@@ -25,6 +25,11 @@ pub mod thir;
 
 use rustc_middle::ty::query::Providers;
 
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
+use rustc_macros::fluent_messages;
+
+fluent_messages! { "../locales/en-US.ftl" }
+
 pub fn provide(providers: &mut Providers) {
     providers.check_match = thir::pattern::check_match;
     providers.lit_to_const = thir::constant::lit_to_const;
@@ -33,5 +38,6 @@ pub fn provide(providers: &mut Providers) {
     providers.thir_check_unsafety = check_unsafety::thir_check_unsafety;
     providers.thir_check_unsafety_for_const_arg = check_unsafety::thir_check_unsafety_for_const_arg;
     providers.thir_body = thir::cx::thir_body;
-    providers.thir_tree = thir::cx::thir_tree;
+    providers.thir_tree = thir::print::thir_tree;
+    providers.thir_flat = thir::print::thir_flat;
 }
diff --git a/compiler/rustc_mir_build/src/thir/cx/expr.rs b/compiler/rustc_mir_build/src/thir/cx/expr.rs
index 261b95ba9..9086412c0 100644
--- a/compiler/rustc_mir_build/src/thir/cx/expr.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/expr.rs
@@ -1,3 +1,4 @@
+use crate::errors;
 use crate::thir::cx::region::Scope;
 use crate::thir::cx::Cx;
 use crate::thir::util::UserAnnotatedTyHelpers;
@@ -18,7 +19,7 @@ use rustc_middle::ty::subst::InternalSubsts;
 use rustc_middle::ty::{
     self, AdtKind, InlineConstSubsts, InlineConstSubstsParts, ScalarInt, Ty, UpvarSubsts, UserType,
 };
-use rustc_span::Span;
+use rustc_span::{sym, Span};
 use rustc_target::abi::VariantIdx;
 
 impl<'tcx> Cx<'tcx> {
@@ -33,8 +34,6 @@ impl<'tcx> Cx<'tcx> {
 
     #[instrument(level = "trace", skip(self, hir_expr))]
     pub(super) fn mirror_expr_inner(&mut self, hir_expr: &'tcx hir::Expr<'tcx>) -> ExprId {
-        let temp_lifetime =
-            self.rvalue_scopes.temporary_scope(self.region_scope_tree, hir_expr.hir_id.local_id);
         let expr_scope =
             region::Scope { id: hir_expr.hir_id.local_id, data: region::ScopeData::Node };
 
@@ -67,7 +66,7 @@ impl<'tcx> Cx<'tcx> {
 
         // Next, wrap this up in the expr's scope.
         expr = Expr {
-            temp_lifetime,
+            temp_lifetime: expr.temp_lifetime,
             ty: expr.ty,
             span: hir_expr.span,
             kind: ExprKind::Scope {
@@ -82,7 +81,7 @@ impl<'tcx> Cx<'tcx> {
             self.region_scope_tree.opt_destruction_scope(hir_expr.hir_id.local_id)
         {
             expr = Expr {
-                temp_lifetime,
+                temp_lifetime: expr.temp_lifetime,
                 ty: expr.ty,
                 span: hir_expr.span,
                 kind: ExprKind::Scope {
@@ -262,6 +261,7 @@ impl<'tcx> Cx<'tcx> {
         }
     }
 
+    #[instrument(level = "debug", skip(self), ret)]
     fn make_mirror_unadjusted(&mut self, expr: &'tcx hir::Expr<'tcx>) -> Expr<'tcx> {
         let tcx = self.tcx;
         let expr_ty = self.typeck_results().expr_ty(expr);
@@ -307,7 +307,7 @@ impl<'tcx> Cx<'tcx> {
 
                     let arg_tys = args.iter().map(|e| self.typeck_results().expr_ty_adjusted(e));
                     let tupled_args = Expr {
-                        ty: tcx.mk_tup(arg_tys),
+                        ty: tcx.mk_tup_from_iter(arg_tys),
                         temp_lifetime,
                         span: expr.span,
                         kind: ExprKind::Tuple { fields: self.mirror_exprs(args) },
@@ -322,6 +322,34 @@ impl<'tcx> Cx<'tcx> {
                         fn_span: expr.span,
                     }
                 } else {
+                    let attrs = tcx.hir().attrs(expr.hir_id);
+                    if attrs.iter().any(|a| a.name_or_empty() == sym::rustc_box) {
+                        if attrs.len() != 1 {
+                            tcx.sess.emit_err(errors::RustcBoxAttributeError {
+                                span: attrs[0].span,
+                                reason: errors::RustcBoxAttrReason::Attributes,
+                            });
+                        } else if let Some(box_item) = tcx.lang_items().owned_box() {
+                            if let hir::ExprKind::Path(hir::QPath::TypeRelative(ty, fn_path)) = fun.kind
+                                && let hir::TyKind::Path(hir::QPath::Resolved(_, path)) = ty.kind
+                                && path.res.opt_def_id().map_or(false, |did| did == box_item)
+                                && fn_path.ident.name == sym::new
+                                && let [value] = args
+                            {
+                                return Expr { temp_lifetime, ty: expr_ty, span: expr.span, kind: ExprKind::Box { value: self.mirror_expr(value) } }
+                            } else {
+                                tcx.sess.emit_err(errors::RustcBoxAttributeError {
+                                    span: expr.span,
+                                    reason: errors::RustcBoxAttrReason::NotBoxNew
+                                });
+                            }
+                        } else {
+                            tcx.sess.emit_err(errors::RustcBoxAttributeError {
+                                span: attrs[0].span,
+                                reason: errors::RustcBoxAttrReason::MissingBox,
+                            });
+                        }
+                    }
                     let adt_data =
                         if let hir::ExprKind::Path(hir::QPath::Resolved(_, ref path)) = fun.kind {
                             // Tuple-like ADTs are represented as ExprKind::Call. We convert them here.
@@ -541,8 +569,9 @@ impl<'tcx> Cx<'tcx> {
                 let def_id = def_id.expect_local();
 
                 let upvars = self
-                    .typeck_results
-                    .closure_min_captures_flattened(def_id)
+                    .tcx
+                    .closure_captures(def_id)
+                    .iter()
                     .zip(substs.upvar_tys())
                     .map(|(captured_place, ty)| {
                         let upvars = self.capture_upvar(expr, captured_place, ty);
@@ -758,7 +787,7 @@ impl<'tcx> Cx<'tcx> {
             hir::ExprKind::Tup(ref fields) => ExprKind::Tuple { fields: self.mirror_exprs(fields) },
 
             hir::ExprKind::Yield(ref v, _) => ExprKind::Yield { value: self.mirror_expr(v) },
-            hir::ExprKind::Err => unreachable!(),
+            hir::ExprKind::Err(_) => unreachable!(),
         };
 
         Expr { temp_lifetime, ty: expr_ty, span: expr.span, kind }
diff --git a/compiler/rustc_mir_build/src/thir/cx/mod.rs b/compiler/rustc_mir_build/src/thir/cx/mod.rs
index a355e1bda..070544446 100644
--- a/compiler/rustc_mir_build/src/thir/cx/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/mod.rs
@@ -52,13 +52,6 @@ pub(crate) fn thir_body(
     Ok((tcx.alloc_steal_thir(cx.thir), expr))
 }
 
-pub(crate) fn thir_tree(tcx: TyCtxt<'_>, owner_def: ty::WithOptConstParam<LocalDefId>) -> String {
-    match thir_body(tcx, owner_def) {
-        Ok((thir, _)) => format!("{:#?}", thir.steal()),
-        Err(_) => "error".into(),
-    }
-}
-
 struct Cx<'tcx> {
     tcx: TyCtxt<'tcx>,
     thir: Thir<'tcx>,
@@ -89,9 +82,30 @@ impl<'tcx> Cx<'tcx> {
         let typeck_results = tcx.typeck_opt_const_arg(def);
         let did = def.did;
         let hir = tcx.hir();
+        let hir_id = hir.local_def_id_to_hir_id(did);
+
+        let body_type = if hir.body_owner_kind(did).is_fn_or_closure() {
+            // fetch the fully liberated fn signature (that is, all bound
+            // types/lifetimes replaced)
+            BodyTy::Fn(typeck_results.liberated_fn_sigs()[hir_id])
+        } else {
+            // Get the revealed type of this const. This is *not* the adjusted
+            // type of its body, which may be a subtype of this type. For
+            // example:
+            //
+            // fn foo(_: &()) {}
+            // static X: fn(&'static ()) = foo;
+            //
+            // The adjusted type of the body of X is `for<'a> fn(&'a ())` which
+            // is not the same as the type of X. We need the type of the return
+            // place to be the type of the constant because NLL typeck will
+            // equate them.
+            BodyTy::Const(typeck_results.node_type(hir_id))
+        };
+
         Cx {
             tcx,
-            thir: Thir::new(),
+            thir: Thir::new(body_type),
             param_env: tcx.param_env(def.did),
             region_scope_tree: tcx.region_scope_tree(def.did),
             typeck_results,
@@ -99,7 +113,7 @@ impl<'tcx> Cx<'tcx> {
             body_owner: did.to_def_id(),
             adjustment_span: None,
             apply_adjustments: hir
-                .attrs(hir.local_def_id_to_hir_id(did))
+                .attrs(hir_id)
                 .iter()
                 .all(|attr| attr.name_or_empty() != rustc_span::sym::custom_mir),
         }
@@ -123,14 +137,13 @@ impl<'tcx> Cx<'tcx> {
                     bug!("closure expr does not have closure type: {:?}", closure_ty);
                 };
 
-                let bound_vars = self.tcx.mk_bound_variable_kinds(std::iter::once(
-                    ty::BoundVariableKind::Region(ty::BrEnv),
-                ));
+                let bound_vars =
+                    self.tcx.mk_bound_variable_kinds(&[ty::BoundVariableKind::Region(ty::BrEnv)]);
                 let br = ty::BoundRegion {
                     var: ty::BoundVar::from_usize(bound_vars.len() - 1),
                     kind: ty::BrEnv,
                 };
-                let env_region = ty::ReLateBound(ty::INNERMOST, br);
+                let env_region = self.tcx.mk_re_late_bound(ty::INNERMOST, br);
                 let closure_env_ty =
                     self.tcx.closure_env_ty(closure_def_id, closure_substs, env_region).unwrap();
                 let liberated_closure_env_ty = self.tcx.erase_late_bound_regions(
@@ -183,7 +196,7 @@ impl<'tcx> Cx<'tcx> {
                 let va_list_did = self.tcx.require_lang_item(LangItem::VaList, Some(param.span));
 
                 self.tcx
-                    .bound_type_of(va_list_did)
+                    .type_of(va_list_did)
                     .subst(self.tcx, &[self.tcx.lifetimes.re_erased.into()])
             } else {
                 fn_sig.inputs()[index]
diff --git a/compiler/rustc_mir_build/src/thir/mod.rs b/compiler/rustc_mir_build/src/thir/mod.rs
index e0e6ac266..ca26cc13b 100644
--- a/compiler/rustc_mir_build/src/thir/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/mod.rs
@@ -5,9 +5,7 @@
 //! structures.
 
 pub(crate) mod constant;
-
 pub(crate) mod cx;
-
 pub(crate) mod pattern;
-
+pub(crate) mod print;
 mod util;
diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
index 34e637f59..2640ca56b 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
@@ -208,9 +208,9 @@ impl<'p, 'tcx> MatchVisitor<'_, 'p, 'tcx> {
             // Don't report arm reachability of desugared `match $iter.into_iter() { iter => .. }`
             // when the iterator is an uninhabited type. unreachable_code will trigger instead.
             hir::MatchSource::ForLoopDesugar if arms.len() == 1 => {}
-            hir::MatchSource::ForLoopDesugar | hir::MatchSource::Normal => {
-                report_arm_reachability(&cx, &report)
-            }
+            hir::MatchSource::ForLoopDesugar
+            | hir::MatchSource::Normal
+            | hir::MatchSource::FormatArgs => report_arm_reachability(&cx, &report),
             // Unreachable patterns in try and await expressions occur when one of
             // the arms are an uninhabited type. Which is OK.
             hir::MatchSource::AwaitDesugar | hir::MatchSource::TryDesugar => {}
@@ -926,58 +926,55 @@ fn check_borrow_conflicts_in_at_patterns(cx: &MatchVisitor<'_, '_, '_>, pat: &Pa
     sub.each_binding(|_, hir_id, span, name| {
         match typeck_results.extract_binding_mode(sess, hir_id, span) {
             Some(ty::BindByReference(mut_inner)) => match (mut_outer, mut_inner) {
-                (Mutability::Not, Mutability::Not) => {} // Both sides are `ref`.
-                (Mutability::Mut, Mutability::Mut) => conflicts_mut_mut.push((span, name)), // 2x `ref mut`.
-                _ => conflicts_mut_ref.push((span, name)), // `ref` + `ref mut` in either direction.
+                // Both sides are `ref`.
+                (Mutability::Not, Mutability::Not) => {}
+                // 2x `ref mut`.
+                (Mutability::Mut, Mutability::Mut) => {
+                    conflicts_mut_mut.push(Conflict::Mut { span, name })
+                }
+                (Mutability::Not, Mutability::Mut) => {
+                    conflicts_mut_ref.push(Conflict::Mut { span, name })
+                }
+                (Mutability::Mut, Mutability::Not) => {
+                    conflicts_mut_ref.push(Conflict::Ref { span, name })
+                }
             },
             Some(ty::BindByValue(_)) if is_binding_by_move(cx, hir_id) => {
-                conflicts_move.push((span, name)) // `ref mut?` + by-move conflict.
+                conflicts_move.push(Conflict::Moved { span, name }) // `ref mut?` + by-move conflict.
             }
             Some(ty::BindByValue(_)) | None => {} // `ref mut?` + by-copy is fine.
         }
     });
 
+    let report_mut_mut = !conflicts_mut_mut.is_empty();
+    let report_mut_ref = !conflicts_mut_ref.is_empty();
+    let report_move_conflict = !conflicts_move.is_empty();
+
+    let mut occurences = match mut_outer {
+        Mutability::Mut => vec![Conflict::Mut { span: binding_span, name }],
+        Mutability::Not => vec![Conflict::Ref { span: binding_span, name }],
+    };
+    occurences.extend(conflicts_mut_mut);
+    occurences.extend(conflicts_mut_ref);
+    occurences.extend(conflicts_move);
+
     // Report errors if any.
-    if !conflicts_mut_mut.is_empty() {
+    if report_mut_mut {
         // Report mutability conflicts for e.g. `ref mut x @ Some(ref mut y)`.
-        let mut occurences = vec![];
-
-        for (span, name_mut) in conflicts_mut_mut {
-            occurences.push(MultipleMutBorrowOccurence::Mutable { span, name_mut });
-        }
-        for (span, name_immut) in conflicts_mut_ref {
-            occurences.push(MultipleMutBorrowOccurence::Immutable { span, name_immut });
-        }
-        for (span, name_moved) in conflicts_move {
-            occurences.push(MultipleMutBorrowOccurence::Moved { span, name_moved });
-        }
-        sess.emit_err(MultipleMutBorrows { span: pat.span, binding_span, occurences, name });
-    } else if !conflicts_mut_ref.is_empty() {
+        sess.emit_err(MultipleMutBorrows { span: pat.span, occurences });
+    } else if report_mut_ref {
         // Report mutability conflicts for e.g. `ref x @ Some(ref mut y)` or the converse.
-        let (primary, also) = match mut_outer {
-            Mutability::Mut => ("mutable", "immutable"),
-            Mutability::Not => ("immutable", "mutable"),
+        match mut_outer {
+            Mutability::Mut => {
+                sess.emit_err(AlreadyMutBorrowed { span: pat.span, occurences });
+            }
+            Mutability::Not => {
+                sess.emit_err(AlreadyBorrowed { span: pat.span, occurences });
+            }
         };
-        let msg =
-            format!("cannot borrow value as {} because it is also borrowed as {}", also, primary);
-        let mut err = sess.struct_span_err(pat.span, &msg);
-        err.span_label(binding_span, format!("{} borrow, by `{}`, occurs here", primary, name));
-        for (span, name) in conflicts_mut_ref {
-            err.span_label(span, format!("{} borrow, by `{}`, occurs here", also, name));
-        }
-        for (span, name) in conflicts_move {
-            err.span_label(span, format!("also moved into `{}` here", name));
-        }
-        err.emit();
-    } else if !conflicts_move.is_empty() {
+    } else if report_move_conflict {
         // Report by-ref and by-move conflicts, e.g. `ref x @ y`.
-        let mut err =
-            sess.struct_span_err(pat.span, "cannot move out of value because it is borrowed");
-        err.span_label(binding_span, format!("value borrowed, by `{}`, here", name));
-        for (span, name) in conflicts_move {
-            err.span_label(span, format!("value moved into `{}` here", name));
-        }
-        err.emit();
+        sess.emit_err(MovedWhileBorrowed { span: pat.span, occurences });
     }
 }
 
diff --git a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
index 7f3519945..ff88d0013 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
@@ -192,7 +192,7 @@ impl<'tcx> ConstToPat<'tcx> {
         let obligation: PredicateObligation<'_> = predicate_for_trait_def(
             self.tcx(),
             self.param_env,
-            ObligationCause::misc(self.span, self.id),
+            ObligationCause::misc(self.span, self.id.owner.def_id),
             partial_eq_trait_id,
             0,
             [ty, ty],
diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
index aba5429da..e5b7d685c 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
@@ -172,7 +172,7 @@ impl IntRange {
         ty: Ty<'tcx>,
         end: &RangeEnd,
     ) -> Option<IntRange> {
-        if Self::is_integral(ty) {
+        Self::is_integral(ty).then(|| {
             // Perform a shift if the underlying types are signed,
             // which makes the interval arithmetic simpler.
             let bias = IntRange::signed_bias(tcx, ty);
@@ -182,10 +182,8 @@ impl IntRange {
                 // This should have been caught earlier by E0030.
                 bug!("malformed range pattern: {}..={}", lo, (hi - offset));
             }
-            Some(IntRange { range: lo..=(hi - offset), bias })
-        } else {
-            None
-        }
+            IntRange { range: lo..=(hi - offset), bias }
+        })
     }
 
     // The return value of `signed_bias` should be XORed with an endpoint to encode/decode it.
@@ -922,8 +920,8 @@ impl<'tcx> SplitWildcard<'tcx> {
         // `cx.is_uninhabited()`).
         let all_ctors = match pcx.ty.kind() {
             ty::Bool => smallvec![make_range(0, 1)],
-            ty::Array(sub_ty, len) if len.try_eval_usize(cx.tcx, cx.param_env).is_some() => {
-                let len = len.eval_usize(cx.tcx, cx.param_env) as usize;
+            ty::Array(sub_ty, len) if len.try_eval_target_usize(cx.tcx, cx.param_env).is_some() => {
+                let len = len.eval_target_usize(cx.tcx, cx.param_env) as usize;
                 if len != 0 && cx.is_uninhabited(*sub_ty) {
                     smallvec![]
                 } else {
@@ -1406,7 +1404,9 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
             }
             PatKind::Array { prefix, slice, suffix } | PatKind::Slice { prefix, slice, suffix } => {
                 let array_len = match pat.ty.kind() {
-                    ty::Array(_, length) => Some(length.eval_usize(cx.tcx, cx.param_env) as usize),
+                    ty::Array(_, length) => {
+                        Some(length.eval_target_usize(cx.tcx, cx.param_env) as usize)
+                    }
                     ty::Slice(_) => None,
                     _ => span_bug!(pat.span, "bad ty {:?} for slice pattern", pat.ty),
                 };
diff --git a/compiler/rustc_mir_build/src/thir/pattern/mod.rs b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
index 3a6ef87c9..41306dd80 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
@@ -203,11 +203,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                 if !lower_overflow && !higher_overflow {
                     self.tcx.sess.emit_err(LowerRangeBoundMustBeLessThanOrEqualToUpper {
                         span,
-                        teach: if self.tcx.sess.teach(&error_code!(E0030)) {
-                            Some(())
-                        } else {
-                            None
-                        },
+                        teach: self.tcx.sess.teach(&error_code!(E0030)).then_some(()),
                     });
                 }
                 PatKind::Wild
@@ -416,7 +412,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
             ty::Slice(..) => PatKind::Slice { prefix, slice, suffix },
             // Fixed-length array, `[T; len]`.
             ty::Array(_, len) => {
-                let len = len.eval_usize(self.tcx, self.param_env);
+                let len = len.eval_target_usize(self.tcx, self.param_env);
                 assert!(len >= prefix.len() as u64 + suffix.len() as u64);
                 PatKind::Array { prefix, slice, suffix }
             }
diff --git a/compiler/rustc_mir_build/src/thir/print.rs b/compiler/rustc_mir_build/src/thir/print.rs
new file mode 100644
index 000000000..8028227aa
--- /dev/null
+++ b/compiler/rustc_mir_build/src/thir/print.rs
@@ -0,0 +1,892 @@
+use rustc_middle::thir::*;
+use rustc_middle::ty::{self, TyCtxt};
+use rustc_span::def_id::LocalDefId;
+use std::fmt::{self, Write};
+
+pub(crate) fn thir_tree(tcx: TyCtxt<'_>, owner_def: ty::WithOptConstParam<LocalDefId>) -> String {
+    match super::cx::thir_body(tcx, owner_def) {
+        Ok((thir, _)) => {
+            let thir = thir.steal();
+            let mut printer = ThirPrinter::new(&thir);
+            printer.print();
+            printer.into_buffer()
+        }
+        Err(_) => "error".into(),
+    }
+}
+
+pub(crate) fn thir_flat(tcx: TyCtxt<'_>, owner_def: ty::WithOptConstParam<LocalDefId>) -> String {
+    match super::cx::thir_body(tcx, owner_def) {
+        Ok((thir, _)) => format!("{:#?}", thir.steal()),
+        Err(_) => "error".into(),
+    }
+}
+
+struct ThirPrinter<'a, 'tcx> {
+    thir: &'a Thir<'tcx>,
+    fmt: String,
+}
+
+const INDENT: &str = "    ";
+
+macro_rules! print_indented {
+    ($writer:ident, $s:expr, $indent_lvl:expr) => {
+        let indent = (0..$indent_lvl).map(|_| INDENT).collect::<Vec<_>>().concat();
+        writeln!($writer, "{}{}", indent, $s).expect("unable to write to ThirPrinter");
+    };
+}
+
+impl<'a, 'tcx> Write for ThirPrinter<'a, 'tcx> {
+    fn write_str(&mut self, s: &str) -> fmt::Result {
+        self.fmt.push_str(s);
+        Ok(())
+    }
+}
+
+impl<'a, 'tcx> ThirPrinter<'a, 'tcx> {
+    fn new(thir: &'a Thir<'tcx>) -> Self {
+        Self { thir, fmt: String::new() }
+    }
+
+    fn print(&mut self) {
+        print_indented!(self, "params: [", 0);
+        for param in self.thir.params.iter() {
+            self.print_param(param, 1);
+        }
+        print_indented!(self, "]", 0);
+
+        print_indented!(self, "body:", 0);
+        let expr = ExprId::from_usize(self.thir.exprs.len() - 1);
+        self.print_expr(expr, 1);
+    }
+
+    fn into_buffer(self) -> String {
+        self.fmt
+    }
+
+    fn print_param(&mut self, param: &Param<'tcx>, depth_lvl: usize) {
+        let Param { pat, ty, ty_span, self_kind, hir_id } = param;
+
+        print_indented!(self, "Param {", depth_lvl);
+        print_indented!(self, format!("ty: {:?}", ty), depth_lvl + 1);
+        print_indented!(self, format!("ty_span: {:?}", ty_span), depth_lvl + 1);
+        print_indented!(self, format!("self_kind: {:?}", self_kind), depth_lvl + 1);
+        print_indented!(self, format!("hir_id: {:?}", hir_id), depth_lvl + 1);
+
+        if let Some(pat) = pat {
+            print_indented!(self, "param: Some( ", depth_lvl + 1);
+            self.print_pat(pat, depth_lvl + 2);
+            print_indented!(self, ")", depth_lvl + 1);
+        } else {
+            print_indented!(self, "param: None", depth_lvl + 1);
+        }
+
+        print_indented!(self, "}", depth_lvl);
+    }
+
+    fn print_block(&mut self, block_id: BlockId, depth_lvl: usize) {
+        let Block {
+            targeted_by_break,
+            opt_destruction_scope,
+            span,
+            region_scope,
+            stmts,
+            expr,
+            safety_mode,
+        } = &self.thir.blocks[block_id];
+
+        print_indented!(self, "Block {", depth_lvl);
+        print_indented!(self, format!("targeted_by_break: {}", targeted_by_break), depth_lvl + 1);
+        print_indented!(
+            self,
+            format!("opt_destruction_scope: {:?}", opt_destruction_scope),
+            depth_lvl + 1
+        );
+        print_indented!(self, format!("span: {:?}", span), depth_lvl + 1);
+        print_indented!(self, format!("region_scope: {:?}", region_scope), depth_lvl + 1);
+        print_indented!(self, format!("safety_mode: {:?}", safety_mode), depth_lvl + 1);
+
+        if stmts.len() > 0 {
+            print_indented!(self, "stmts: [", depth_lvl + 1);
+            for stmt in stmts.iter() {
+                self.print_stmt(*stmt, depth_lvl + 2);
+            }
+            print_indented!(self, "]", depth_lvl + 1);
+        } else {
+            print_indented!(self, "stmts: []", depth_lvl + 1);
+        }
+
+        if let Some(expr_id) = expr {
+            print_indented!(self, "expr:", depth_lvl + 1);
+            self.print_expr(*expr_id, depth_lvl + 2);
+        } else {
+            print_indented!(self, "expr: []", depth_lvl + 1);
+        }
+
+        print_indented!(self, "}", depth_lvl);
+    }
+
+    fn print_stmt(&mut self, stmt_id: StmtId, depth_lvl: usize) {
+        let Stmt { kind, opt_destruction_scope } = &self.thir.stmts[stmt_id];
+
+        print_indented!(self, "Stmt {", depth_lvl);
+        print_indented!(
+            self,
+            format!("opt_destruction_scope: {:?}", opt_destruction_scope),
+            depth_lvl + 1
+        );
+
+        match kind {
+            StmtKind::Expr { scope, expr } => {
+                print_indented!(self, "kind: Expr {", depth_lvl + 1);
+                print_indented!(self, format!("scope: {:?}", scope), depth_lvl + 2);
+                print_indented!(self, "expr:", depth_lvl + 2);
+                self.print_expr(*expr, depth_lvl + 3);
+                print_indented!(self, "}", depth_lvl + 1);
+            }
+            StmtKind::Let {
+                remainder_scope,
+                init_scope,
+                pattern,
+                initializer,
+                else_block,
+                lint_level,
+            } => {
+                print_indented!(self, "kind: Let {", depth_lvl + 1);
+                print_indented!(
+                    self,
+                    format!("remainder_scope: {:?}", remainder_scope),
+                    depth_lvl + 2
+                );
+                print_indented!(self, format!("init_scope: {:?}", init_scope), depth_lvl + 2);
+
+                print_indented!(self, "pattern: ", depth_lvl + 2);
+                self.print_pat(pattern, depth_lvl + 3);
+                print_indented!(self, ",", depth_lvl + 2);
+
+                if let Some(init) = initializer {
+                    print_indented!(self, "initializer: Some(", depth_lvl + 2);
+                    self.print_expr(*init, depth_lvl + 3);
+                    print_indented!(self, ")", depth_lvl + 2);
+                } else {
+                    print_indented!(self, "initializer: None", depth_lvl + 2);
+                }
+
+                if let Some(else_block) = else_block {
+                    print_indented!(self, "else_block: Some(", depth_lvl + 2);
+                    self.print_block(*else_block, depth_lvl + 3);
+                    print_indented!(self, ")", depth_lvl + 2);
+                } else {
+                    print_indented!(self, "else_block: None", depth_lvl + 2);
+                }
+
+                print_indented!(self, format!("lint_level: {:?}", lint_level), depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl + 1);
+            }
+        }
+
+        print_indented!(self, "}", depth_lvl);
+    }
+
+    fn print_expr(&mut self, expr: ExprId, depth_lvl: usize) {
+        let Expr { ty, temp_lifetime, span, kind } = &self.thir[expr];
+        print_indented!(self, "Expr {", depth_lvl);
+        print_indented!(self, format!("ty: {:?}", ty), depth_lvl + 1);
+        print_indented!(self, format!("temp_lifetime: {:?}", temp_lifetime), depth_lvl + 1);
+        print_indented!(self, format!("span: {:?}", span), depth_lvl + 1);
+        print_indented!(self, "kind: ", depth_lvl + 1);
+        self.print_expr_kind(kind, depth_lvl + 2);
+        print_indented!(self, "}", depth_lvl);
+    }
+
+    fn print_expr_kind(&mut self, expr_kind: &ExprKind<'tcx>, depth_lvl: usize) {
+        use rustc_middle::thir::ExprKind::*;
+
+        match expr_kind {
+            Scope { region_scope, value, lint_level } => {
+                print_indented!(self, "Scope {", depth_lvl);
+                print_indented!(self, format!("region_scope: {:?}", region_scope), depth_lvl + 1);
+                print_indented!(self, format!("lint_level: {:?}", lint_level), depth_lvl + 1);
+                print_indented!(self, "value:", depth_lvl + 1);
+                self.print_expr(*value, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Box { value } => {
+                print_indented!(self, "Box {", depth_lvl);
+                self.print_expr(*value, depth_lvl + 1);
+                print_indented!(self, "}", depth_lvl);
+            }
+            If { if_then_scope, cond, then, else_opt } => {
+                print_indented!(self, "If {", depth_lvl);
+                print_indented!(self, format!("if_then_scope: {:?}", if_then_scope), depth_lvl + 1);
+                print_indented!(self, "cond:", depth_lvl + 1);
+                self.print_expr(*cond, depth_lvl + 2);
+                print_indented!(self, "then:", depth_lvl + 1);
+                self.print_expr(*then, depth_lvl + 2);
+
+                if let Some(else_expr) = else_opt {
+                    print_indented!(self, "else:", depth_lvl + 1);
+                    self.print_expr(*else_expr, depth_lvl + 2);
+                }
+
+                print_indented!(self, "}", depth_lvl);
+            }
+            Call { fun, args, ty, from_hir_call, fn_span } => {
+                print_indented!(self, "Call {", depth_lvl);
+                print_indented!(self, format!("ty: {:?}", ty), depth_lvl + 1);
+                print_indented!(self, format!("from_hir_call: {}", from_hir_call), depth_lvl + 1);
+                print_indented!(self, format!("fn_span: {:?}", fn_span), depth_lvl + 1);
+                print_indented!(self, "fun:", depth_lvl + 1);
+                self.print_expr(*fun, depth_lvl + 2);
+
+                if args.len() > 0 {
+                    print_indented!(self, "args: [", depth_lvl + 1);
+                    for arg in args.iter() {
+                        self.print_expr(*arg, depth_lvl + 2);
+                    }
+                    print_indented!(self, "]", depth_lvl + 1);
+                } else {
+                    print_indented!(self, "args: []", depth_lvl + 1);
+                }
+
+                print_indented!(self, "}", depth_lvl);
+            }
+            Deref { arg } => {
+                print_indented!(self, "Deref {", depth_lvl);
+                self.print_expr(*arg, depth_lvl + 1);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Binary { op, lhs, rhs } => {
+                print_indented!(self, "Binary {", depth_lvl);
+                print_indented!(self, format!("op: {:?}", op), depth_lvl + 1);
+                print_indented!(self, "lhs:", depth_lvl + 1);
+                self.print_expr(*lhs, depth_lvl + 2);
+                print_indented!(self, "rhs:", depth_lvl + 1);
+                self.print_expr(*rhs, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl);
+            }
+            LogicalOp { op, lhs, rhs } => {
+                print_indented!(self, "LogicalOp {", depth_lvl);
+                print_indented!(self, format!("op: {:?}", op), depth_lvl + 1);
+                print_indented!(self, "lhs:", depth_lvl + 1);
+                self.print_expr(*lhs, depth_lvl + 2);
+                print_indented!(self, "rhs:", depth_lvl + 1);
+                self.print_expr(*rhs, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Unary { op, arg } => {
+                print_indented!(self, "Unary {", depth_lvl);
+                print_indented!(self, format!("op: {:?}", op), depth_lvl + 1);
+                print_indented!(self, "arg:", depth_lvl + 1);
+                self.print_expr(*arg, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Cast { source } => {
+                print_indented!(self, "Cast {", depth_lvl);
+                print_indented!(self, "source:", depth_lvl + 1);
+                self.print_expr(*source, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Use { source } => {
+                print_indented!(self, "Use {", depth_lvl);
+                print_indented!(self, "source:", depth_lvl + 1);
+                self.print_expr(*source, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl);
+            }
+            NeverToAny { source } => {
+                print_indented!(self, "NeverToAny {", depth_lvl);
+                print_indented!(self, "source:", depth_lvl + 1);
+                self.print_expr(*source, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Pointer { cast, source } => {
+                print_indented!(self, "Pointer {", depth_lvl);
+                print_indented!(self, format!("cast: {:?}", cast), depth_lvl + 1);
+                print_indented!(self, "source:", depth_lvl + 1);
+                self.print_expr(*source, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Loop { body } => {
+                print_indented!(self, "Loop (", depth_lvl);
+                print_indented!(self, "body:", depth_lvl + 1);
+                self.print_expr(*body, depth_lvl + 2);
+                print_indented!(self, ")", depth_lvl);
+            }
+            Let { expr, pat } => {
+                print_indented!(self, "Let {", depth_lvl);
+                print_indented!(self, "expr:", depth_lvl + 1);
+                self.print_expr(*expr, depth_lvl + 2);
+                print_indented!(self, format!("pat: {:?}", pat), depth_lvl + 1);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Match { scrutinee, arms } => {
+                print_indented!(self, "Match {", depth_lvl);
+                print_indented!(self, "scrutinee:", depth_lvl + 1);
+                self.print_expr(*scrutinee, depth_lvl + 2);
+
+                print_indented!(self, "arms: [", depth_lvl + 1);
+                for arm_id in arms.iter() {
+                    self.print_arm(*arm_id, depth_lvl + 2);
+                }
+                print_indented!(self, "]", depth_lvl + 1);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Block { block } => self.print_block(*block, depth_lvl),
+            Assign { lhs, rhs } => {
+                print_indented!(self, "Assign {", depth_lvl);
+                print_indented!(self, "lhs:", depth_lvl + 1);
+                self.print_expr(*lhs, depth_lvl + 2);
+                print_indented!(self, "rhs:", depth_lvl + 1);
+                self.print_expr(*rhs, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl);
+            }
+            AssignOp { op, lhs, rhs } => {
+                print_indented!(self, "AssignOp {", depth_lvl);
+                print_indented!(self, format!("op: {:?}", op), depth_lvl + 1);
+                print_indented!(self, "lhs:", depth_lvl + 1);
+                self.print_expr(*lhs, depth_lvl + 2);
+                print_indented!(self, "rhs:", depth_lvl + 1);
+                self.print_expr(*rhs, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Field { lhs, variant_index, name } => {
+                print_indented!(self, "Field {", depth_lvl);
+                print_indented!(self, format!("variant_index: {:?}", variant_index), depth_lvl + 1);
+                print_indented!(self, format!("name: {:?}", name), depth_lvl + 1);
+                print_indented!(self, "lhs:", depth_lvl + 1);
+                self.print_expr(*lhs, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Index { lhs, index } => {
+                print_indented!(self, "Index {", depth_lvl);
+                print_indented!(self, format!("index: {:?}", index), depth_lvl + 1);
+                print_indented!(self, "lhs:", depth_lvl + 1);
+                self.print_expr(*lhs, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl);
+            }
+            VarRef { id } => {
+                print_indented!(self, "VarRef {", depth_lvl);
+                print_indented!(self, format!("id: {:?}", id), depth_lvl + 1);
+                print_indented!(self, "}", depth_lvl);
+            }
+            UpvarRef { closure_def_id, var_hir_id } => {
+                print_indented!(self, "UpvarRef {", depth_lvl);
+                print_indented!(
+                    self,
+                    format!("closure_def_id: {:?}", closure_def_id),
+                    depth_lvl + 1
+                );
+                print_indented!(self, format!("var_hir_id: {:?}", var_hir_id), depth_lvl + 1);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Borrow { borrow_kind, arg } => {
+                print_indented!(self, "Borrow (", depth_lvl);
+                print_indented!(self, format!("borrow_kind: {:?}", borrow_kind), depth_lvl + 1);
+                print_indented!(self, "arg:", depth_lvl + 1);
+                self.print_expr(*arg, depth_lvl + 2);
+                print_indented!(self, ")", depth_lvl);
+            }
+            AddressOf { mutability, arg } => {
+                print_indented!(self, "AddressOf {", depth_lvl);
+                print_indented!(self, format!("mutability: {:?}", mutability), depth_lvl + 1);
+                print_indented!(self, "arg:", depth_lvl + 1);
+                self.print_expr(*arg, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Break { label, value } => {
+                print_indented!(self, "Break (", depth_lvl);
+                print_indented!(self, format!("label: {:?}", label), depth_lvl + 1);
+
+                if let Some(value) = value {
+                    print_indented!(self, "value:", depth_lvl + 1);
+                    self.print_expr(*value, depth_lvl + 2);
+                }
+
+                print_indented!(self, ")", depth_lvl);
+            }
+            Continue { label } => {
+                print_indented!(self, "Continue {", depth_lvl);
+                print_indented!(self, format!("label: {:?}", label), depth_lvl + 1);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Return { value } => {
+                print_indented!(self, "Return {", depth_lvl);
+                print_indented!(self, "value:", depth_lvl + 1);
+
+                if let Some(value) = value {
+                    self.print_expr(*value, depth_lvl + 2);
+                }
+
+                print_indented!(self, "}", depth_lvl);
+            }
+            ConstBlock { did, substs } => {
+                print_indented!(self, "ConstBlock {", depth_lvl);
+                print_indented!(self, format!("did: {:?}", did), depth_lvl + 1);
+                print_indented!(self, format!("substs: {:?}", substs), depth_lvl + 1);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Repeat { value, count } => {
+                print_indented!(self, "Repeat {", depth_lvl);
+                print_indented!(self, format!("count: {:?}", count), depth_lvl + 1);
+                print_indented!(self, "value:", depth_lvl + 1);
+                self.print_expr(*value, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Array { fields } => {
+                print_indented!(self, "Array {", depth_lvl);
+                print_indented!(self, "fields: [", depth_lvl + 1);
+                for field_id in fields.iter() {
+                    self.print_expr(*field_id, depth_lvl + 2);
+                }
+                print_indented!(self, "]", depth_lvl + 1);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Tuple { fields } => {
+                print_indented!(self, "Tuple {", depth_lvl);
+                print_indented!(self, "fields: [", depth_lvl + 1);
+                for field_id in fields.iter() {
+                    self.print_expr(*field_id, depth_lvl + 2);
+                }
+                print_indented!(self, "]", depth_lvl + 1);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Adt(adt_expr) => {
+                print_indented!(self, "Adt {", depth_lvl);
+                self.print_adt_expr(&**adt_expr, depth_lvl + 1);
+                print_indented!(self, "}", depth_lvl);
+            }
+            PlaceTypeAscription { source, user_ty } => {
+                print_indented!(self, "PlaceTypeAscription {", depth_lvl);
+                print_indented!(self, format!("user_ty: {:?}", user_ty), depth_lvl + 1);
+                print_indented!(self, "source:", depth_lvl + 1);
+                self.print_expr(*source, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl);
+            }
+            ValueTypeAscription { source, user_ty } => {
+                print_indented!(self, "ValueTypeAscription {", depth_lvl);
+                print_indented!(self, format!("user_ty: {:?}", user_ty), depth_lvl + 1);
+                print_indented!(self, "source:", depth_lvl + 1);
+                self.print_expr(*source, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Closure(closure_expr) => {
+                print_indented!(self, "Closure {", depth_lvl);
+                print_indented!(self, "closure_expr:", depth_lvl + 1);
+                self.print_closure_expr(&**closure_expr, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Literal { lit, neg } => {
+                print_indented!(
+                    self,
+                    format!("Literal( lit: {:?}, neg: {:?})\n", lit, neg),
+                    depth_lvl
+                );
+            }
+            NonHirLiteral { lit, user_ty } => {
+                print_indented!(self, "NonHirLiteral {", depth_lvl);
+                print_indented!(self, format!("lit: {:?}", lit), depth_lvl + 1);
+                print_indented!(self, format!("user_ty: {:?}", user_ty), depth_lvl + 1);
+                print_indented!(self, "}", depth_lvl);
+            }
+            ZstLiteral { user_ty } => {
+                print_indented!(self, format!("ZstLiteral(user_ty: {:?})", user_ty), depth_lvl);
+            }
+            NamedConst { def_id, substs, user_ty } => {
+                print_indented!(self, "NamedConst {", depth_lvl);
+                print_indented!(self, format!("def_id: {:?}", def_id), depth_lvl + 1);
+                print_indented!(self, format!("user_ty: {:?}", user_ty), depth_lvl + 1);
+                print_indented!(self, format!("substs: {:?}", substs), depth_lvl + 1);
+                print_indented!(self, "}", depth_lvl);
+            }
+            ConstParam { param, def_id } => {
+                print_indented!(self, "ConstParam {", depth_lvl);
+                print_indented!(self, format!("def_id: {:?}", def_id), depth_lvl + 1);
+                print_indented!(self, format!("param: {:?}", param), depth_lvl + 1);
+                print_indented!(self, "}", depth_lvl);
+            }
+            StaticRef { alloc_id, ty, def_id } => {
+                print_indented!(self, "StaticRef {", depth_lvl);
+                print_indented!(self, format!("def_id: {:?}", def_id), depth_lvl + 1);
+                print_indented!(self, format!("ty: {:?}", ty), depth_lvl + 1);
+                print_indented!(self, format!("alloc_id: {:?}", alloc_id), depth_lvl + 1);
+                print_indented!(self, "}", depth_lvl);
+            }
+            InlineAsm(expr) => {
+                print_indented!(self, "InlineAsm {", depth_lvl);
+                print_indented!(self, "expr:", depth_lvl + 1);
+                self.print_inline_asm_expr(&**expr, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl);
+            }
+            ThreadLocalRef(def_id) => {
+                print_indented!(self, "ThreadLocalRef {", depth_lvl);
+                print_indented!(self, format!("def_id: {:?}", def_id), depth_lvl + 1);
+                print_indented!(self, "}", depth_lvl);
+            }
+            Yield { value } => {
+                print_indented!(self, "Yield {", depth_lvl);
+                print_indented!(self, "value:", depth_lvl + 1);
+                self.print_expr(*value, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl);
+            }
+        }
+    }
+
+    fn print_adt_expr(&mut self, adt_expr: &AdtExpr<'tcx>, depth_lvl: usize) {
+        print_indented!(self, "adt_def:", depth_lvl);
+        self.print_adt_def(adt_expr.adt_def, depth_lvl + 1);
+        print_indented!(
+            self,
+            format!("variant_index: {:?}", adt_expr.variant_index),
+            depth_lvl + 1
+        );
+        print_indented!(self, format!("substs: {:?}", adt_expr.substs), depth_lvl + 1);
+        print_indented!(self, format!("user_ty: {:?}", adt_expr.user_ty), depth_lvl + 1);
+
+        for (i, field_expr) in adt_expr.fields.iter().enumerate() {
+            print_indented!(self, format!("field {}:", i), depth_lvl + 1);
+            self.print_expr(field_expr.expr, depth_lvl + 2);
+        }
+
+        if let Some(ref base) = adt_expr.base {
+            print_indented!(self, "base:", depth_lvl + 1);
+            self.print_fru_info(base, depth_lvl + 2);
+        } else {
+            print_indented!(self, "base: None", depth_lvl + 1);
+        }
+    }
+
+    fn print_adt_def(&mut self, adt_def: ty::AdtDef<'tcx>, depth_lvl: usize) {
+        print_indented!(self, "AdtDef {", depth_lvl);
+        print_indented!(self, format!("did: {:?}", adt_def.did()), depth_lvl + 1);
+        print_indented!(self, format!("variants: {:?}", adt_def.variants()), depth_lvl + 1);
+        print_indented!(self, format!("flags: {:?}", adt_def.flags()), depth_lvl + 1);
+        print_indented!(self, format!("repr: {:?}", adt_def.repr()), depth_lvl + 1);
+    }
+
+    fn print_fru_info(&mut self, fru_info: &FruInfo<'tcx>, depth_lvl: usize) {
+        print_indented!(self, "FruInfo {", depth_lvl);
+        print_indented!(self, "base: ", depth_lvl + 1);
+        self.print_expr(fru_info.base, depth_lvl + 2);
+        print_indented!(self, "field_types: [", depth_lvl + 1);
+        for ty in fru_info.field_types.iter() {
+            print_indented!(self, format!("ty: {:?}", ty), depth_lvl + 2);
+        }
+        print_indented!(self, "}", depth_lvl);
+    }
+
+    fn print_arm(&mut self, arm_id: ArmId, depth_lvl: usize) {
+        print_indented!(self, "Arm {", depth_lvl);
+
+        let arm = &self.thir.arms[arm_id];
+        let Arm { pattern, guard, body, lint_level, scope, span } = arm;
+
+        print_indented!(self, "pattern: ", depth_lvl + 1);
+        self.print_pat(pattern, depth_lvl + 2);
+
+        if let Some(guard) = guard {
+            print_indented!(self, "guard: ", depth_lvl + 1);
+            self.print_guard(guard, depth_lvl + 2);
+        } else {
+            print_indented!(self, "guard: None", depth_lvl + 1);
+        }
+
+        print_indented!(self, "body: ", depth_lvl + 1);
+        self.print_expr(*body, depth_lvl + 2);
+        print_indented!(self, format!("lint_level: {:?}", lint_level), depth_lvl + 1);
+        print_indented!(self, format!("scope: {:?}", scope), depth_lvl + 1);
+        print_indented!(self, format!("span: {:?}", span), depth_lvl + 1);
+        print_indented!(self, "}", depth_lvl);
+    }
+
+    fn print_pat(&mut self, pat: &Box<Pat<'tcx>>, depth_lvl: usize) {
+        let Pat { ty, span, kind } = &**pat;
+
+        print_indented!(self, "Pat: {", depth_lvl);
+        print_indented!(self, format!("ty: {:?}", ty), depth_lvl + 1);
+        print_indented!(self, format!("span: {:?}", span), depth_lvl + 1);
+        self.print_pat_kind(kind, depth_lvl + 1);
+        print_indented!(self, "}", depth_lvl);
+    }
+
+    fn print_pat_kind(&mut self, pat_kind: &PatKind<'tcx>, depth_lvl: usize) {
+        print_indented!(self, "kind: PatKind {", depth_lvl);
+
+        match pat_kind {
+            PatKind::Wild => {
+                print_indented!(self, "Wild", depth_lvl + 1);
+            }
+            PatKind::AscribeUserType { ascription, subpattern } => {
+                print_indented!(self, "AscribeUserType: {", depth_lvl + 1);
+                print_indented!(self, format!("ascription: {:?}", ascription), depth_lvl + 2);
+                print_indented!(self, "subpattern: ", depth_lvl + 2);
+                self.print_pat(subpattern, depth_lvl + 3);
+                print_indented!(self, "}", depth_lvl + 1);
+            }
+            PatKind::Binding { mutability, name, mode, var, ty, subpattern, is_primary } => {
+                print_indented!(self, "Binding {", depth_lvl + 1);
+                print_indented!(self, format!("mutability: {:?}", mutability), depth_lvl + 2);
+                print_indented!(self, format!("name: {:?}", name), depth_lvl + 2);
+                print_indented!(self, format!("mode: {:?}", mode), depth_lvl + 2);
+                print_indented!(self, format!("var: {:?}", var), depth_lvl + 2);
+                print_indented!(self, format!("ty: {:?}", ty), depth_lvl + 2);
+                print_indented!(self, format!("is_primary: {:?}", is_primary), depth_lvl + 2);
+
+                if let Some(subpattern) = subpattern {
+                    print_indented!(self, "subpattern: Some( ", depth_lvl + 2);
+                    self.print_pat(subpattern, depth_lvl + 3);
+                    print_indented!(self, ")", depth_lvl + 2);
+                } else {
+                    print_indented!(self, "subpattern: None", depth_lvl + 2);
+                }
+
+                print_indented!(self, "}", depth_lvl + 1);
+            }
+            PatKind::Variant { adt_def, substs, variant_index, subpatterns } => {
+                print_indented!(self, "Variant {", depth_lvl + 1);
+                print_indented!(self, "adt_def: ", depth_lvl + 2);
+                self.print_adt_def(*adt_def, depth_lvl + 3);
+                print_indented!(self, format!("substs: {:?}", substs), depth_lvl + 2);
+                print_indented!(self, format!("variant_index: {:?}", variant_index), depth_lvl + 2);
+
+                if subpatterns.len() > 0 {
+                    print_indented!(self, "subpatterns: [", depth_lvl + 2);
+                    for field_pat in subpatterns.iter() {
+                        self.print_pat(&field_pat.pattern, depth_lvl + 3);
+                    }
+                    print_indented!(self, "]", depth_lvl + 2);
+                } else {
+                    print_indented!(self, "subpatterns: []", depth_lvl + 2);
+                }
+
+                print_indented!(self, "}", depth_lvl + 1);
+            }
+            PatKind::Leaf { subpatterns } => {
+                print_indented!(self, "Leaf { ", depth_lvl + 1);
+                print_indented!(self, "subpatterns: [", depth_lvl + 2);
+                for field_pat in subpatterns.iter() {
+                    self.print_pat(&field_pat.pattern, depth_lvl + 3);
+                }
+                print_indented!(self, "]", depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl + 1);
+            }
+            PatKind::Deref { subpattern } => {
+                print_indented!(self, "Deref { ", depth_lvl + 1);
+                print_indented!(self, "subpattern: ", depth_lvl + 2);
+                self.print_pat(subpattern, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl + 1);
+            }
+            PatKind::Constant { value } => {
+                print_indented!(self, "Constant {", depth_lvl + 1);
+                print_indented!(self, format!("value: {:?}", value), depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl + 1);
+            }
+            PatKind::Range(pat_range) => {
+                print_indented!(self, format!("Range ( {:?} )", pat_range), depth_lvl + 1);
+            }
+            PatKind::Slice { prefix, slice, suffix } => {
+                print_indented!(self, "Slice {", depth_lvl + 1);
+
+                print_indented!(self, "prefix: [", depth_lvl + 2);
+                for prefix_pat in prefix.iter() {
+                    self.print_pat(prefix_pat, depth_lvl + 3);
+                }
+                print_indented!(self, "]", depth_lvl + 2);
+
+                if let Some(slice) = slice {
+                    print_indented!(self, "slice: ", depth_lvl + 2);
+                    self.print_pat(slice, depth_lvl + 3);
+                }
+
+                print_indented!(self, "suffix: [", depth_lvl + 2);
+                for suffix_pat in suffix.iter() {
+                    self.print_pat(suffix_pat, depth_lvl + 3);
+                }
+                print_indented!(self, "]", depth_lvl + 2);
+
+                print_indented!(self, "}", depth_lvl + 1);
+            }
+            PatKind::Array { prefix, slice, suffix } => {
+                print_indented!(self, "Array {", depth_lvl + 1);
+
+                print_indented!(self, "prefix: [", depth_lvl + 2);
+                for prefix_pat in prefix.iter() {
+                    self.print_pat(prefix_pat, depth_lvl + 3);
+                }
+                print_indented!(self, "]", depth_lvl + 2);
+
+                if let Some(slice) = slice {
+                    print_indented!(self, "slice: ", depth_lvl + 2);
+                    self.print_pat(slice, depth_lvl + 3);
+                }
+
+                print_indented!(self, "suffix: [", depth_lvl + 2);
+                for suffix_pat in suffix.iter() {
+                    self.print_pat(suffix_pat, depth_lvl + 3);
+                }
+                print_indented!(self, "]", depth_lvl + 2);
+
+                print_indented!(self, "}", depth_lvl + 1);
+            }
+            PatKind::Or { pats } => {
+                print_indented!(self, "Or {", depth_lvl + 1);
+                print_indented!(self, "pats: [", depth_lvl + 2);
+                for pat in pats.iter() {
+                    self.print_pat(pat, depth_lvl + 3);
+                }
+                print_indented!(self, "]", depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl + 1);
+            }
+        }
+
+        print_indented!(self, "}", depth_lvl);
+    }
+
+    fn print_guard(&mut self, guard: &Guard<'tcx>, depth_lvl: usize) {
+        print_indented!(self, "Guard {", depth_lvl);
+
+        match guard {
+            Guard::If(expr_id) => {
+                print_indented!(self, "If (", depth_lvl + 1);
+                self.print_expr(*expr_id, depth_lvl + 2);
+                print_indented!(self, ")", depth_lvl + 1);
+            }
+            Guard::IfLet(pat, expr_id) => {
+                print_indented!(self, "IfLet (", depth_lvl + 1);
+                self.print_pat(pat, depth_lvl + 2);
+                print_indented!(self, ",", depth_lvl + 1);
+                self.print_expr(*expr_id, depth_lvl + 2);
+                print_indented!(self, ")", depth_lvl + 1);
+            }
+        }
+
+        print_indented!(self, "}", depth_lvl);
+    }
+
+    fn print_closure_expr(&mut self, expr: &ClosureExpr<'tcx>, depth_lvl: usize) {
+        let ClosureExpr { closure_id, substs, upvars, movability, fake_reads } = expr;
+
+        print_indented!(self, "ClosureExpr {", depth_lvl);
+        print_indented!(self, format!("closure_id: {:?}", closure_id), depth_lvl + 1);
+        print_indented!(self, format!("substs: {:?}", substs), depth_lvl + 1);
+
+        if upvars.len() > 0 {
+            print_indented!(self, "upvars: [", depth_lvl + 1);
+            for upvar in upvars.iter() {
+                self.print_expr(*upvar, depth_lvl + 2);
+                print_indented!(self, ",", depth_lvl + 1);
+            }
+            print_indented!(self, "]", depth_lvl + 1);
+        } else {
+            print_indented!(self, "upvars: []", depth_lvl + 1);
+        }
+
+        print_indented!(self, format!("movability: {:?}", movability), depth_lvl + 1);
+
+        if fake_reads.len() > 0 {
+            print_indented!(self, "fake_reads: [", depth_lvl + 1);
+            for (fake_read_expr, cause, hir_id) in fake_reads.iter() {
+                print_indented!(self, "(", depth_lvl + 2);
+                self.print_expr(*fake_read_expr, depth_lvl + 3);
+                print_indented!(self, ",", depth_lvl + 2);
+                print_indented!(self, format!("cause: {:?}", cause), depth_lvl + 3);
+                print_indented!(self, ",", depth_lvl + 2);
+                print_indented!(self, format!("hir_id: {:?}", hir_id), depth_lvl + 3);
+                print_indented!(self, "),", depth_lvl + 2);
+            }
+            print_indented!(self, "]", depth_lvl + 1);
+        } else {
+            print_indented!(self, "fake_reads: []", depth_lvl + 1);
+        }
+
+        print_indented!(self, "}", depth_lvl);
+    }
+
+    fn print_inline_asm_expr(&mut self, expr: &InlineAsmExpr<'tcx>, depth_lvl: usize) {
+        let InlineAsmExpr { template, operands, options, line_spans } = expr;
+
+        print_indented!(self, "InlineAsmExpr {", depth_lvl);
+
+        print_indented!(self, "template: [", depth_lvl + 1);
+        for template_piece in template.iter() {
+            print_indented!(self, format!("{:?}", template_piece), depth_lvl + 2);
+        }
+        print_indented!(self, "]", depth_lvl + 1);
+
+        print_indented!(self, "operands: [", depth_lvl + 1);
+        for operand in operands.iter() {
+            self.print_inline_operand(operand, depth_lvl + 2);
+        }
+        print_indented!(self, "]", depth_lvl + 1);
+
+        print_indented!(self, format!("options: {:?}", options), depth_lvl + 1);
+        print_indented!(self, format!("line_spans: {:?}", line_spans), depth_lvl + 1);
+    }
+
+    fn print_inline_operand(&mut self, operand: &InlineAsmOperand<'tcx>, depth_lvl: usize) {
+        match operand {
+            InlineAsmOperand::In { reg, expr } => {
+                print_indented!(self, "InlineAsmOperand::In {", depth_lvl);
+                print_indented!(self, format!("reg: {:?}", reg), depth_lvl + 1);
+                print_indented!(self, "expr: ", depth_lvl + 1);
+                self.print_expr(*expr, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl + 1);
+            }
+            InlineAsmOperand::Out { reg, late, expr } => {
+                print_indented!(self, "InlineAsmOperand::Out {", depth_lvl);
+                print_indented!(self, format!("reg: {:?}", reg), depth_lvl + 1);
+                print_indented!(self, format!("late: {:?}", late), depth_lvl + 1);
+
+                if let Some(out) = expr {
+                    print_indented!(self, "place: Some( ", depth_lvl + 1);
+                    self.print_expr(*out, depth_lvl + 2);
+                    print_indented!(self, ")", depth_lvl + 1);
+                } else {
+                    print_indented!(self, "place: None", depth_lvl + 1);
+                }
+                print_indented!(self, "}", depth_lvl + 1);
+            }
+            InlineAsmOperand::InOut { reg, late, expr } => {
+                print_indented!(self, "InlineAsmOperand::InOut {", depth_lvl);
+                print_indented!(self, format!("reg: {:?}", reg), depth_lvl + 1);
+                print_indented!(self, format!("late: {:?}", late), depth_lvl + 1);
+                print_indented!(self, "expr: ", depth_lvl + 1);
+                self.print_expr(*expr, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl + 1);
+            }
+            InlineAsmOperand::SplitInOut { reg, late, in_expr, out_expr } => {
+                print_indented!(self, "InlineAsmOperand::SplitInOut {", depth_lvl);
+                print_indented!(self, format!("reg: {:?}", reg), depth_lvl + 1);
+                print_indented!(self, format!("late: {:?}", late), depth_lvl + 1);
+                print_indented!(self, "in_expr: ", depth_lvl + 1);
+                self.print_expr(*in_expr, depth_lvl + 2);
+
+                if let Some(out_expr) = out_expr {
+                    print_indented!(self, "out_expr: Some( ", depth_lvl + 1);
+                    self.print_expr(*out_expr, depth_lvl + 2);
+                    print_indented!(self, ")", depth_lvl + 1);
+                } else {
+                    print_indented!(self, "out_expr: None", depth_lvl + 1);
+                }
+
+                print_indented!(self, "}", depth_lvl + 1);
+            }
+            InlineAsmOperand::Const { value, span } => {
+                print_indented!(self, "InlineAsmOperand::Const {", depth_lvl);
+                print_indented!(self, format!("value: {:?}", value), depth_lvl + 1);
+                print_indented!(self, format!("span: {:?}", span), depth_lvl + 1);
+                print_indented!(self, "}", depth_lvl + 1);
+            }
+            InlineAsmOperand::SymFn { value, span } => {
+                print_indented!(self, "InlineAsmOperand::SymFn {", depth_lvl);
+                print_indented!(self, format!("value: {:?}", *value), depth_lvl + 1);
+                print_indented!(self, format!("span: {:?}", span), depth_lvl + 1);
+                print_indented!(self, "}", depth_lvl + 1);
+            }
+            InlineAsmOperand::SymStatic { def_id } => {
+                print_indented!(self, "InlineAsmOperand::SymStatic {", depth_lvl);
+                print_indented!(self, format!("def_id: {:?}", def_id), depth_lvl + 1);
+                print_indented!(self, "}", depth_lvl + 1);
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_mir_dataflow/Cargo.toml b/compiler/rustc_mir_dataflow/Cargo.toml
index 324644b67..68c61a18d 100644
--- a/compiler/rustc_mir_dataflow/Cargo.toml
+++ b/compiler/rustc_mir_dataflow/Cargo.toml
@@ -19,6 +19,5 @@ rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_serialize = { path = "../rustc_serialize" }
-rustc_session = { path = "../rustc_session" }
 rustc_target = { path = "../rustc_target" }
 rustc_span = { path = "../rustc_span" }
diff --git a/compiler/rustc_error_messages/locales/en-US/mir_dataflow.ftl b/compiler/rustc_mir_dataflow/locales/en-US.ftl
index 988541525..988541525 100644
--- a/compiler/rustc_error_messages/locales/en-US/mir_dataflow.ftl
+++ b/compiler/rustc_mir_dataflow/locales/en-US.ftl
diff --git a/compiler/rustc_mir_dataflow/src/drop_flag_effects.rs b/compiler/rustc_mir_dataflow/src/drop_flag_effects.rs
index 3224e13f7..0d466bbe5 100644
--- a/compiler/rustc_mir_dataflow/src/drop_flag_effects.rs
+++ b/compiler/rustc_mir_dataflow/src/drop_flag_effects.rs
@@ -1,5 +1,5 @@
 use crate::elaborate_drops::DropFlagState;
-use rustc_middle::mir::{self, Body, Location};
+use rustc_middle::mir::{self, Body, Location, Terminator, TerminatorKind};
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_target::abi::VariantIdx;
 
@@ -194,6 +194,17 @@ pub fn drop_flag_effects_for_location<'tcx, F>(
         on_all_children_bits(tcx, body, move_data, path, |mpi| callback(mpi, DropFlagState::Absent))
     }
 
+    // Drop does not count as a move but we should still consider the variable uninitialized.
+    if let Some(Terminator { kind: TerminatorKind::Drop { place, .. }, .. }) =
+        body.stmt_at(loc).right()
+    {
+        if let LookupResult::Exact(mpi) = move_data.rev_lookup.find(place.as_ref()) {
+            on_all_children_bits(tcx, body, move_data, mpi, |mpi| {
+                callback(mpi, DropFlagState::Absent)
+            })
+        }
+    }
+
     debug!("drop_flag_effects: assignment for location({:?})", loc);
 
     for_location_inits(tcx, body, move_data, loc, |mpi| callback(mpi, DropFlagState::Present));
diff --git a/compiler/rustc_mir_dataflow/src/elaborate_drops.rs b/compiler/rustc_mir_dataflow/src/elaborate_drops.rs
index 7836ae2e7..bd1208762 100644
--- a/compiler/rustc_mir_dataflow/src/elaborate_drops.rs
+++ b/compiler/rustc_mir_dataflow/src/elaborate_drops.rs
@@ -890,7 +890,7 @@ where
             }
             ty::Dynamic(..) => self.complete_drop(self.succ, self.unwind),
             ty::Array(ety, size) => {
-                let size = size.try_eval_usize(self.tcx(), self.elaborator.param_env());
+                let size = size.try_eval_target_usize(self.tcx(), self.elaborator.param_env());
                 self.open_drop_for_array(*ety, size)
             }
             ty::Slice(ety) => self.open_drop_for_array(*ety, None),
diff --git a/compiler/rustc_mir_dataflow/src/framework/direction.rs b/compiler/rustc_mir_dataflow/src/framework/direction.rs
index 077a21fc8..2ae3ae02f 100644
--- a/compiler/rustc_mir_dataflow/src/framework/direction.rs
+++ b/compiler/rustc_mir_dataflow/src/framework/direction.rs
@@ -1,4 +1,3 @@
-use rustc_index::bit_set::BitSet;
 use rustc_middle::mir::{self, BasicBlock, Location, SwitchTargets};
 use rustc_middle::ty::TyCtxt;
 use std::ops::RangeInclusive;
@@ -54,7 +53,6 @@ pub trait Direction {
         analysis: &A,
         tcx: TyCtxt<'tcx>,
         body: &mir::Body<'tcx>,
-        dead_unwinds: Option<&BitSet<BasicBlock>>,
         exit_state: &mut A::Domain,
         block: (BasicBlock, &'_ mir::BasicBlockData<'tcx>),
         propagate: impl FnMut(BasicBlock, &A::Domain),
@@ -221,7 +219,6 @@ impl Direction for Backward {
         analysis: &A,
         _tcx: TyCtxt<'tcx>,
         body: &mir::Body<'tcx>,
-        dead_unwinds: Option<&BitSet<BasicBlock>>,
         exit_state: &mut A::Domain,
         (bb, _bb_data): (BasicBlock, &'_ mir::BasicBlockData<'tcx>),
         mut propagate: impl FnMut(BasicBlock, &A::Domain),
@@ -278,20 +275,6 @@ impl Direction for Backward {
                     }
                 }
 
-                // Ignore dead unwinds.
-                mir::TerminatorKind::Call { cleanup: Some(unwind), .. }
-                | mir::TerminatorKind::Assert { cleanup: Some(unwind), .. }
-                | mir::TerminatorKind::Drop { unwind: Some(unwind), .. }
-                | mir::TerminatorKind::DropAndReplace { unwind: Some(unwind), .. }
-                | mir::TerminatorKind::FalseUnwind { unwind: Some(unwind), .. }
-                | mir::TerminatorKind::InlineAsm { cleanup: Some(unwind), .. }
-                    if unwind == bb =>
-                {
-                    if dead_unwinds.map_or(true, |dead| !dead.contains(pred)) {
-                        propagate(pred, exit_state);
-                    }
-                }
-
                 _ => propagate(pred, exit_state),
             }
         }
@@ -304,7 +287,6 @@ struct BackwardSwitchIntEdgeEffectsApplier<'a, 'tcx, D, F> {
     exit_state: &'a mut D,
     bb: BasicBlock,
     propagate: &'a mut F,
-
     effects_applied: bool,
 }
 
@@ -484,7 +466,6 @@ impl Direction for Forward {
         analysis: &A,
         _tcx: TyCtxt<'tcx>,
         _body: &mir::Body<'tcx>,
-        dead_unwinds: Option<&BitSet<BasicBlock>>,
         exit_state: &mut A::Domain,
         (bb, bb_data): (BasicBlock, &'_ mir::BasicBlockData<'tcx>),
         mut propagate: impl FnMut(BasicBlock, &A::Domain),
@@ -502,9 +483,7 @@ impl Direction for Forward {
             | DropAndReplace { target, unwind, value: _, place: _ }
             | FalseUnwind { real_target: target, unwind } => {
                 if let Some(unwind) = unwind {
-                    if dead_unwinds.map_or(true, |dead| !dead.contains(bb)) {
-                        propagate(unwind, exit_state);
-                    }
+                    propagate(unwind, exit_state);
                 }
 
                 propagate(target, exit_state);
@@ -534,9 +513,7 @@ impl Direction for Forward {
                 fn_span: _,
             } => {
                 if let Some(unwind) = cleanup {
-                    if dead_unwinds.map_or(true, |dead| !dead.contains(bb)) {
-                        propagate(unwind, exit_state);
-                    }
+                    propagate(unwind, exit_state);
                 }
 
                 if let Some(target) = target {
@@ -560,9 +537,7 @@ impl Direction for Forward {
                 cleanup,
             } => {
                 if let Some(unwind) = cleanup {
-                    if dead_unwinds.map_or(true, |dead| !dead.contains(bb)) {
-                        propagate(unwind, exit_state);
-                    }
+                    propagate(unwind, exit_state);
                 }
 
                 if let Some(target) = destination {
diff --git a/compiler/rustc_mir_dataflow/src/framework/engine.rs b/compiler/rustc_mir_dataflow/src/framework/engine.rs
index 6ddbe69e1..91c3bf0ad 100644
--- a/compiler/rustc_mir_dataflow/src/framework/engine.rs
+++ b/compiler/rustc_mir_dataflow/src/framework/engine.rs
@@ -12,7 +12,6 @@ use rustc_ast as ast;
 use rustc_data_structures::work_queue::WorkQueue;
 use rustc_graphviz as dot;
 use rustc_hir::def_id::DefId;
-use rustc_index::bit_set::BitSet;
 use rustc_index::vec::{Idx, IndexVec};
 use rustc_middle::mir::{self, traversal, BasicBlock};
 use rustc_middle::mir::{create_dump_file, dump_enabled};
@@ -78,7 +77,6 @@ where
 {
     tcx: TyCtxt<'tcx>,
     body: &'a mir::Body<'tcx>,
-    dead_unwinds: Option<&'a BitSet<BasicBlock>>,
     entry_sets: IndexVec<BasicBlock, A::Domain>,
     pass_name: Option<&'static str>,
     analysis: A,
@@ -154,25 +152,7 @@ where
             bug!("`initialize_start_block` is not yet supported for backward dataflow analyses");
         }
 
-        Engine {
-            analysis,
-            tcx,
-            body,
-            dead_unwinds: None,
-            pass_name: None,
-            entry_sets,
-            apply_trans_for_block,
-        }
-    }
-
-    /// Signals that we do not want dataflow state to propagate across unwind edges for these
-    /// `BasicBlock`s.
-    ///
-    /// You must take care that `dead_unwinds` does not contain a `BasicBlock` that *can* actually
-    /// unwind during execution. Otherwise, your dataflow results will not be correct.
-    pub fn dead_unwinds(mut self, dead_unwinds: &'a BitSet<BasicBlock>) -> Self {
-        self.dead_unwinds = Some(dead_unwinds);
-        self
+        Engine { analysis, tcx, body, pass_name: None, entry_sets, apply_trans_for_block }
     }
 
     /// Adds an identifier to the graphviz output for this particular run of a dataflow analysis.
@@ -190,14 +170,7 @@ where
         A::Domain: DebugWithContext<A>,
     {
         let Engine {
-            analysis,
-            body,
-            dead_unwinds,
-            mut entry_sets,
-            tcx,
-            apply_trans_for_block,
-            pass_name,
-            ..
+            analysis, body, mut entry_sets, tcx, apply_trans_for_block, pass_name, ..
         } = self;
 
         let mut dirty_queue: WorkQueue<BasicBlock> = WorkQueue::with_none(body.basic_blocks.len());
@@ -236,7 +209,6 @@ where
                 &analysis,
                 tcx,
                 body,
-                dead_unwinds,
                 &mut state,
                 (bb, bb_data),
                 |target: BasicBlock, state: &A::Domain| {
diff --git a/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs b/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs
index 0f8e86d1d..6f4e7fd46 100644
--- a/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs
+++ b/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs
@@ -121,7 +121,9 @@ where
                 // for now. See discussion on [#61069].
                 //
                 // [#61069]: https://github.com/rust-lang/rust/pull/61069
-                self.trans.gen(dropped_place.local);
+                if !dropped_place.is_indirect() {
+                    self.trans.gen(dropped_place.local);
+                }
             }
 
             TerminatorKind::Abort
diff --git a/compiler/rustc_mir_dataflow/src/impls/liveness.rs b/compiler/rustc_mir_dataflow/src/impls/liveness.rs
index 923dc16c1..633a5674f 100644
--- a/compiler/rustc_mir_dataflow/src/impls/liveness.rs
+++ b/compiler/rustc_mir_dataflow/src/impls/liveness.rs
@@ -254,13 +254,7 @@ impl<'a, 'tcx> Analysis<'tcx> for MaybeTransitiveLiveLocals<'a> {
     ) {
         // Compute the place that we are storing to, if any
         let destination = match &statement.kind {
-            StatementKind::Assign(assign) => {
-                if assign.1.is_safe_to_remove() {
-                    Some(assign.0)
-                } else {
-                    None
-                }
-            }
+            StatementKind::Assign(assign) => assign.1.is_safe_to_remove().then_some(assign.0),
             StatementKind::SetDiscriminant { place, .. } | StatementKind::Deinit(place) => {
                 Some(**place)
             }
@@ -271,6 +265,7 @@ impl<'a, 'tcx> Analysis<'tcx> for MaybeTransitiveLiveLocals<'a> {
             | StatementKind::AscribeUserType(..)
             | StatementKind::Coverage(..)
             | StatementKind::Intrinsic(..)
+            | StatementKind::ConstEvalCounter
             | StatementKind::Nop => None,
         };
         if let Some(destination) = destination {
diff --git a/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs b/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs
index 8d379b90a..fcf0ce9d8 100644
--- a/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs
+++ b/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs
@@ -141,6 +141,7 @@ impl<'mir, 'tcx> crate::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'mir, 'tc
             StatementKind::AscribeUserType(..)
             | StatementKind::Coverage(..)
             | StatementKind::FakeRead(..)
+            | StatementKind::ConstEvalCounter
             | StatementKind::Nop
             | StatementKind::Retag(..)
             | StatementKind::Intrinsic(..)
diff --git a/compiler/rustc_mir_dataflow/src/lib.rs b/compiler/rustc_mir_dataflow/src/lib.rs
index 7f40cfca3..b1e03faff 100644
--- a/compiler/rustc_mir_dataflow/src/lib.rs
+++ b/compiler/rustc_mir_dataflow/src/lib.rs
@@ -1,6 +1,7 @@
 #![feature(associated_type_defaults)]
 #![feature(box_patterns)]
 #![feature(exact_size_is_empty)]
+#![feature(let_chains)]
 #![feature(min_specialization)]
 #![feature(once_cell)]
 #![feature(stmt_expr_attributes)]
@@ -15,7 +16,9 @@ extern crate tracing;
 extern crate rustc_middle;
 
 use rustc_ast::MetaItem;
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
 use rustc_hir::def_id::DefId;
+use rustc_macros::fluent_messages;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_span::symbol::{sym, Symbol};
 
@@ -43,6 +46,8 @@ pub mod storage;
 pub mod un_derefer;
 pub mod value_analysis;
 
+fluent_messages! { "../locales/en-US.ftl" }
+
 pub(crate) mod indexes {
     pub(crate) use super::move_paths::MovePathIndex;
 }
diff --git a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs
index f46fd118b..4a163028f 100644
--- a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs
+++ b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs
@@ -126,7 +126,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
                         BorrowedContent {
                             target_place: Place {
                                 local: place.local,
-                                projection: tcx.intern_place_elems(proj),
+                                projection: tcx.mk_place_elems(proj),
                             },
                         },
                     ));
@@ -165,7 +165,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
             if union_path.is_none() {
                 base = self.add_move_path(base, elem, |tcx| Place {
                     local: place.local,
-                    projection: tcx.intern_place_elems(&place.projection[..i + 1]),
+                    projection: tcx.mk_place_elems(&place.projection[..i + 1]),
                 });
             }
         }
@@ -331,6 +331,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
             | StatementKind::AscribeUserType(..)
             | StatementKind::Coverage(..)
             | StatementKind::Intrinsic(..)
+            | StatementKind::ConstEvalCounter
             | StatementKind::Nop => {}
         }
     }
@@ -375,7 +376,8 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
             | TerminatorKind::Resume
             | TerminatorKind::Abort
             | TerminatorKind::GeneratorDrop
-            | TerminatorKind::Unreachable => {}
+            | TerminatorKind::Unreachable
+            | TerminatorKind::Drop { .. } => {}
 
             TerminatorKind::Assert { ref cond, .. } => {
                 self.gather_operand(cond);
@@ -390,10 +392,6 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
                 self.create_move_path(place);
                 self.gather_init(place.as_ref(), InitKind::Deep);
             }
-
-            TerminatorKind::Drop { place, target: _, unwind: _ } => {
-                self.gather_move(place);
-            }
             TerminatorKind::DropAndReplace { place, ref value, .. } => {
                 self.create_move_path(place);
                 self.gather_operand(value);
@@ -478,7 +476,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
             // `ConstIndex` patterns. This is done to ensure that all move paths
             // are disjoint, which is expected by drop elaboration.
             let base_place =
-                Place { local: place.local, projection: self.builder.tcx.intern_place_elems(base) };
+                Place { local: place.local, projection: self.builder.tcx.mk_place_elems(base) };
             let base_path = match self.move_path_for(base_place) {
                 Ok(path) => path,
                 Err(MoveError::UnionMove { path }) => {
@@ -492,7 +490,9 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
             };
             let base_ty = base_place.ty(self.builder.body, self.builder.tcx).ty;
             let len: u64 = match base_ty.kind() {
-                ty::Array(_, size) => size.eval_usize(self.builder.tcx, self.builder.param_env),
+                ty::Array(_, size) => {
+                    size.eval_target_usize(self.builder.tcx, self.builder.param_env)
+                }
                 _ => bug!("from_end: false slice pattern of non-array type"),
             };
             for offset in from..to {
diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs
index 0522c6579..401db890a 100644
--- a/compiler/rustc_mir_dataflow/src/value_analysis.rs
+++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs
@@ -24,7 +24,7 @@
 //! - The bottom state denotes uninitialized memory. Because we are only doing a sound approximation
 //! of the actual execution, we can also use this state for places where access would be UB.
 //!
-//! - The assignment logic in `State::assign_place_idx` assumes that the places are non-overlapping,
+//! - The assignment logic in `State::insert_place_idx` assumes that the places are non-overlapping,
 //! or identical. Note that this refers to place expressions, not memory locations.
 //!
 //! - Currently, places that have their reference taken cannot be tracked. Although this would be
@@ -35,6 +35,7 @@
 use std::fmt::{Debug, Formatter};
 
 use rustc_data_structures::fx::FxHashMap;
+use rustc_index::bit_set::BitSet;
 use rustc_index::vec::IndexVec;
 use rustc_middle::mir::visit::{MutatingUseContext, PlaceContext, Visitor};
 use rustc_middle::mir::*;
@@ -64,10 +65,8 @@ pub trait ValueAnalysis<'tcx> {
             StatementKind::Assign(box (place, rvalue)) => {
                 self.handle_assign(*place, rvalue, state);
             }
-            StatementKind::SetDiscriminant { .. } => {
-                // Could treat this as writing a constant to a pseudo-place.
-                // But discriminants are currently not tracked, so we do nothing.
-                // Related: https://github.com/rust-lang/unsafe-code-guidelines/issues/84
+            StatementKind::SetDiscriminant { box ref place, .. } => {
+                state.flood_discr(place.as_ref(), self.map());
             }
             StatementKind::Intrinsic(box intrinsic) => {
                 self.handle_intrinsic(intrinsic, state);
@@ -84,7 +83,8 @@ pub trait ValueAnalysis<'tcx> {
             StatementKind::Retag(..) => {
                 // We don't track references.
             }
-            StatementKind::Nop
+            StatementKind::ConstEvalCounter
+            | StatementKind::Nop
             | StatementKind::FakeRead(..)
             | StatementKind::Coverage(..)
             | StatementKind::AscribeUserType(..) => (),
@@ -222,13 +222,13 @@ pub trait ValueAnalysis<'tcx> {
         self.super_terminator(terminator, state)
     }
 
-    fn super_terminator(&self, terminator: &Terminator<'tcx>, _state: &mut State<Self::Value>) {
+    fn super_terminator(&self, terminator: &Terminator<'tcx>, state: &mut State<Self::Value>) {
         match &terminator.kind {
             TerminatorKind::Call { .. } | TerminatorKind::InlineAsm { .. } => {
                 // Effect is applied by `handle_call_return`.
             }
-            TerminatorKind::Drop { .. } => {
-                // We don't track dropped places.
+            TerminatorKind::Drop { place, .. } => {
+                state.flood_with(place.as_ref(), self.map(), Self::Value::bottom());
             }
             TerminatorKind::DropAndReplace { .. } | TerminatorKind::Yield { .. } => {
                 // They would have an effect, but are not allowed in this phase.
@@ -445,26 +445,51 @@ impl<V: Clone + HasTop + HasBottom> State<V> {
     }
 
     pub fn flood_with(&mut self, place: PlaceRef<'_>, map: &Map, value: V) {
-        if let Some(root) = map.find(place) {
-            self.flood_idx_with(root, map, value);
-        }
+        let StateData::Reachable(values) = &mut self.0 else { return };
+        map.for_each_aliasing_place(place, None, &mut |place| {
+            if let Some(vi) = map.places[place].value_index {
+                values[vi] = value.clone();
+            }
+        });
     }
 
     pub fn flood(&mut self, place: PlaceRef<'_>, map: &Map) {
         self.flood_with(place, map, V::top())
     }
 
-    pub fn flood_idx_with(&mut self, place: PlaceIndex, map: &Map, value: V) {
+    pub fn flood_discr_with(&mut self, place: PlaceRef<'_>, map: &Map, value: V) {
         let StateData::Reachable(values) = &mut self.0 else { return };
-        map.preorder_invoke(place, &mut |place| {
+        map.for_each_aliasing_place(place, Some(TrackElem::Discriminant), &mut |place| {
             if let Some(vi) = map.places[place].value_index {
                 values[vi] = value.clone();
             }
         });
     }
 
-    pub fn flood_idx(&mut self, place: PlaceIndex, map: &Map) {
-        self.flood_idx_with(place, map, V::top())
+    pub fn flood_discr(&mut self, place: PlaceRef<'_>, map: &Map) {
+        self.flood_discr_with(place, map, V::top())
+    }
+
+    /// Low-level method that assigns to a place.
+    /// This does nothing if the place is not tracked.
+    ///
+    /// The target place must have been flooded before calling this method.
+    pub fn insert_idx(&mut self, target: PlaceIndex, result: ValueOrPlace<V>, map: &Map) {
+        match result {
+            ValueOrPlace::Value(value) => self.insert_value_idx(target, value, map),
+            ValueOrPlace::Place(source) => self.insert_place_idx(target, source, map),
+        }
+    }
+
+    /// Low-level method that assigns a value to a place.
+    /// This does nothing if the place is not tracked.
+    ///
+    /// The target place must have been flooded before calling this method.
+    pub fn insert_value_idx(&mut self, target: PlaceIndex, value: V, map: &Map) {
+        let StateData::Reachable(values) = &mut self.0 else { return };
+        if let Some(value_index) = map.places[target].value_index {
+            values[value_index] = value;
+        }
     }
 
     /// Copies `source` to `target`, including all tracked places beneath.
@@ -472,50 +497,41 @@ impl<V: Clone + HasTop + HasBottom> State<V> {
     /// If `target` contains a place that is not contained in `source`, it will be overwritten with
     /// Top. Also, because this will copy all entries one after another, it may only be used for
     /// places that are non-overlapping or identical.
-    pub fn assign_place_idx(&mut self, target: PlaceIndex, source: PlaceIndex, map: &Map) {
+    ///
+    /// The target place must have been flooded before calling this method.
+    fn insert_place_idx(&mut self, target: PlaceIndex, source: PlaceIndex, map: &Map) {
         let StateData::Reachable(values) = &mut self.0 else { return };
 
-        // If both places are tracked, we copy the value to the target. If the target is tracked,
-        // but the source is not, we have to invalidate the value in target. If the target is not
-        // tracked, then we don't have to do anything.
+        // If both places are tracked, we copy the value to the target.
+        // If the target is tracked, but the source is not, we do nothing, as invalidation has
+        // already been performed.
         if let Some(target_value) = map.places[target].value_index {
             if let Some(source_value) = map.places[source].value_index {
                 values[target_value] = values[source_value].clone();
-            } else {
-                values[target_value] = V::top();
             }
         }
         for target_child in map.children(target) {
             // Try to find corresponding child and recurse. Reasoning is similar as above.
             let projection = map.places[target_child].proj_elem.unwrap();
             if let Some(source_child) = map.projections.get(&(source, projection)) {
-                self.assign_place_idx(target_child, *source_child, map);
-            } else {
-                self.flood_idx(target_child, map);
+                self.insert_place_idx(target_child, *source_child, map);
             }
         }
     }
 
+    /// Helper method to interpret `target = result`.
     pub fn assign(&mut self, target: PlaceRef<'_>, result: ValueOrPlace<V>, map: &Map) {
+        self.flood(target, map);
         if let Some(target) = map.find(target) {
-            self.assign_idx(target, result, map);
-        } else {
-            // We don't track this place nor any projections, assignment can be ignored.
+            self.insert_idx(target, result, map);
         }
     }
 
-    pub fn assign_idx(&mut self, target: PlaceIndex, result: ValueOrPlace<V>, map: &Map) {
-        match result {
-            ValueOrPlace::Value(value) => {
-                // First flood the target place in case we also track any projections (although
-                // this scenario is currently not well-supported by the API).
-                self.flood_idx(target, map);
-                let StateData::Reachable(values) = &mut self.0 else { return };
-                if let Some(value_index) = map.places[target].value_index {
-                    values[value_index] = value;
-                }
-            }
-            ValueOrPlace::Place(source) => self.assign_place_idx(target, source, map),
+    /// Helper method for assignments to a discriminant.
+    pub fn assign_discr(&mut self, target: PlaceRef<'_>, result: ValueOrPlace<V>, map: &Map) {
+        self.flood_discr(target, map);
+        if let Some(target) = map.find_discr(target) {
+            self.insert_idx(target, result, map);
         }
     }
 
@@ -524,6 +540,14 @@ impl<V: Clone + HasTop + HasBottom> State<V> {
         map.find(place).map(|place| self.get_idx(place, map)).unwrap_or(V::top())
     }
 
+    /// Retrieve the value stored for a place, or ⊤ if it is not tracked.
+    pub fn get_discr(&self, place: PlaceRef<'_>, map: &Map) -> V {
+        match map.find_discr(place) {
+            Some(place) => self.get_idx(place, map),
+            None => V::top(),
+        }
+    }
+
     /// Retrieve the value stored for a place index, or ⊤ if it is not tracked.
     pub fn get_idx(&self, place: PlaceIndex, map: &Map) -> V {
         match &self.0 {
@@ -580,15 +604,15 @@ impl Map {
     /// This is currently the only way to create a [`Map`]. The way in which the tracked places are
     /// chosen is an implementation detail and may not be relied upon (other than that their type
     /// passes the filter).
-    #[instrument(skip_all, level = "debug")]
     pub fn from_filter<'tcx>(
         tcx: TyCtxt<'tcx>,
         body: &Body<'tcx>,
         filter: impl FnMut(Ty<'tcx>) -> bool,
+        place_limit: Option<usize>,
     ) -> Self {
         let mut map = Self::new();
         let exclude = excluded_locals(body);
-        map.register_with_filter(tcx, body, filter, &exclude);
+        map.register_with_filter(tcx, body, filter, exclude, place_limit);
         debug!("registered {} places ({} nodes in total)", map.value_count, map.places.len());
         map
     }
@@ -599,20 +623,28 @@ impl Map {
         tcx: TyCtxt<'tcx>,
         body: &Body<'tcx>,
         mut filter: impl FnMut(Ty<'tcx>) -> bool,
-        exclude: &IndexVec<Local, bool>,
+        exclude: BitSet<Local>,
+        place_limit: Option<usize>,
     ) {
         // We use this vector as stack, pushing and popping projections.
         let mut projection = Vec::new();
         for (local, decl) in body.local_decls.iter_enumerated() {
-            if !exclude[local] {
-                self.register_with_filter_rec(tcx, local, &mut projection, decl.ty, &mut filter);
+            if !exclude.contains(local) {
+                self.register_with_filter_rec(
+                    tcx,
+                    local,
+                    &mut projection,
+                    decl.ty,
+                    &mut filter,
+                    place_limit,
+                );
             }
         }
     }
 
     /// Potentially register the (local, projection) place and its fields, recursively.
     ///
-    /// Invariant: The projection must only contain fields.
+    /// Invariant: The projection must only contain trackable elements.
     fn register_with_filter_rec<'tcx>(
         &mut self,
         tcx: TyCtxt<'tcx>,
@@ -620,27 +652,56 @@ impl Map {
         projection: &mut Vec<PlaceElem<'tcx>>,
         ty: Ty<'tcx>,
         filter: &mut impl FnMut(Ty<'tcx>) -> bool,
+        place_limit: Option<usize>,
     ) {
-        // Note: The framework supports only scalars for now.
-        if filter(ty) && ty.is_scalar() {
-            // We know that the projection only contains trackable elements.
-            let place = self.make_place(local, projection).unwrap();
+        if let Some(place_limit) = place_limit && self.value_count >= place_limit {
+            return
+        }
 
-            // Allocate a value slot if it doesn't have one.
-            if self.places[place].value_index.is_none() {
-                self.places[place].value_index = Some(self.value_count.into());
-                self.value_count += 1;
+        // We know that the projection only contains trackable elements.
+        let place = self.make_place(local, projection).unwrap();
+
+        // Allocate a value slot if it doesn't have one, and the user requested one.
+        if self.places[place].value_index.is_none() && filter(ty) {
+            self.places[place].value_index = Some(self.value_count.into());
+            self.value_count += 1;
+        }
+
+        if ty.is_enum() {
+            let discr_ty = ty.discriminant_ty(tcx);
+            if filter(discr_ty) {
+                let discr = *self
+                    .projections
+                    .entry((place, TrackElem::Discriminant))
+                    .or_insert_with(|| {
+                        // Prepend new child to the linked list.
+                        let next = self.places.push(PlaceInfo::new(Some(TrackElem::Discriminant)));
+                        self.places[next].next_sibling = self.places[place].first_child;
+                        self.places[place].first_child = Some(next);
+                        next
+                    });
+
+                // Allocate a value slot if it doesn't have one.
+                if self.places[discr].value_index.is_none() {
+                    self.places[discr].value_index = Some(self.value_count.into());
+                    self.value_count += 1;
+                }
             }
         }
 
         // Recurse with all fields of this place.
         iter_fields(ty, tcx, |variant, field, ty| {
-            if variant.is_some() {
-                // Downcasts are currently not supported.
+            if let Some(variant) = variant {
+                projection.push(PlaceElem::Downcast(None, variant));
+                let _ = self.make_place(local, projection);
+                projection.push(PlaceElem::Field(field, ty));
+                self.register_with_filter_rec(tcx, local, projection, ty, filter, place_limit);
+                projection.pop();
+                projection.pop();
                 return;
             }
             projection.push(PlaceElem::Field(field, ty));
-            self.register_with_filter_rec(tcx, local, projection, ty, filter);
+            self.register_with_filter_rec(tcx, local, projection, ty, filter, place_limit);
             projection.pop();
         });
     }
@@ -683,23 +744,105 @@ impl Map {
     }
 
     /// Locates the given place, if it exists in the tree.
-    pub fn find(&self, place: PlaceRef<'_>) -> Option<PlaceIndex> {
+    pub fn find_extra(
+        &self,
+        place: PlaceRef<'_>,
+        extra: impl IntoIterator<Item = TrackElem>,
+    ) -> Option<PlaceIndex> {
         let mut index = *self.locals.get(place.local)?.as_ref()?;
 
         for &elem in place.projection {
             index = self.apply(index, elem.try_into().ok()?)?;
         }
+        for elem in extra {
+            index = self.apply(index, elem)?;
+        }
 
         Some(index)
     }
 
+    /// Locates the given place, if it exists in the tree.
+    pub fn find(&self, place: PlaceRef<'_>) -> Option<PlaceIndex> {
+        self.find_extra(place, [])
+    }
+
+    /// Locates the given place and applies `Discriminant`, if it exists in the tree.
+    pub fn find_discr(&self, place: PlaceRef<'_>) -> Option<PlaceIndex> {
+        self.find_extra(place, [TrackElem::Discriminant])
+    }
+
     /// Iterate over all direct children.
     pub fn children(&self, parent: PlaceIndex) -> impl Iterator<Item = PlaceIndex> + '_ {
         Children::new(self, parent)
     }
 
+    /// Invoke a function on the given place and all places that may alias it.
+    ///
+    /// In particular, when the given place has a variant downcast, we invoke the function on all
+    /// the other variants.
+    ///
+    /// `tail_elem` allows to support discriminants that are not a place in MIR, but that we track
+    /// as such.
+    pub fn for_each_aliasing_place(
+        &self,
+        place: PlaceRef<'_>,
+        tail_elem: Option<TrackElem>,
+        f: &mut impl FnMut(PlaceIndex),
+    ) {
+        if place.is_indirect() {
+            // We do not track indirect places.
+            return;
+        }
+        let Some(&Some(mut index)) = self.locals.get(place.local) else {
+            // The local is not tracked at all, so it does not alias anything.
+            return;
+        };
+        let elems = place
+            .projection
+            .iter()
+            .map(|&elem| elem.try_into())
+            .chain(tail_elem.map(Ok).into_iter());
+        for elem in elems {
+            // A field aliases the parent place.
+            f(index);
+
+            let Ok(elem) = elem else { return };
+            let sub = self.apply(index, elem);
+            if let TrackElem::Variant(..) | TrackElem::Discriminant = elem {
+                // Enum variant fields and enum discriminants alias each another.
+                self.for_each_variant_sibling(index, sub, f);
+            }
+            if let Some(sub) = sub {
+                index = sub
+            } else {
+                return;
+            }
+        }
+        self.preorder_invoke(index, f);
+    }
+
+    /// Invoke the given function on all the descendants of the given place, except one branch.
+    fn for_each_variant_sibling(
+        &self,
+        parent: PlaceIndex,
+        preserved_child: Option<PlaceIndex>,
+        f: &mut impl FnMut(PlaceIndex),
+    ) {
+        for sibling in self.children(parent) {
+            let elem = self.places[sibling].proj_elem;
+            // Only invalidate variants and discriminant. Fields (for generators) are not
+            // invalidated by assignment to a variant.
+            if let Some(TrackElem::Variant(..) | TrackElem::Discriminant) = elem
+                // Only invalidate the other variants, the current one is fine.
+                && Some(sibling) != preserved_child
+            {
+                self.preorder_invoke(sibling, f);
+            }
+        }
+    }
+
     /// Invoke a function on the given place and all descendants.
-    pub fn preorder_invoke(&self, root: PlaceIndex, f: &mut impl FnMut(PlaceIndex)) {
+    fn preorder_invoke(&self, root: PlaceIndex, f: &mut impl FnMut(PlaceIndex)) {
         f(root);
         for child in self.children(root) {
             self.preorder_invoke(child, f);
@@ -758,6 +901,7 @@ impl<'a> Iterator for Children<'a> {
 }
 
 /// Used as the result of an operand or r-value.
+#[derive(Debug)]
 pub enum ValueOrPlace<V> {
     Value(V),
     Place(PlaceIndex),
@@ -775,6 +919,8 @@ impl<V: HasTop> ValueOrPlace<V> {
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
 pub enum TrackElem {
     Field(Field),
+    Variant(VariantIdx),
+    Discriminant,
 }
 
 impl<V, T> TryFrom<ProjectionElem<V, T>> for TrackElem {
@@ -783,13 +929,14 @@ impl<V, T> TryFrom<ProjectionElem<V, T>> for TrackElem {
     fn try_from(value: ProjectionElem<V, T>) -> Result<Self, Self::Error> {
         match value {
             ProjectionElem::Field(field, _) => Ok(TrackElem::Field(field)),
+            ProjectionElem::Downcast(_, idx) => Ok(TrackElem::Variant(idx)),
             _ => Err(()),
         }
     }
 }
 
 /// Invokes `f` on all direct fields of `ty`.
-fn iter_fields<'tcx>(
+pub fn iter_fields<'tcx>(
     ty: Ty<'tcx>,
     tcx: TyCtxt<'tcx>,
     mut f: impl FnMut(Option<VariantIdx>, Field, Ty<'tcx>),
@@ -823,26 +970,27 @@ fn iter_fields<'tcx>(
 }
 
 /// Returns all locals with projections that have their reference or address taken.
-fn excluded_locals(body: &Body<'_>) -> IndexVec<Local, bool> {
+pub fn excluded_locals(body: &Body<'_>) -> BitSet<Local> {
     struct Collector {
-        result: IndexVec<Local, bool>,
+        result: BitSet<Local>,
     }
 
     impl<'tcx> Visitor<'tcx> for Collector {
         fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, _location: Location) {
-            if context.is_borrow()
+            if (context.is_borrow()
                 || context.is_address_of()
                 || context.is_drop()
-                || context == PlaceContext::MutatingUse(MutatingUseContext::AsmOutput)
+                || context == PlaceContext::MutatingUse(MutatingUseContext::AsmOutput))
+                && !place.is_indirect()
             {
                 // A pointer to a place could be used to access other places with the same local,
                 // hence we have to exclude the local completely.
-                self.result[place.local] = true;
+                self.result.insert(place.local);
             }
         }
     }
 
-    let mut collector = Collector { result: IndexVec::from_elem(false, &body.local_decls) };
+    let mut collector = Collector { result: BitSet::new_empty(body.local_decls.len()) };
     collector.visit_body(body);
     collector.result
 }
@@ -898,6 +1046,12 @@ fn debug_with_context_rec<V: Debug + Eq>(
     for child in map.children(place) {
         let info_elem = map.places[child].proj_elem.unwrap();
         let child_place_str = match info_elem {
+            TrackElem::Discriminant => {
+                format!("discriminant({})", place_str)
+            }
+            TrackElem::Variant(idx) => {
+                format!("({} as {:?})", place_str, idx)
+            }
             TrackElem::Field(field) => {
                 if place_str.starts_with('*') {
                     format!("({}).{}", place_str, field.index())
diff --git a/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs b/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs
index d8f85d2e3..9b4b72070 100644
--- a/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs
+++ b/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs
@@ -41,7 +41,7 @@ impl<'tcx> MirPass<'tcx> for AbortUnwindingCalls {
         //
         // Here we test for this function itself whether its ABI allows
         // unwinding or not.
-        let body_ty = tcx.type_of(def_id);
+        let body_ty = tcx.type_of(def_id).skip_binder();
         let body_abi = match body_ty.kind() {
             ty::FnDef(..) => body_ty.fn_sig(tcx).abi(),
             ty::Closure(..) => Abi::RustCall,
diff --git a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs
index fa5f392fa..536745d2c 100644
--- a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs
+++ b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs
@@ -74,7 +74,7 @@ impl<'tcx> ConstMutationChecker<'_, 'tcx> {
         //
         // `unsafe { *FOO = 0; *BAR.field = 1; }`
         // `unsafe { &mut *FOO }`
-        // `unsafe { (*ARRAY)[0] = val; }
+        // `unsafe { (*ARRAY)[0] = val; }`
         if !place.projection.iter().any(|p| matches!(p, PlaceElem::Deref)) {
             let source_info = self.body.source_info(location);
             let lint_root = self.body.source_scopes[source_info.scope]
diff --git a/compiler/rustc_mir_transform/src/check_packed_ref.rs b/compiler/rustc_mir_transform/src/check_packed_ref.rs
index 51abcf511..f5f1c1010 100644
--- a/compiler/rustc_mir_transform/src/check_packed_ref.rs
+++ b/compiler/rustc_mir_transform/src/check_packed_ref.rs
@@ -1,17 +1,11 @@
-use rustc_hir::def_id::LocalDefId;
+use rustc_errors::struct_span_err;
 use rustc_middle::mir::visit::{PlaceContext, Visitor};
 use rustc_middle::mir::*;
-use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, TyCtxt};
-use rustc_session::lint::builtin::UNALIGNED_REFERENCES;
 
 use crate::util;
 use crate::MirLint;
 
-pub(crate) fn provide(providers: &mut Providers) {
-    *providers = Providers { unsafe_derive_on_repr_packed, ..*providers };
-}
-
 pub struct CheckPackedRef;
 
 impl<'tcx> MirLint<'tcx> for CheckPackedRef {
@@ -30,32 +24,6 @@ struct PackedRefChecker<'a, 'tcx> {
     source_info: SourceInfo,
 }
 
-fn unsafe_derive_on_repr_packed(tcx: TyCtxt<'_>, def_id: LocalDefId) {
-    let lint_hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
-
-    // FIXME: when we make this a hard error, this should have its
-    // own error code.
-
-    let extra = if tcx.generics_of(def_id).own_requires_monomorphization() {
-        "with type or const parameters"
-    } else {
-        "that does not derive `Copy`"
-    };
-    let message = format!(
-        "`{}` can't be derived on this `#[repr(packed)]` struct {}",
-        tcx.item_name(tcx.trait_id_of_impl(def_id.to_def_id()).expect("derived trait name")),
-        extra
-    );
-
-    tcx.struct_span_lint_hir(
-        UNALIGNED_REFERENCES,
-        lint_hir_id,
-        tcx.def_span(def_id),
-        message,
-        |lint| lint,
-    );
-}
-
 impl<'tcx> Visitor<'tcx> for PackedRefChecker<'_, 'tcx> {
     fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
         // Make sure we know where in the MIR we are.
@@ -73,40 +41,30 @@ impl<'tcx> Visitor<'tcx> for PackedRefChecker<'_, 'tcx> {
         if context.is_borrow() {
             if util::is_disaligned(self.tcx, self.body, self.param_env, *place) {
                 let def_id = self.body.source.instance.def_id();
-                if let Some(impl_def_id) = self
-                    .tcx
-                    .impl_of_method(def_id)
-                    .filter(|&def_id| self.tcx.is_builtin_derive(def_id))
+                if let Some(impl_def_id) = self.tcx.impl_of_method(def_id)
+                    && self.tcx.is_builtin_derived(impl_def_id)
                 {
-                    // If a method is defined in the local crate,
-                    // the impl containing that method should also be.
-                    self.tcx.ensure().unsafe_derive_on_repr_packed(impl_def_id.expect_local());
+                    // If we ever reach here it means that the generated derive
+                    // code is somehow doing an unaligned reference, which it
+                    // shouldn't do.
+                    span_bug!(self.source_info.span, "builtin derive created an unaligned reference");
                 } else {
-                    let source_info = self.source_info;
-                    let lint_root = self.body.source_scopes[source_info.scope]
-                        .local_data
-                        .as_ref()
-                        .assert_crate_local()
-                        .lint_root;
-                    self.tcx.struct_span_lint_hir(
-                        UNALIGNED_REFERENCES,
-                        lint_root,
-                        source_info.span,
-                        "reference to packed field is unaligned",
-                        |lint| {
-                            lint
-                                .note(
-                                    "fields of packed structs are not properly aligned, and creating \
-                                    a misaligned reference is undefined behavior (even if that \
-                                    reference is never dereferenced)",
-                                )
-                                .help(
-                                    "copy the field contents to a local variable, or replace the \
-                                    reference with a raw pointer and use `read_unaligned`/`write_unaligned` \
-                                    (loads and stores via `*p` must be properly aligned even when using raw pointers)"
-                                )
-                        },
-                    );
+                    struct_span_err!(
+                        self.tcx.sess,
+                        self.source_info.span,
+                        E0793,
+                        "reference to packed field is unaligned"
+                    )
+                    .note(
+                        "fields of packed structs are not properly aligned, and creating \
+                        a misaligned reference is undefined behavior (even if that \
+                        reference is never dereferenced)",
+                    ).help(
+                        "copy the field contents to a local variable, or replace the \
+                        reference with a raw pointer and use `read_unaligned`/`write_unaligned` \
+                        (loads and stores via `*p` must be properly aligned even when using raw pointers)"
+                    )
+                    .emit();
                 }
             }
         }
diff --git a/compiler/rustc_mir_transform/src/check_unsafety.rs b/compiler/rustc_mir_transform/src/check_unsafety.rs
index adf6ae4c7..d00ee1f4b 100644
--- a/compiler/rustc_mir_transform/src/check_unsafety.rs
+++ b/compiler/rustc_mir_transform/src/check_unsafety.rs
@@ -104,6 +104,7 @@ impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> {
             | StatementKind::AscribeUserType(..)
             | StatementKind::Coverage(..)
             | StatementKind::Intrinsic(..)
+            | StatementKind::ConstEvalCounter
             | StatementKind::Nop => {
                 // safe (at least as emitted during MIR construction)
             }
@@ -125,6 +126,7 @@ impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> {
                     }
                 }
                 &AggregateKind::Closure(def_id, _) | &AggregateKind::Generator(def_id, _, _) => {
+                    let def_id = def_id.expect_local();
                     let UnsafetyCheckResult { violations, used_unsafe_blocks, .. } =
                         self.tcx.unsafety_check_result(def_id);
                     self.register_violations(violations, used_unsafe_blocks.iter().copied());
@@ -445,7 +447,7 @@ impl<'tcx> intravisit::Visitor<'tcx> for UnusedUnsafeVisitor<'_, 'tcx> {
         _fd: &'tcx hir::FnDecl<'tcx>,
         b: hir::BodyId,
         _s: rustc_span::Span,
-        _id: HirId,
+        _id: LocalDefId,
     ) {
         if matches!(fk, intravisit::FnKind::Closure) {
             self.visit_body(self.tcx.hir().body(b))
diff --git a/compiler/rustc_mir_transform/src/const_goto.rs b/compiler/rustc_mir_transform/src/const_goto.rs
index 40eefda4f..da101ca7a 100644
--- a/compiler/rustc_mir_transform/src/const_goto.rs
+++ b/compiler/rustc_mir_transform/src/const_goto.rs
@@ -57,6 +57,15 @@ impl<'tcx> MirPass<'tcx> for ConstGoto {
 }
 
 impl<'tcx> Visitor<'tcx> for ConstGotoOptimizationFinder<'_, 'tcx> {
+    fn visit_basic_block_data(&mut self, block: BasicBlock, data: &BasicBlockData<'tcx>) {
+        if data.is_cleanup {
+            // Because of the restrictions around control flow in cleanup blocks, we don't perform
+            // this optimization at all in such blocks.
+            return;
+        }
+        self.super_basic_block_data(block, data);
+    }
+
     fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
         let _: Option<_> = try {
             let target = terminator.kind.as_goto()?;
diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs
index 5c45abc5a..6b2eefce2 100644
--- a/compiler/rustc_mir_transform/src/const_prop.rs
+++ b/compiler/rustc_mir_transform/src/const_prop.rs
@@ -5,7 +5,6 @@ use std::cell::Cell;
 
 use either::Right;
 
-use rustc_ast::Mutability;
 use rustc_const_eval::const_eval::CheckAlignment;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_hir::def::DefKind;
@@ -14,14 +13,10 @@ use rustc_index::vec::IndexVec;
 use rustc_middle::mir::visit::{
     MutVisitor, MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor,
 };
-use rustc_middle::mir::{
-    BasicBlock, BinOp, Body, Constant, ConstantKind, Local, LocalDecl, LocalKind, Location,
-    Operand, Place, Rvalue, SourceInfo, Statement, StatementKind, Terminator, TerminatorKind, UnOp,
-    RETURN_PLACE,
-};
+use rustc_middle::mir::*;
 use rustc_middle::ty::layout::{LayoutError, LayoutOf, LayoutOfHelpers, TyAndLayout};
 use rustc_middle::ty::InternalSubsts;
-use rustc_middle::ty::{self, ConstKind, Instance, ParamEnv, Ty, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, ConstKind, Instance, ParamEnv, Ty, TyCtxt, TypeVisitableExt};
 use rustc_span::{def_id::DefId, Span};
 use rustc_target::abi::{self, Align, HasDataLayout, Size, TargetDataLayout};
 use rustc_target::spec::abi::Abi as CallAbi;
@@ -83,7 +78,7 @@ impl<'tcx> MirPass<'tcx> for ConstProp {
             return;
         }
 
-        let is_generator = tcx.type_of(def_id.to_def_id()).is_generator();
+        let is_generator = tcx.type_of(def_id.to_def_id()).subst_identity().is_generator();
         // FIXME(welseywiser) const prop doesn't work on generators because of query cycles
         // computing their layout.
         if is_generator {
@@ -289,7 +284,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx>
         }
         // If the static allocation is mutable, then we can't const prop it as its content
         // might be different at runtime.
-        if alloc.inner().mutability == Mutability::Mut {
+        if alloc.inner().mutability.is_mut() {
             throw_machine_stop_str!("can't access mutable globals in ConstProp");
         }
 
@@ -457,27 +452,6 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         };
     }
 
-    fn use_ecx<F, T>(&mut self, f: F) -> Option<T>
-    where
-        F: FnOnce(&mut Self) -> InterpResult<'tcx, T>,
-    {
-        match f(self) {
-            Ok(val) => Some(val),
-            Err(error) => {
-                trace!("InterpCx operation failed: {:?}", error);
-                // Some errors shouldn't come up because creating them causes
-                // an allocation, which we should avoid. When that happens,
-                // dedicated error variants should be introduced instead.
-                assert!(
-                    !error.kind().formatted_string(),
-                    "const-prop encountered formatting error: {}",
-                    error
-                );
-                None
-            }
-        }
-    }
-
     /// Returns the value, if any, of evaluating `c`.
     fn eval_constant(&mut self, c: &Constant<'tcx>) -> Option<OpTy<'tcx>> {
         // FIXME we need to revisit this for #67176
@@ -492,7 +466,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
     /// Returns the value, if any, of evaluating `place`.
     fn eval_place(&mut self, place: Place<'tcx>) -> Option<OpTy<'tcx>> {
         trace!("eval_place(place={:?})", place);
-        self.use_ecx(|this| this.ecx.eval_place_to_op(place, None))
+        self.ecx.eval_place_to_op(place, None).ok()
     }
 
     /// Returns the value, if any, of evaluating `op`. Calls upon `eval_constant`
@@ -504,55 +478,6 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         }
     }
 
-    fn check_unary_op(&mut self, op: UnOp, arg: &Operand<'tcx>) -> Option<()> {
-        if self.use_ecx(|this| {
-            let val = this.ecx.read_immediate(&this.ecx.eval_operand(arg, None)?)?;
-            let (_res, overflow, _ty) = this.ecx.overflowing_unary_op(op, &val)?;
-            Ok(overflow)
-        })? {
-            // `AssertKind` only has an `OverflowNeg` variant, so make sure that is
-            // appropriate to use.
-            assert_eq!(op, UnOp::Neg, "Neg is the only UnOp that can overflow");
-            return None;
-        }
-
-        Some(())
-    }
-
-    fn check_binary_op(
-        &mut self,
-        op: BinOp,
-        left: &Operand<'tcx>,
-        right: &Operand<'tcx>,
-    ) -> Option<()> {
-        let r = self.use_ecx(|this| this.ecx.read_immediate(&this.ecx.eval_operand(right, None)?));
-        let l = self.use_ecx(|this| this.ecx.read_immediate(&this.ecx.eval_operand(left, None)?));
-        // Check for exceeding shifts *even if* we cannot evaluate the LHS.
-        if op == BinOp::Shr || op == BinOp::Shl {
-            let r = r.clone()?;
-            // We need the type of the LHS. We cannot use `place_layout` as that is the type
-            // of the result, which for checked binops is not the same!
-            let left_ty = left.ty(self.local_decls, self.tcx);
-            let left_size = self.ecx.layout_of(left_ty).ok()?.size;
-            let right_size = r.layout.size;
-            let r_bits = r.to_scalar().to_bits(right_size).ok();
-            if r_bits.map_or(false, |b| b >= left_size.bits() as u128) {
-                return None;
-            }
-        }
-
-        if let (Some(l), Some(r)) = (&l, &r) {
-            // The remaining operators are handled through `overflowing_binary_op`.
-            if self.use_ecx(|this| {
-                let (_res, overflow, _ty) = this.ecx.overflowing_binary_op(op, l, r)?;
-                Ok(overflow)
-            })? {
-                return None;
-            }
-        }
-        Some(())
-    }
-
     fn propagate_operand(&mut self, operand: &mut Operand<'tcx>) {
         match *operand {
             Operand::Copy(l) | Operand::Move(l) => {
@@ -588,28 +513,6 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         //   2. Working around bugs in other parts of the compiler
         //        - In this case, we'll return `None` from this function to stop evaluation.
         match rvalue {
-            // Additional checking: give lints to the user if an overflow would occur.
-            // We do this here and not in the `Assert` terminator as that terminator is
-            // only sometimes emitted (overflow checks can be disabled), but we want to always
-            // lint.
-            Rvalue::UnaryOp(op, arg) => {
-                trace!("checking UnaryOp(op = {:?}, arg = {:?})", op, arg);
-                self.check_unary_op(*op, arg)?;
-            }
-            Rvalue::BinaryOp(op, box (left, right)) => {
-                trace!("checking BinaryOp(op = {:?}, left = {:?}, right = {:?})", op, left, right);
-                self.check_binary_op(*op, left, right)?;
-            }
-            Rvalue::CheckedBinaryOp(op, box (left, right)) => {
-                trace!(
-                    "checking CheckedBinaryOp(op = {:?}, left = {:?}, right = {:?})",
-                    op,
-                    left,
-                    right
-                );
-                self.check_binary_op(*op, left, right)?;
-            }
-
             // Do not try creating references (#67862)
             Rvalue::AddressOf(_, place) | Rvalue::Ref(_, _, place) => {
                 trace!("skipping AddressOf | Ref for {:?}", place);
@@ -639,7 +542,10 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
             | Rvalue::Cast(..)
             | Rvalue::ShallowInitBox(..)
             | Rvalue::Discriminant(..)
-            | Rvalue::NullaryOp(..) => {}
+            | Rvalue::NullaryOp(..)
+            | Rvalue::UnaryOp(..)
+            | Rvalue::BinaryOp(..)
+            | Rvalue::CheckedBinaryOp(..) => {}
         }
 
         // FIXME we need to revisit this for #67176
@@ -664,35 +570,37 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         rvalue: &Rvalue<'tcx>,
         place: Place<'tcx>,
     ) -> Option<()> {
-        self.use_ecx(|this| match rvalue {
+        match rvalue {
             Rvalue::BinaryOp(op, box (left, right))
             | Rvalue::CheckedBinaryOp(op, box (left, right)) => {
-                let l = this.ecx.eval_operand(left, None).and_then(|x| this.ecx.read_immediate(&x));
+                let l = self.ecx.eval_operand(left, None).and_then(|x| self.ecx.read_immediate(&x));
                 let r =
-                    this.ecx.eval_operand(right, None).and_then(|x| this.ecx.read_immediate(&x));
+                    self.ecx.eval_operand(right, None).and_then(|x| self.ecx.read_immediate(&x));
 
                 let const_arg = match (l, r) {
                     (Ok(x), Err(_)) | (Err(_), Ok(x)) => x, // exactly one side is known
-                    (Err(e), Err(_)) => return Err(e),      // neither side is known
-                    (Ok(_), Ok(_)) => return this.ecx.eval_rvalue_into_place(rvalue, place), // both sides are known
+                    (Err(_), Err(_)) => return None,        // neither side is known
+                    (Ok(_), Ok(_)) => return self.ecx.eval_rvalue_into_place(rvalue, place).ok(), // both sides are known
                 };
 
                 if !matches!(const_arg.layout.abi, abi::Abi::Scalar(..)) {
                     // We cannot handle Scalar Pair stuff.
                     // No point in calling `eval_rvalue_into_place`, since only one side is known
-                    throw_machine_stop_str!("cannot optimize this")
+                    return None;
                 }
 
-                let arg_value = const_arg.to_scalar().to_bits(const_arg.layout.size)?;
-                let dest = this.ecx.eval_place(place)?;
+                let arg_value = const_arg.to_scalar().to_bits(const_arg.layout.size).ok()?;
+                let dest = self.ecx.eval_place(place).ok()?;
 
                 match op {
-                    BinOp::BitAnd if arg_value == 0 => this.ecx.write_immediate(*const_arg, &dest),
+                    BinOp::BitAnd if arg_value == 0 => {
+                        self.ecx.write_immediate(*const_arg, &dest).ok()
+                    }
                     BinOp::BitOr
                         if arg_value == const_arg.layout.size.truncate(u128::MAX)
                             || (const_arg.layout.ty.is_bool() && arg_value == 1) =>
                     {
-                        this.ecx.write_immediate(*const_arg, &dest)
+                        self.ecx.write_immediate(*const_arg, &dest).ok()
                     }
                     BinOp::Mul if const_arg.layout.ty.is_integral() && arg_value == 0 => {
                         if let Rvalue::CheckedBinaryOp(_, _) = rvalue {
@@ -700,16 +608,16 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
                                 const_arg.to_scalar(),
                                 Scalar::from_bool(false),
                             );
-                            this.ecx.write_immediate(val, &dest)
+                            self.ecx.write_immediate(val, &dest).ok()
                         } else {
-                            this.ecx.write_immediate(*const_arg, &dest)
+                            self.ecx.write_immediate(*const_arg, &dest).ok()
                         }
                     }
-                    _ => throw_machine_stop_str!("cannot optimize this"),
+                    _ => None,
                 }
             }
-            _ => this.ecx.eval_rvalue_into_place(rvalue, place),
-        })
+            _ => self.ecx.eval_rvalue_into_place(rvalue, place).ok(),
+        }
     }
 
     /// Creates a new `Operand::Constant` from a `Scalar` value
@@ -751,7 +659,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         }
 
         // FIXME> figure out what to do when read_immediate_raw fails
-        let imm = self.use_ecx(|this| this.ecx.read_immediate_raw(value));
+        let imm = self.ecx.read_immediate_raw(value).ok();
 
         if let Some(Right(imm)) = imm {
             match *imm {
@@ -771,25 +679,23 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
                     if let ty::Tuple(types) = ty.kind() {
                         // Only do it if tuple is also a pair with two scalars
                         if let [ty1, ty2] = types[..] {
-                            let alloc = self.use_ecx(|this| {
-                                let ty_is_scalar = |ty| {
-                                    this.ecx.layout_of(ty).ok().map(|layout| layout.abi.is_scalar())
-                                        == Some(true)
-                                };
-                                if ty_is_scalar(ty1) && ty_is_scalar(ty2) {
-                                    let alloc = this
-                                        .ecx
-                                        .intern_with_temp_alloc(value.layout, |ecx, dest| {
-                                            ecx.write_immediate(*imm, dest)
-                                        })
-                                        .unwrap();
-                                    Ok(Some(alloc))
-                                } else {
-                                    Ok(None)
-                                }
-                            });
-
-                            if let Some(Some(alloc)) = alloc {
+                            let ty_is_scalar = |ty| {
+                                self.ecx.layout_of(ty).ok().map(|layout| layout.abi.is_scalar())
+                                    == Some(true)
+                            };
+                            let alloc = if ty_is_scalar(ty1) && ty_is_scalar(ty2) {
+                                let alloc = self
+                                    .ecx
+                                    .intern_with_temp_alloc(value.layout, |ecx, dest| {
+                                        ecx.write_immediate(*imm, dest)
+                                    })
+                                    .unwrap();
+                                Some(alloc)
+                            } else {
+                                None
+                            };
+
+                            if let Some(alloc) = alloc {
                                 // Assign entire constant in a single statement.
                                 // We can't use aggregates, as we run after the aggregate-lowering `MirPhase`.
                                 let const_val = ConstValue::ByRef { alloc, offset: Size::ZERO };
@@ -990,84 +896,80 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> {
         trace!("visit_statement: {:?}", statement);
         let source_info = statement.source_info;
         self.source_info = Some(source_info);
-        if let StatementKind::Assign(box (place, ref mut rval)) = statement.kind {
-            let can_const_prop = self.ecx.machine.can_const_prop[place.local];
-            if let Some(()) = self.const_prop(rval, place) {
-                // This will return None if the above `const_prop` invocation only "wrote" a
-                // type whose creation requires no write. E.g. a generator whose initial state
-                // consists solely of uninitialized memory (so it doesn't capture any locals).
-                if let Some(ref value) = self.get_const(place) && self.should_const_prop(value) {
-                    trace!("replacing {:?} with {:?}", rval, value);
-                    self.replace_with_const(rval, value, source_info);
-                    if can_const_prop == ConstPropMode::FullConstProp
-                        || can_const_prop == ConstPropMode::OnlyInsideOwnBlock
-                    {
-                        trace!("propagated into {:?}", place);
+        match statement.kind {
+            StatementKind::Assign(box (place, ref mut rval)) => {
+                let can_const_prop = self.ecx.machine.can_const_prop[place.local];
+                if let Some(()) = self.const_prop(rval, place) {
+                    // This will return None if the above `const_prop` invocation only "wrote" a
+                    // type whose creation requires no write. E.g. a generator whose initial state
+                    // consists solely of uninitialized memory (so it doesn't capture any locals).
+                    if let Some(ref value) = self.get_const(place) && self.should_const_prop(value) {
+                        trace!("replacing {:?} with {:?}", rval, value);
+                        self.replace_with_const(rval, value, source_info);
+                        if can_const_prop == ConstPropMode::FullConstProp
+                            || can_const_prop == ConstPropMode::OnlyInsideOwnBlock
+                        {
+                            trace!("propagated into {:?}", place);
+                        }
                     }
-                }
-                match can_const_prop {
-                    ConstPropMode::OnlyInsideOwnBlock => {
-                        trace!(
-                            "found local restricted to its block. \
+                    match can_const_prop {
+                        ConstPropMode::OnlyInsideOwnBlock => {
+                            trace!(
+                                "found local restricted to its block. \
                                 Will remove it from const-prop after block is finished. Local: {:?}",
-                            place.local
-                        );
-                    }
-                    ConstPropMode::OnlyPropagateInto | ConstPropMode::NoPropagation => {
-                        trace!("can't propagate into {:?}", place);
-                        if place.local != RETURN_PLACE {
-                            Self::remove_const(&mut self.ecx, place.local);
+                                place.local
+                            );
                         }
-                    }
-                    ConstPropMode::FullConstProp => {}
-                }
-            } else {
-                // Const prop failed, so erase the destination, ensuring that whatever happens
-                // from here on, does not know about the previous value.
-                // This is important in case we have
-                // ```rust
-                // let mut x = 42;
-                // x = SOME_MUTABLE_STATIC;
-                // // x must now be uninit
-                // ```
-                // FIXME: we overzealously erase the entire local, because that's easier to
-                // implement.
-                trace!(
-                    "propagation into {:?} failed.
-                        Nuking the entire site from orbit, it's the only way to be sure",
-                    place,
-                );
-                Self::remove_const(&mut self.ecx, place.local);
-            }
-        } else {
-            match statement.kind {
-                StatementKind::SetDiscriminant { ref place, .. } => {
-                    match self.ecx.machine.can_const_prop[place.local] {
-                        ConstPropMode::FullConstProp | ConstPropMode::OnlyInsideOwnBlock => {
-                            if self.use_ecx(|this| this.ecx.statement(statement)).is_some() {
-                                trace!("propped discriminant into {:?}", place);
-                            } else {
+                        ConstPropMode::OnlyPropagateInto | ConstPropMode::NoPropagation => {
+                            trace!("can't propagate into {:?}", place);
+                            if place.local != RETURN_PLACE {
                                 Self::remove_const(&mut self.ecx, place.local);
                             }
                         }
-                        ConstPropMode::OnlyPropagateInto | ConstPropMode::NoPropagation => {
-                            Self::remove_const(&mut self.ecx, place.local);
-                        }
+                        ConstPropMode::FullConstProp => {}
                     }
+                } else {
+                    // Const prop failed, so erase the destination, ensuring that whatever happens
+                    // from here on, does not know about the previous value.
+                    // This is important in case we have
+                    // ```rust
+                    // let mut x = 42;
+                    // x = SOME_MUTABLE_STATIC;
+                    // // x must now be uninit
+                    // ```
+                    // FIXME: we overzealously erase the entire local, because that's easier to
+                    // implement.
+                    trace!(
+                        "propagation into {:?} failed.
+                        Nuking the entire site from orbit, it's the only way to be sure",
+                        place,
+                    );
+                    Self::remove_const(&mut self.ecx, place.local);
                 }
-                StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => {
-                    let frame = self.ecx.frame_mut();
-                    frame.locals[local].value =
-                        if let StatementKind::StorageLive(_) = statement.kind {
-                            LocalValue::Live(interpret::Operand::Immediate(
-                                interpret::Immediate::Uninit,
-                            ))
+            }
+            StatementKind::SetDiscriminant { ref place, .. } => {
+                match self.ecx.machine.can_const_prop[place.local] {
+                    ConstPropMode::FullConstProp | ConstPropMode::OnlyInsideOwnBlock => {
+                        if self.ecx.statement(statement).is_ok() {
+                            trace!("propped discriminant into {:?}", place);
                         } else {
-                            LocalValue::Dead
-                        };
+                            Self::remove_const(&mut self.ecx, place.local);
+                        }
+                    }
+                    ConstPropMode::OnlyPropagateInto | ConstPropMode::NoPropagation => {
+                        Self::remove_const(&mut self.ecx, place.local);
+                    }
                 }
-                _ => {}
             }
+            StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => {
+                let frame = self.ecx.frame_mut();
+                frame.locals[local].value = if let StatementKind::StorageLive(_) = statement.kind {
+                    LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit))
+                } else {
+                    LocalValue::Dead
+                };
+            }
+            _ => {}
         }
 
         self.super_statement(statement, location);
@@ -1077,34 +979,19 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> {
         let source_info = terminator.source_info;
         self.source_info = Some(source_info);
         self.super_terminator(terminator, location);
-        // Do NOT early return in this function, it does some crucial fixup of the state at the end!
+
         match &mut terminator.kind {
             TerminatorKind::Assert { expected, ref mut cond, .. } => {
-                if let Some(ref value) = self.eval_operand(&cond) {
+                if let Some(ref value) = self.eval_operand(&cond)
+                    && let Ok(value_const) = self.ecx.read_scalar(&value)
+                    && self.should_const_prop(value)
+                {
                     trace!("assertion on {:?} should be {:?}", value, expected);
-                    let expected = Scalar::from_bool(*expected);
-                    // FIXME should be used use_ecx rather than a local match... but we have
-                    // quite a few of these read_scalar/read_immediate that need fixing.
-                    if let Ok(value_const) = self.ecx.read_scalar(&value) {
-                        if expected != value_const {
-                            // Poison all places this operand references so that further code
-                            // doesn't use the invalid value
-                            match cond {
-                                Operand::Move(ref place) | Operand::Copy(ref place) => {
-                                    Self::remove_const(&mut self.ecx, place.local);
-                                }
-                                Operand::Constant(_) => {}
-                            }
-                        } else {
-                            if self.should_const_prop(value) {
-                                *cond = self.operand_from_scalar(
-                                    value_const,
-                                    self.tcx.types.bool,
-                                    source_info.span,
-                                );
-                            }
-                        }
-                    }
+                    *cond = self.operand_from_scalar(
+                        value_const,
+                        self.tcx.types.bool,
+                        source_info.span,
+                    );
                 }
             }
             TerminatorKind::SwitchInt { ref mut discr, .. } => {
@@ -1132,6 +1019,10 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> {
             // gated on `mir_opt_level=3`.
             TerminatorKind::Call { .. } => {}
         }
+    }
+
+    fn visit_basic_block_data(&mut self, block: BasicBlock, data: &mut BasicBlockData<'tcx>) {
+        self.super_basic_block_data(block, data);
 
         // We remove all Locals which are restricted in propagation to their containing blocks and
         // which were modified in the current block.
diff --git a/compiler/rustc_mir_transform/src/const_prop_lint.rs b/compiler/rustc_mir_transform/src/const_prop_lint.rs
index 0ab67228f..fd9475748 100644
--- a/compiler/rustc_mir_transform/src/const_prop_lint.rs
+++ b/compiler/rustc_mir_transform/src/const_prop_lint.rs
@@ -21,7 +21,9 @@ use rustc_middle::mir::{
 };
 use rustc_middle::ty::layout::{LayoutError, LayoutOf, LayoutOfHelpers, TyAndLayout};
 use rustc_middle::ty::InternalSubsts;
-use rustc_middle::ty::{self, ConstInt, Instance, ParamEnv, ScalarInt, Ty, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{
+    self, ConstInt, Instance, ParamEnv, ScalarInt, Ty, TyCtxt, TypeVisitableExt,
+};
 use rustc_session::lint;
 use rustc_span::Span;
 use rustc_target::abi::{HasDataLayout, Size, TargetDataLayout};
@@ -57,7 +59,7 @@ impl<'tcx> MirLint<'tcx> for ConstProp {
             return;
         }
 
-        let is_generator = tcx.type_of(def_id.to_def_id()).is_generator();
+        let is_generator = tcx.type_of(def_id.to_def_id()).subst_identity().is_generator();
         // FIXME(welseywiser) const prop doesn't work on generators because of query cycles
         // computing their layout.
         if is_generator {
@@ -296,7 +298,15 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
             return None;
         }
 
-        self.use_ecx(source_info, |this| this.ecx.eval_mir_constant(&c.literal, Some(c.span), None))
+        // Normalization needed b/c const prop lint runs in
+        // `mir_drops_elaborated_and_const_checked`, which happens before
+        // optimized MIR. Only after optimizing the MIR can we guarantee
+        // that the `RevealAll` pass has happened and that the body's consts
+        // are normalized, so any call to resolve before that needs to be
+        // manually normalized.
+        let val = self.tcx.try_normalize_erasing_regions(self.param_env, c.literal).ok()?;
+
+        self.use_ecx(source_info, |this| this.ecx.eval_mir_constant(&val, Some(c.span), None))
     }
 
     /// Returns the value, if any, of evaluating `place`.
@@ -368,7 +378,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
             this.ecx.read_immediate(&this.ecx.eval_operand(left, None)?)
         });
         // Check for exceeding shifts *even if* we cannot evaluate the LHS.
-        if op == BinOp::Shr || op == BinOp::Shl {
+        if matches!(op, BinOp::Shr | BinOp::Shl) {
             let r = r.clone()?;
             // We need the type of the LHS. We cannot use `place_layout` as that is the type
             // of the result, which for checked binops is not the same!
diff --git a/compiler/rustc_mir_transform/src/copy_prop.rs b/compiler/rustc_mir_transform/src/copy_prop.rs
new file mode 100644
index 000000000..f27beb64a
--- /dev/null
+++ b/compiler/rustc_mir_transform/src/copy_prop.rs
@@ -0,0 +1,182 @@
+use rustc_index::bit_set::BitSet;
+use rustc_index::vec::IndexVec;
+use rustc_middle::mir::visit::*;
+use rustc_middle::mir::*;
+use rustc_middle::ty::TyCtxt;
+use rustc_mir_dataflow::impls::borrowed_locals;
+
+use crate::ssa::SsaLocals;
+use crate::MirPass;
+
+/// Unify locals that copy each other.
+///
+/// We consider patterns of the form
+///   _a = rvalue
+///   _b = move? _a
+///   _c = move? _a
+///   _d = move? _c
+/// where each of the locals is only assigned once.
+///
+/// We want to replace all those locals by `_a`, either copied or moved.
+pub struct CopyProp;
+
+impl<'tcx> MirPass<'tcx> for CopyProp {
+    fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
+        sess.mir_opt_level() >= 1
+    }
+
+    #[instrument(level = "trace", skip(self, tcx, body))]
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+        debug!(def_id = ?body.source.def_id());
+        propagate_ssa(tcx, body);
+    }
+}
+
+fn propagate_ssa<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id());
+    let borrowed_locals = borrowed_locals(body);
+    let ssa = SsaLocals::new(tcx, param_env, body, &borrowed_locals);
+
+    let fully_moved = fully_moved_locals(&ssa, body);
+    debug!(?fully_moved);
+
+    let mut storage_to_remove = BitSet::new_empty(fully_moved.domain_size());
+    for (local, &head) in ssa.copy_classes().iter_enumerated() {
+        if local != head {
+            storage_to_remove.insert(head);
+        }
+    }
+
+    let any_replacement = ssa.copy_classes().iter_enumerated().any(|(l, &h)| l != h);
+
+    Replacer {
+        tcx,
+        copy_classes: &ssa.copy_classes(),
+        fully_moved,
+        borrowed_locals,
+        storage_to_remove,
+    }
+    .visit_body_preserves_cfg(body);
+
+    if any_replacement {
+        crate::simplify::remove_unused_definitions(body);
+    }
+}
+
+/// `SsaLocals` computed equivalence classes between locals considering copy/move assignments.
+///
+/// This function also returns whether all the `move?` in the pattern are `move` and not copies.
+/// A local which is in the bitset can be replaced by `move _a`. Otherwise, it must be
+/// replaced by `copy _a`, as we cannot move multiple times from `_a`.
+///
+/// If an operand copies `_c`, it must happen before the assignment `_d = _c`, otherwise it is UB.
+/// This means that replacing it by a copy of `_a` if ok, since this copy happens before `_c` is
+/// moved, and therefore that `_d` is moved.
+#[instrument(level = "trace", skip(ssa, body))]
+fn fully_moved_locals(ssa: &SsaLocals, body: &Body<'_>) -> BitSet<Local> {
+    let mut fully_moved = BitSet::new_filled(body.local_decls.len());
+
+    for (_, rvalue) in ssa.assignments(body) {
+        let (Rvalue::Use(Operand::Copy(place) | Operand::Move(place)) | Rvalue::CopyForDeref(place))
+            = rvalue
+        else { continue };
+
+        let Some(rhs) = place.as_local() else { continue };
+        if !ssa.is_ssa(rhs) {
+            continue;
+        }
+
+        if let Rvalue::Use(Operand::Copy(_)) | Rvalue::CopyForDeref(_) = rvalue {
+            fully_moved.remove(rhs);
+        }
+    }
+
+    ssa.meet_copy_equivalence(&mut fully_moved);
+
+    fully_moved
+}
+
+/// Utility to help performing substitution of `*pattern` by `target`.
+struct Replacer<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    fully_moved: BitSet<Local>,
+    storage_to_remove: BitSet<Local>,
+    borrowed_locals: BitSet<Local>,
+    copy_classes: &'a IndexVec<Local, Local>,
+}
+
+impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> {
+    fn tcx(&self) -> TyCtxt<'tcx> {
+        self.tcx
+    }
+
+    fn visit_local(&mut self, local: &mut Local, ctxt: PlaceContext, _: Location) {
+        let new_local = self.copy_classes[*local];
+        match ctxt {
+            // Do not modify the local in storage statements.
+            PlaceContext::NonUse(NonUseContext::StorageLive | NonUseContext::StorageDead) => {}
+            // The local should have been marked as non-SSA.
+            PlaceContext::MutatingUse(_) => assert_eq!(*local, new_local),
+            // We access the value.
+            _ => *local = new_local,
+        }
+    }
+
+    fn visit_place(&mut self, place: &mut Place<'tcx>, ctxt: PlaceContext, loc: Location) {
+        if let Some(new_projection) = self.process_projection(&place.projection, loc) {
+            place.projection = self.tcx().mk_place_elems(&new_projection);
+        }
+
+        let observes_address = match ctxt {
+            PlaceContext::NonMutatingUse(
+                NonMutatingUseContext::SharedBorrow
+                | NonMutatingUseContext::ShallowBorrow
+                | NonMutatingUseContext::UniqueBorrow
+                | NonMutatingUseContext::AddressOf,
+            ) => true,
+            // For debuginfo, merging locals is ok.
+            PlaceContext::NonUse(NonUseContext::VarDebugInfo) => {
+                self.borrowed_locals.contains(place.local)
+            }
+            _ => false,
+        };
+        if observes_address && !place.is_indirect() {
+            // We observe the address of `place.local`. Do not replace it.
+        } else {
+            self.visit_local(
+                &mut place.local,
+                PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy),
+                loc,
+            )
+        }
+    }
+
+    fn visit_operand(&mut self, operand: &mut Operand<'tcx>, loc: Location) {
+        if let Operand::Move(place) = *operand
+            // A move out of a projection of a copy is equivalent to a copy of the original projection.
+            && !place.has_deref()
+            && !self.fully_moved.contains(place.local)
+        {
+            *operand = Operand::Copy(place);
+        }
+        self.super_operand(operand, loc);
+    }
+
+    fn visit_statement(&mut self, stmt: &mut Statement<'tcx>, loc: Location) {
+        match stmt.kind {
+            // When removing storage statements, we need to remove both (#107511).
+            StatementKind::StorageLive(l) | StatementKind::StorageDead(l)
+                if self.storage_to_remove.contains(l) =>
+            {
+                stmt.make_nop()
+            }
+            StatementKind::Assign(box (ref place, ref mut rvalue))
+                if place.as_local().is_some() =>
+            {
+                // Do not replace assignments.
+                self.visit_rvalue(rvalue, loc)
+            }
+            _ => self.super_statement(stmt, loc),
+        }
+    }
+}
diff --git a/compiler/rustc_mir_transform/src/coverage/counters.rs b/compiler/rustc_mir_transform/src/coverage/counters.rs
index 45de0c280..658e01d93 100644
--- a/compiler/rustc_mir_transform/src/coverage/counters.rs
+++ b/compiler/rustc_mir_transform/src/coverage/counters.rs
@@ -520,7 +520,7 @@ impl<'a> BcbCounters<'a> {
                 let mut found_loop_exit = false;
                 for &branch in branches.iter() {
                     if backedge_from_bcbs.iter().any(|&backedge_from_bcb| {
-                        self.bcb_is_dominated_by(backedge_from_bcb, branch.target_bcb)
+                        self.bcb_dominates(branch.target_bcb, backedge_from_bcb)
                     }) {
                         if let Some(reloop_branch) = some_reloop_branch {
                             if reloop_branch.counter(&self.basic_coverage_blocks).is_none() {
@@ -603,8 +603,8 @@ impl<'a> BcbCounters<'a> {
     }
 
     #[inline]
-    fn bcb_is_dominated_by(&self, node: BasicCoverageBlock, dom: BasicCoverageBlock) -> bool {
-        self.basic_coverage_blocks.is_dominated_by(node, dom)
+    fn bcb_dominates(&self, dom: BasicCoverageBlock, node: BasicCoverageBlock) -> bool {
+        self.basic_coverage_blocks.dominates(dom, node)
     }
 
     #[inline]
diff --git a/compiler/rustc_mir_transform/src/coverage/debug.rs b/compiler/rustc_mir_transform/src/coverage/debug.rs
index d6a298fad..22ea8710e 100644
--- a/compiler/rustc_mir_transform/src/coverage/debug.rs
+++ b/compiler/rustc_mir_transform/src/coverage/debug.rs
@@ -323,7 +323,10 @@ impl DebugCounters {
                         String::new()
                     },
                     self.format_operand(lhs),
-                    if op == Op::Add { "+" } else { "-" },
+                    match op {
+                        Op::Add => "+",
+                        Op::Subtract => "-",
+                    },
                     self.format_operand(rhs),
                 );
             }
diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs
index 78d28f1eb..a2671eef2 100644
--- a/compiler/rustc_mir_transform/src/coverage/graph.rs
+++ b/compiler/rustc_mir_transform/src/coverage/graph.rs
@@ -209,8 +209,8 @@ impl CoverageGraph {
     }
 
     #[inline(always)]
-    pub fn is_dominated_by(&self, node: BasicCoverageBlock, dom: BasicCoverageBlock) -> bool {
-        self.dominators.as_ref().unwrap().is_dominated_by(node, dom)
+    pub fn dominates(&self, dom: BasicCoverageBlock, node: BasicCoverageBlock) -> bool {
+        self.dominators.as_ref().unwrap().dominates(dom, node)
     }
 
     #[inline(always)]
@@ -312,7 +312,7 @@ rustc_index::newtype_index! {
 /// to the BCB's primary counter or expression).
 ///
 /// The BCB CFG is critical to simplifying the coverage analysis by ensuring graph path-based
-/// queries (`is_dominated_by()`, `predecessors`, `successors`, etc.) have branch (control flow)
+/// queries (`dominates()`, `predecessors`, `successors`, etc.) have branch (control flow)
 /// significance.
 #[derive(Debug, Clone)]
 pub(super) struct BasicCoverageBlockData {
@@ -594,7 +594,7 @@ impl TraverseCoverageGraphWithLoops {
                 // branching block would have given an `Expression` (or vice versa).
                 let (some_successor_to_add, some_loop_header) =
                     if let Some((_, loop_header)) = context.loop_backedges {
-                        if basic_coverage_blocks.is_dominated_by(successor, loop_header) {
+                        if basic_coverage_blocks.dominates(loop_header, successor) {
                             (Some(successor), Some(loop_header))
                         } else {
                             (None, None)
@@ -666,15 +666,15 @@ pub(super) fn find_loop_backedges(
     //
     // The overall complexity appears to be comparable to many other MIR transform algorithms, and I
     // don't expect that this function is creating a performance hot spot, but if this becomes an
-    // issue, there may be ways to optimize the `is_dominated_by` algorithm (as indicated by an
+    // issue, there may be ways to optimize the `dominates` algorithm (as indicated by an
     // existing `FIXME` comment in that code), or possibly ways to optimize it's usage here, perhaps
     // by keeping track of results for visited `BasicCoverageBlock`s if they can be used to short
-    // circuit downstream `is_dominated_by` checks.
+    // circuit downstream `dominates` checks.
     //
     // For now, that kind of optimization seems unnecessarily complicated.
     for (bcb, _) in basic_coverage_blocks.iter_enumerated() {
         for &successor in &basic_coverage_blocks.successors[bcb] {
-            if basic_coverage_blocks.is_dominated_by(bcb, successor) {
+            if basic_coverage_blocks.dominates(successor, bcb) {
                 let loop_header = successor;
                 let backedge_from_bcb = bcb;
                 debug!(
diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs
index 1468afc64..9a6171598 100644
--- a/compiler/rustc_mir_transform/src/coverage/mod.rs
+++ b/compiler/rustc_mir_transform/src/coverage/mod.rs
@@ -540,7 +540,8 @@ fn fn_sig_and_body(
     // FIXME(#79625): Consider improving MIR to provide the information needed, to avoid going back
     // to HIR for it.
     let hir_node = tcx.hir().get_if_local(def_id).expect("expected DefId is local");
-    let fn_body_id = hir::map::associated_body(hir_node).expect("HIR node is a function with body");
+    let (_, fn_body_id) =
+        hir::map::associated_body(hir_node).expect("HIR node is a function with body");
     (hir_node.fn_sig(), tcx.hir().body(fn_body_id))
 }
 
diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs
index c54348404..8ee316773 100644
--- a/compiler/rustc_mir_transform/src/coverage/spans.rs
+++ b/compiler/rustc_mir_transform/src/coverage/spans.rs
@@ -63,7 +63,7 @@ impl CoverageStatement {
 /// Note: A `CoverageStatement` merged into another CoverageSpan may come from a `BasicBlock` that
 /// is not part of the `CoverageSpan` bcb if the statement was included because it's `Span` matches
 /// or is subsumed by the `Span` associated with this `CoverageSpan`, and it's `BasicBlock`
-/// `is_dominated_by()` the `BasicBlock`s in this `CoverageSpan`.
+/// `dominates()` the `BasicBlock`s in this `CoverageSpan`.
 #[derive(Debug, Clone)]
 pub(super) struct CoverageSpan {
     pub span: Span,
@@ -407,7 +407,7 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> {
                 if self.prev().is_macro_expansion() && self.curr().is_macro_expansion() {
                     // Macros that expand to include branching (such as
                     // `assert_eq!()`, `assert_ne!()`, `info!()`, `debug!()`, or
-                    // `trace!()) typically generate callee spans with identical
+                    // `trace!()`) typically generate callee spans with identical
                     // ranges (typically the full span of the macro) for all
                     // `BasicBlocks`. This makes it impossible to distinguish
                     // the condition (`if val1 != val2`) from the optional
@@ -694,7 +694,7 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> {
     /// `prev.span.hi()` will be greater than (further right of) `prev_original_span.hi()`.
     /// If prev.span() was split off to the right of a closure, prev.span().lo() will be
     /// greater than prev_original_span.lo(). The actual span of `prev_original_span` is
-    /// not as important as knowing that `prev()` **used to have the same span** as `curr(),
+    /// not as important as knowing that `prev()` **used to have the same span** as `curr()`,
     /// which means their sort order is still meaningful for determining the dominator
     /// relationship.
     ///
@@ -705,12 +705,12 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> {
     fn hold_pending_dups_unless_dominated(&mut self) {
         // Equal coverage spans are ordered by dominators before dominated (if any), so it should be
         // impossible for `curr` to dominate any previous `CoverageSpan`.
-        debug_assert!(!self.span_bcb_is_dominated_by(self.prev(), self.curr()));
+        debug_assert!(!self.span_bcb_dominates(self.curr(), self.prev()));
 
         let initial_pending_count = self.pending_dups.len();
         if initial_pending_count > 0 {
             let mut pending_dups = self.pending_dups.split_off(0);
-            pending_dups.retain(|dup| !self.span_bcb_is_dominated_by(self.curr(), dup));
+            pending_dups.retain(|dup| !self.span_bcb_dominates(dup, self.curr()));
             self.pending_dups.append(&mut pending_dups);
             if self.pending_dups.len() < initial_pending_count {
                 debug!(
@@ -721,7 +721,7 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> {
             }
         }
 
-        if self.span_bcb_is_dominated_by(self.curr(), self.prev()) {
+        if self.span_bcb_dominates(self.prev(), self.curr()) {
             debug!(
                 "  different bcbs but SAME spans, and prev dominates curr. Discard prev={:?}",
                 self.prev()
@@ -787,8 +787,8 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> {
         }
     }
 
-    fn span_bcb_is_dominated_by(&self, covspan: &CoverageSpan, dom_covspan: &CoverageSpan) -> bool {
-        self.basic_coverage_blocks.is_dominated_by(covspan.bcb, dom_covspan.bcb)
+    fn span_bcb_dominates(&self, dom_covspan: &CoverageSpan, covspan: &CoverageSpan) -> bool {
+        self.basic_coverage_blocks.dominates(dom_covspan.bcb, covspan.bcb)
     }
 }
 
@@ -802,6 +802,8 @@ pub(super) fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span>
         | StatementKind::StorageDead(_)
         // Coverage should not be encountered, but don't inject coverage coverage
         | StatementKind::Coverage(_)
+        // Ignore `ConstEvalCounter`s
+        | StatementKind::ConstEvalCounter
         // Ignore `Nop`s
         | StatementKind::Nop => None,
 
diff --git a/compiler/rustc_mir_transform/src/ctfe_limit.rs b/compiler/rustc_mir_transform/src/ctfe_limit.rs
new file mode 100644
index 000000000..1b3ac78fb
--- /dev/null
+++ b/compiler/rustc_mir_transform/src/ctfe_limit.rs
@@ -0,0 +1,58 @@
+//! A pass that inserts the `ConstEvalCounter` instruction into any blocks that have a back edge
+//! (thus indicating there is a loop in the CFG), or whose terminator is a function call.
+use crate::MirPass;
+
+use rustc_data_structures::graph::dominators::Dominators;
+use rustc_middle::mir::{
+    BasicBlock, BasicBlockData, Body, Statement, StatementKind, TerminatorKind,
+};
+use rustc_middle::ty::TyCtxt;
+
+pub struct CtfeLimit;
+
+impl<'tcx> MirPass<'tcx> for CtfeLimit {
+    #[instrument(skip(self, _tcx, body))]
+    fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+        let doms = body.basic_blocks.dominators();
+        let indices: Vec<BasicBlock> = body
+            .basic_blocks
+            .iter_enumerated()
+            .filter_map(|(node, node_data)| {
+                if matches!(node_data.terminator().kind, TerminatorKind::Call { .. })
+                    // Back edges in a CFG indicate loops
+                    || has_back_edge(&doms, node, &node_data)
+                {
+                    Some(node)
+                } else {
+                    None
+                }
+            })
+            .collect();
+        for index in indices {
+            insert_counter(
+                body.basic_blocks_mut()
+                    .get_mut(index)
+                    .expect("basic_blocks index {index} should exist"),
+            );
+        }
+    }
+}
+
+fn has_back_edge(
+    doms: &Dominators<BasicBlock>,
+    node: BasicBlock,
+    node_data: &BasicBlockData<'_>,
+) -> bool {
+    if !doms.is_reachable(node) {
+        return false;
+    }
+    // Check if any of the dominators of the node are also the node's successor.
+    doms.dominators(node).any(|dom| node_data.terminator().successors().any(|succ| succ == dom))
+}
+
+fn insert_counter(basic_block_data: &mut BasicBlockData<'_>) {
+    basic_block_data.statements.push(Statement {
+        source_info: basic_block_data.terminator().source_info,
+        kind: StatementKind::ConstEvalCounter,
+    });
+}
diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
index c75fe2327..49ded10ba 100644
--- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
+++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
@@ -5,6 +5,7 @@
 use rustc_const_eval::const_eval::CheckAlignment;
 use rustc_const_eval::interpret::{ConstValue, ImmTy, Immediate, InterpCx, Scalar};
 use rustc_data_structures::fx::FxHashMap;
+use rustc_hir::def::DefKind;
 use rustc_middle::mir::visit::{MutVisitor, Visitor};
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, Ty, TyCtxt};
@@ -12,6 +13,7 @@ use rustc_mir_dataflow::value_analysis::{Map, State, TrackElem, ValueAnalysis, V
 use rustc_mir_dataflow::{lattice::FlatSet, Analysis, ResultsVisitor, SwitchIntEdgeEffects};
 use rustc_span::DUMMY_SP;
 use rustc_target::abi::Align;
+use rustc_target::abi::VariantIdx;
 
 use crate::MirPass;
 
@@ -29,14 +31,12 @@ impl<'tcx> MirPass<'tcx> for DataflowConstProp {
 
     #[instrument(skip_all level = "debug")]
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+        debug!(def_id = ?body.source.def_id());
         if tcx.sess.mir_opt_level() < 4 && body.basic_blocks.len() > BLOCK_LIMIT {
             debug!("aborted dataflow const prop due too many basic blocks");
             return;
         }
 
-        // Decide which places to track during the analysis.
-        let map = Map::from_filter(tcx, body, Ty::is_scalar);
-
         // We want to have a somewhat linear runtime w.r.t. the number of statements/terminators.
         // Let's call this number `n`. Dataflow analysis has `O(h*n)` transfer function
         // applications, where `h` is the height of the lattice. Because the height of our lattice
@@ -45,10 +45,10 @@ impl<'tcx> MirPass<'tcx> for DataflowConstProp {
         // `O(num_nodes * tracked_places * n)` in terms of time complexity. Since the number of
         // map nodes is strongly correlated to the number of tracked places, this becomes more or
         // less `O(n)` if we place a constant limit on the number of tracked places.
-        if tcx.sess.mir_opt_level() < 4 && map.tracked_places() > PLACE_LIMIT {
-            debug!("aborted dataflow const prop due to too many tracked places");
-            return;
-        }
+        let place_limit = if tcx.sess.mir_opt_level() < 4 { Some(PLACE_LIMIT) } else { None };
+
+        // Decide which places to track during the analysis.
+        let map = Map::from_filter(tcx, body, Ty::is_scalar, place_limit);
 
         // Perform the actual dataflow analysis.
         let analysis = ConstAnalysis::new(tcx, body, map);
@@ -62,14 +62,31 @@ impl<'tcx> MirPass<'tcx> for DataflowConstProp {
     }
 }
 
-struct ConstAnalysis<'tcx> {
+struct ConstAnalysis<'a, 'tcx> {
     map: Map,
     tcx: TyCtxt<'tcx>,
+    local_decls: &'a LocalDecls<'tcx>,
     ecx: InterpCx<'tcx, 'tcx, DummyMachine>,
     param_env: ty::ParamEnv<'tcx>,
 }
 
-impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'tcx> {
+impl<'tcx> ConstAnalysis<'_, 'tcx> {
+    fn eval_discriminant(
+        &self,
+        enum_ty: Ty<'tcx>,
+        variant_index: VariantIdx,
+    ) -> Option<ScalarTy<'tcx>> {
+        if !enum_ty.is_enum() {
+            return None;
+        }
+        let discr = enum_ty.discriminant_for_variant(self.tcx, variant_index)?;
+        let discr_layout = self.tcx.layout_of(self.param_env.and(discr.ty)).ok()?;
+        let discr_value = Scalar::try_from_uint(discr.val, discr_layout.size)?;
+        Some(ScalarTy(discr_value, discr.ty))
+    }
+}
+
+impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> {
     type Value = FlatSet<ScalarTy<'tcx>>;
 
     const NAME: &'static str = "ConstAnalysis";
@@ -78,6 +95,25 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'tcx> {
         &self.map
     }
 
+    fn handle_statement(&self, statement: &Statement<'tcx>, state: &mut State<Self::Value>) {
+        match statement.kind {
+            StatementKind::SetDiscriminant { box ref place, variant_index } => {
+                state.flood_discr(place.as_ref(), &self.map);
+                if self.map.find_discr(place.as_ref()).is_some() {
+                    let enum_ty = place.ty(self.local_decls, self.tcx).ty;
+                    if let Some(discr) = self.eval_discriminant(enum_ty, variant_index) {
+                        state.assign_discr(
+                            place.as_ref(),
+                            ValueOrPlace::Value(FlatSet::Elem(discr)),
+                            &self.map,
+                        );
+                    }
+                }
+            }
+            _ => self.super_statement(statement, state),
+        }
+    }
+
     fn handle_assign(
         &self,
         target: Place<'tcx>,
@@ -85,13 +121,59 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'tcx> {
         state: &mut State<Self::Value>,
     ) {
         match rvalue {
+            Rvalue::Aggregate(kind, operands) => {
+                // If we assign `target = Enum::Variant#0(operand)`,
+                // we must make sure that all `target as Variant#i` are `Top`.
+                state.flood(target.as_ref(), self.map());
+
+                if let Some(target_idx) = self.map().find(target.as_ref()) {
+                    let (variant_target, variant_index) = match **kind {
+                        AggregateKind::Tuple | AggregateKind::Closure(..) => {
+                            (Some(target_idx), None)
+                        }
+                        AggregateKind::Adt(def_id, variant_index, ..) => {
+                            match self.tcx.def_kind(def_id) {
+                                DefKind::Struct => (Some(target_idx), None),
+                                DefKind::Enum => (
+                                    self.map.apply(target_idx, TrackElem::Variant(variant_index)),
+                                    Some(variant_index),
+                                ),
+                                _ => (None, None),
+                            }
+                        }
+                        _ => (None, None),
+                    };
+                    if let Some(variant_target_idx) = variant_target {
+                        for (field_index, operand) in operands.iter().enumerate() {
+                            if let Some(field) = self.map().apply(
+                                variant_target_idx,
+                                TrackElem::Field(Field::from_usize(field_index)),
+                            ) {
+                                let result = self.handle_operand(operand, state);
+                                state.insert_idx(field, result, self.map());
+                            }
+                        }
+                    }
+                    if let Some(variant_index) = variant_index
+                        && let Some(discr_idx) = self.map().apply(target_idx, TrackElem::Discriminant)
+                    {
+                        // We are assigning the discriminant as part of an aggregate.
+                        // This discriminant can only alias a variant field's value if the operand
+                        // had an invalid value for that type.
+                        // Using invalid values is UB, so we are allowed to perform the assignment
+                        // without extra flooding.
+                        let enum_ty = target.ty(self.local_decls, self.tcx).ty;
+                        if let Some(discr_val) = self.eval_discriminant(enum_ty, variant_index) {
+                            state.insert_value_idx(discr_idx, FlatSet::Elem(discr_val), &self.map);
+                        }
+                    }
+                }
+            }
             Rvalue::CheckedBinaryOp(op, box (left, right)) => {
+                // Flood everything now, so we can use `insert_value_idx` directly later.
+                state.flood(target.as_ref(), self.map());
+
                 let target = self.map().find(target.as_ref());
-                if let Some(target) = target {
-                    // We should not track any projections other than
-                    // what is overwritten below, but just in case...
-                    state.flood_idx(target, self.map());
-                }
 
                 let value_target = target
                     .and_then(|target| self.map().apply(target, TrackElem::Field(0_u32.into())));
@@ -102,26 +184,19 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'tcx> {
                     let (val, overflow) = self.binary_op(state, *op, left, right);
 
                     if let Some(value_target) = value_target {
-                        state.assign_idx(value_target, ValueOrPlace::Value(val), self.map());
+                        // We have flooded `target` earlier.
+                        state.insert_value_idx(value_target, val, self.map());
                     }
                     if let Some(overflow_target) = overflow_target {
                         let overflow = match overflow {
                             FlatSet::Top => FlatSet::Top,
                             FlatSet::Elem(overflow) => {
-                                if overflow {
-                                    // Overflow cannot be reliably propagated. See: https://github.com/rust-lang/rust/pull/101168#issuecomment-1288091446
-                                    FlatSet::Top
-                                } else {
-                                    self.wrap_scalar(Scalar::from_bool(false), self.tcx.types.bool)
-                                }
+                                self.wrap_scalar(Scalar::from_bool(overflow), self.tcx.types.bool)
                             }
                             FlatSet::Bottom => FlatSet::Bottom,
                         };
-                        state.assign_idx(
-                            overflow_target,
-                            ValueOrPlace::Value(overflow),
-                            self.map(),
-                        );
+                        // We have flooded `target` earlier.
+                        state.insert_value_idx(overflow_target, overflow, self.map());
                     }
                 }
             }
@@ -170,6 +245,9 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'tcx> {
                 FlatSet::Bottom => ValueOrPlace::Value(FlatSet::Bottom),
                 FlatSet::Top => ValueOrPlace::Value(FlatSet::Top),
             },
+            Rvalue::Discriminant(place) => {
+                ValueOrPlace::Value(state.get_discr(place.as_ref(), self.map()))
+            }
             _ => self.super_rvalue(rvalue, state),
         }
     }
@@ -243,12 +321,13 @@ impl<'tcx> std::fmt::Debug for ScalarTy<'tcx> {
     }
 }
 
-impl<'tcx> ConstAnalysis<'tcx> {
-    pub fn new(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, map: Map) -> Self {
+impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> {
+    pub fn new(tcx: TyCtxt<'tcx>, body: &'a Body<'tcx>, map: Map) -> Self {
         let param_env = tcx.param_env(body.source.def_id());
         Self {
             map,
             tcx,
+            local_decls: &body.local_decls,
             ecx: InterpCx::new(tcx, DUMMY_SP, param_env, DummyMachine),
             param_env: param_env,
         }
@@ -441,6 +520,21 @@ impl<'tcx, 'map, 'a> Visitor<'tcx> for OperandCollector<'tcx, 'map, 'a> {
             _ => (),
         }
     }
+
+    fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, location: Location) {
+        match rvalue {
+            Rvalue::Discriminant(place) => {
+                match self.state.get_discr(place.as_ref(), self.visitor.map) {
+                    FlatSet::Top => (),
+                    FlatSet::Elem(value) => {
+                        self.visitor.before_effect.insert((location, *place), value);
+                    }
+                    FlatSet::Bottom => (),
+                }
+            }
+            _ => self.super_rvalue(rvalue, location),
+        }
+    }
 }
 
 struct DummyMachine;
diff --git a/compiler/rustc_mir_transform/src/dead_store_elimination.rs b/compiler/rustc_mir_transform/src/dead_store_elimination.rs
index 09546330c..9dbfb089d 100644
--- a/compiler/rustc_mir_transform/src/dead_store_elimination.rs
+++ b/compiler/rustc_mir_transform/src/dead_store_elimination.rs
@@ -53,6 +53,7 @@ pub fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, borrowed: &BitS
                 | StatementKind::StorageDead(_)
                 | StatementKind::Coverage(_)
                 | StatementKind::Intrinsic(_)
+                | StatementKind::ConstEvalCounter
                 | StatementKind::Nop => (),
 
                 StatementKind::FakeRead(_) | StatementKind::AscribeUserType(_, _) => {
diff --git a/compiler/rustc_mir_transform/src/deaggregator.rs b/compiler/rustc_mir_transform/src/deaggregator.rs
deleted file mode 100644
index fe272de20..000000000
--- a/compiler/rustc_mir_transform/src/deaggregator.rs
+++ /dev/null
@@ -1,45 +0,0 @@
-use crate::util::expand_aggregate;
-use crate::MirPass;
-use rustc_middle::mir::*;
-use rustc_middle::ty::TyCtxt;
-
-pub struct Deaggregator;
-
-impl<'tcx> MirPass<'tcx> for Deaggregator {
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        let basic_blocks = body.basic_blocks.as_mut_preserves_cfg();
-        for bb in basic_blocks {
-            bb.expand_statements(|stmt| {
-                // FIXME(eddyb) don't match twice on `stmt.kind` (post-NLL).
-                match stmt.kind {
-                    // FIXME(#48193) Deaggregate arrays when it's cheaper to do so.
-                    StatementKind::Assign(box (
-                        _,
-                        Rvalue::Aggregate(box AggregateKind::Array(_), _),
-                    )) => {
-                        return None;
-                    }
-                    StatementKind::Assign(box (_, Rvalue::Aggregate(_, _))) => {}
-                    _ => return None,
-                }
-
-                let stmt = stmt.replace_nop();
-                let source_info = stmt.source_info;
-                let StatementKind::Assign(box (lhs, Rvalue::Aggregate(kind, operands))) = stmt.kind else {
-                    bug!();
-                };
-
-                Some(expand_aggregate(
-                    lhs,
-                    operands.into_iter().map(|op| {
-                        let ty = op.ty(&body.local_decls, tcx);
-                        (op, ty)
-                    }),
-                    *kind,
-                    source_info,
-                    tcx,
-                ))
-            });
-        }
-    }
-}
diff --git a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
index ddab7bbb2..89ca04a15 100644
--- a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
+++ b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
@@ -163,7 +163,7 @@ pub fn deduced_param_attrs<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> &'tcx [Ded
 
     // Codegen won't use this information for anything if all the function parameters are passed
     // directly. Detect that and bail, for compilation speed.
-    let fn_ty = tcx.type_of(def_id);
+    let fn_ty = tcx.type_of(def_id).subst_identity();
     if matches!(fn_ty.kind(), ty::FnDef(..)) {
         if fn_ty
             .fn_sig(tcx)
diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs
index 08e296a83..2e481b972 100644
--- a/compiler/rustc_mir_transform/src/dest_prop.rs
+++ b/compiler/rustc_mir_transform/src/dest_prop.rs
@@ -136,8 +136,8 @@ use rustc_index::bit_set::BitSet;
 use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor};
 use rustc_middle::mir::{dump_mir, PassWhere};
 use rustc_middle::mir::{
-    traversal, BasicBlock, Body, InlineAsmOperand, Local, LocalKind, Location, Operand, Place,
-    Rvalue, Statement, StatementKind, TerminatorKind,
+    traversal, Body, InlineAsmOperand, Local, LocalKind, Location, Operand, Place, Rvalue,
+    Statement, StatementKind, TerminatorKind,
 };
 use rustc_middle::ty::TyCtxt;
 use rustc_mir_dataflow::impls::MaybeLiveLocals;
@@ -328,7 +328,8 @@ impl<'a, 'tcx> MutVisitor<'tcx> for Merger<'a, 'tcx> {
         match &statement.kind {
             StatementKind::Assign(box (dest, rvalue)) => {
                 match rvalue {
-                    Rvalue::Use(Operand::Copy(place) | Operand::Move(place)) => {
+                    Rvalue::CopyForDeref(place)
+                    | Rvalue::Use(Operand::Copy(place) | Operand::Move(place)) => {
                         // These might've been turned into self-assignments by the replacement
                         // (this includes the original statement we wanted to eliminate).
                         if dest == place {
@@ -467,7 +468,7 @@ impl<'a, 'body, 'alloc, 'tcx> FilterInformation<'a, 'body, 'alloc, 'tcx> {
             // to reuse the allocation.
             write_info: write_info_alloc,
             // Doesn't matter what we put here, will be overwritten before being used
-            at: Location { block: BasicBlock::from_u32(0), statement_index: 0 },
+            at: Location::START,
         };
         this.internal_filter_liveness();
     }
@@ -577,6 +578,7 @@ impl WriteInfo {
                 self.add_place(**place);
             }
             StatementKind::Intrinsic(_)
+            | StatementKind::ConstEvalCounter
             | StatementKind::Nop
             | StatementKind::Coverage(_)
             | StatementKind::StorageLive(_)
@@ -754,7 +756,7 @@ impl<'tcx> Visitor<'tcx> for FindAssignments<'_, '_, 'tcx> {
     fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) {
         if let StatementKind::Assign(box (
             lhs,
-            Rvalue::Use(Operand::Copy(rhs) | Operand::Move(rhs)),
+            Rvalue::CopyForDeref(rhs) | Rvalue::Use(Operand::Copy(rhs) | Operand::Move(rhs)),
         )) = &statement.kind
         {
             let Some((src, dest)) = places_to_candidate_pair(*lhs, *rhs, self.body) else {
diff --git a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs
index 932134bd6..954bb5aff 100644
--- a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs
+++ b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs
@@ -17,9 +17,9 @@ pub fn build_ptr_tys<'tcx>(
     unique_did: DefId,
     nonnull_did: DefId,
 ) -> (Ty<'tcx>, Ty<'tcx>, Ty<'tcx>) {
-    let substs = tcx.intern_substs(&[pointee.into()]);
-    let unique_ty = tcx.bound_type_of(unique_did).subst(tcx, substs);
-    let nonnull_ty = tcx.bound_type_of(nonnull_did).subst(tcx, substs);
+    let substs = tcx.mk_substs(&[pointee.into()]);
+    let unique_ty = tcx.type_of(unique_did).subst(tcx, substs);
+    let nonnull_ty = tcx.type_of(nonnull_did).subst(tcx, substs);
     let ptr_ty = tcx.mk_imm_ptr(pointee);
 
     (unique_ty, nonnull_ty, ptr_ty)
@@ -93,7 +93,7 @@ impl<'tcx> MirPass<'tcx> for ElaborateBoxDerefs {
         if let Some(def_id) = tcx.lang_items().owned_box() {
             let unique_did = tcx.adt_def(def_id).non_enum_variant().fields[0].did;
 
-            let Some(nonnull_def) = tcx.type_of(unique_did).ty_adt_def() else {
+            let Some(nonnull_def) = tcx.type_of(unique_did).subst_identity().ty_adt_def() else {
                 span_bug!(tcx.def_span(unique_did), "expected Box to contain Unique")
             };
 
@@ -138,7 +138,7 @@ impl<'tcx> MirPass<'tcx> for ElaborateBoxDerefs {
 
                     if let Some(mut new_projections) = new_projections {
                         new_projections.extend_from_slice(&place.projection[last_deref..]);
-                        place.projection = tcx.intern_place_elems(&new_projections);
+                        place.projection = tcx.mk_place_elems(&new_projections);
                     }
                 }
             }
diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs
index 65f4956d2..bdfd8dc6e 100644
--- a/compiler/rustc_mir_transform/src/elaborate_drops.rs
+++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs
@@ -18,6 +18,35 @@ use rustc_span::Span;
 use rustc_target::abi::VariantIdx;
 use std::fmt;
 
+/// During MIR building, Drop and DropAndReplace terminators are inserted in every place where a drop may occur.
+/// However, in this phase, the presence of these terminators does not guarantee that a destructor will run,
+/// as the target of the drop may be uninitialized.
+/// In general, the compiler cannot determine at compile time whether a destructor will run or not.
+///
+/// At a high level, this pass refines Drop and DropAndReplace to only run the destructor if the
+/// target is initialized. The way this is achievied is by inserting drop flags for every variable
+/// that may be dropped, and then using those flags to determine whether a destructor should run.
+/// This pass also removes DropAndReplace, replacing it with a Drop paired with an assign statement.
+/// Once this is complete, Drop terminators in the MIR correspond to a call to the "drop glue" or
+/// "drop shim" for the type of the dropped place.
+///
+/// This pass relies on dropped places having an associated move path, which is then used to determine
+/// the initialization status of the place and its descendants.
+/// It's worth noting that a MIR containing a Drop without an associated move path is probably ill formed,
+/// as it would allow running a destructor on a place behind a reference:
+///
+/// ```text
+// fn drop_term<T>(t: &mut T) {
+//     mir!(
+//         {
+//             Drop(*t, exit)
+//         }
+//         exit = {
+//             Return()
+//         }
+//     )
+// }
+/// ```
 pub struct ElaborateDrops;
 
 impl<'tcx> MirPass<'tcx> for ElaborateDrops {
@@ -38,13 +67,11 @@ impl<'tcx> MirPass<'tcx> for ElaborateDrops {
         };
         let un_derefer = UnDerefer { tcx: tcx, derefer_sidetable: side_table };
         let elaborate_patch = {
-            let body = &*body;
             let env = MoveDataParamEnv { move_data, param_env };
-            let dead_unwinds = find_dead_unwinds(tcx, body, &env, &un_derefer);
+            remove_dead_unwinds(tcx, body, &env, &un_derefer);
 
             let inits = MaybeInitializedPlaces::new(tcx, body, &env)
                 .into_engine(tcx, body)
-                .dead_unwinds(&dead_unwinds)
                 .pass_name("elaborate_drops")
                 .iterate_to_fixpoint()
                 .into_results_cursor(body);
@@ -52,11 +79,12 @@ impl<'tcx> MirPass<'tcx> for ElaborateDrops {
             let uninits = MaybeUninitializedPlaces::new(tcx, body, &env)
                 .mark_inactive_variants_as_uninit()
                 .into_engine(tcx, body)
-                .dead_unwinds(&dead_unwinds)
                 .pass_name("elaborate_drops")
                 .iterate_to_fixpoint()
                 .into_results_cursor(body);
 
+            let reachable = traversal::reachable_as_bitset(body);
+
             ElaborateDropsCtxt {
                 tcx,
                 body,
@@ -65,6 +93,7 @@ impl<'tcx> MirPass<'tcx> for ElaborateDrops {
                 drop_flags: Default::default(),
                 patch: MirPatch::new(body),
                 un_derefer: un_derefer,
+                reachable,
             }
             .elaborate()
         };
@@ -73,22 +102,21 @@ impl<'tcx> MirPass<'tcx> for ElaborateDrops {
     }
 }
 
-/// Returns the set of basic blocks whose unwind edges are known
-/// to not be reachable, because they are `drop` terminators
+/// Removes unwind edges which are known to be unreachable, because they are in `drop` terminators
 /// that can't drop anything.
-fn find_dead_unwinds<'tcx>(
+fn remove_dead_unwinds<'tcx>(
     tcx: TyCtxt<'tcx>,
-    body: &Body<'tcx>,
+    body: &mut Body<'tcx>,
     env: &MoveDataParamEnv<'tcx>,
     und: &UnDerefer<'tcx>,
-) -> BitSet<BasicBlock> {
-    debug!("find_dead_unwinds({:?})", body.span);
+) {
+    debug!("remove_dead_unwinds({:?})", body.span);
     // We only need to do this pass once, because unwind edges can only
     // reach cleanup blocks, which can't have unwind edges themselves.
-    let mut dead_unwinds = BitSet::new_empty(body.basic_blocks.len());
+    let mut dead_unwinds = Vec::new();
     let mut flow_inits = MaybeInitializedPlaces::new(tcx, body, &env)
         .into_engine(tcx, body)
-        .pass_name("find_dead_unwinds")
+        .pass_name("remove_dead_unwinds")
         .iterate_to_fixpoint()
         .into_results_cursor(body);
     for (bb, bb_data) in body.basic_blocks.iter_enumerated() {
@@ -100,16 +128,16 @@ fn find_dead_unwinds<'tcx>(
             _ => continue,
         };
 
-        debug!("find_dead_unwinds @ {:?}: {:?}", bb, bb_data);
+        debug!("remove_dead_unwinds @ {:?}: {:?}", bb, bb_data);
 
         let LookupResult::Exact(path) = env.move_data.rev_lookup.find(place.as_ref()) else {
-            debug!("find_dead_unwinds: has parent; skipping");
+            debug!("remove_dead_unwinds: has parent; skipping");
             continue;
         };
 
         flow_inits.seek_before_primary_effect(body.terminator_loc(bb));
         debug!(
-            "find_dead_unwinds @ {:?}: path({:?})={:?}; init_data={:?}",
+            "remove_dead_unwinds @ {:?}: path({:?})={:?}; init_data={:?}",
             bb,
             place,
             path,
@@ -121,13 +149,22 @@ fn find_dead_unwinds<'tcx>(
             maybe_live |= flow_inits.contains(child);
         });
 
-        debug!("find_dead_unwinds @ {:?}: maybe_live={}", bb, maybe_live);
+        debug!("remove_dead_unwinds @ {:?}: maybe_live={}", bb, maybe_live);
         if !maybe_live {
-            dead_unwinds.insert(bb);
+            dead_unwinds.push(bb);
         }
     }
 
-    dead_unwinds
+    if dead_unwinds.is_empty() {
+        return;
+    }
+
+    let basic_blocks = body.basic_blocks.as_mut();
+    for &bb in dead_unwinds.iter() {
+        if let Some(unwind) = basic_blocks[bb].terminator_mut().unwind_mut() {
+            *unwind = None;
+        }
+    }
 }
 
 struct InitializationData<'mir, 'tcx> {
@@ -261,6 +298,7 @@ struct ElaborateDropsCtxt<'a, 'tcx> {
     drop_flags: FxHashMap<MovePathIndex, Local>,
     patch: MirPatch<'tcx>,
     un_derefer: UnDerefer<'tcx>,
+    reachable: BitSet<BasicBlock>,
 }
 
 impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
@@ -300,6 +338,9 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
 
     fn collect_drop_flags(&mut self) {
         for (bb, data) in self.body.basic_blocks.iter_enumerated() {
+            if !self.reachable.contains(bb) {
+                continue;
+            }
             let terminator = data.terminator();
             let place = match terminator.kind {
                 TerminatorKind::Drop { ref place, .. }
@@ -355,6 +396,9 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
 
     fn elaborate_drops(&mut self) {
         for (bb, data) in self.body.basic_blocks.iter_enumerated() {
+            if !self.reachable.contains(bb) {
+                continue;
+            }
             let loc = Location { block: bb, statement_index: data.statements.len() };
             let terminator = data.terminator();
 
@@ -512,6 +556,9 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
 
     fn drop_flags_for_fn_rets(&mut self) {
         for (bb, data) in self.body.basic_blocks.iter_enumerated() {
+            if !self.reachable.contains(bb) {
+                continue;
+            }
             if let TerminatorKind::Call {
                 destination, target: Some(tgt), cleanup: Some(_), ..
             } = data.terminator().kind
@@ -547,6 +594,9 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
         // clobbered before they are read.
 
         for (bb, data) in self.body.basic_blocks.iter_enumerated() {
+            if !self.reachable.contains(bb) {
+                continue;
+            }
             debug!("drop_flags_for_locs({:?})", data);
             for i in 0..(data.statements.len() + 1) {
                 debug!("drop_flag_for_locs: stmt {}", i);
diff --git a/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs b/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs
index 1244c1802..e6546911a 100644
--- a/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs
+++ b/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs
@@ -49,7 +49,7 @@ fn has_ffi_unwind_calls(tcx: TyCtxt<'_>, local_def_id: LocalDefId) -> bool {
 
     let body = &*tcx.mir_built(ty::WithOptConstParam::unknown(local_def_id)).borrow();
 
-    let body_ty = tcx.type_of(def_id);
+    let body_ty = tcx.type_of(def_id).skip_binder();
     let body_abi = match body_ty.kind() {
         ty::FnDef(..) => body_ty.fn_sig(tcx).abi(),
         ty::Closure(..) => Abi::RustCall,
diff --git a/compiler/rustc_mir_transform/src/function_item_references.rs b/compiler/rustc_mir_transform/src/function_item_references.rs
index b708d780b..66d32b954 100644
--- a/compiler/rustc_mir_transform/src/function_item_references.rs
+++ b/compiler/rustc_mir_transform/src/function_item_references.rs
@@ -79,7 +79,7 @@ impl<'tcx> FunctionItemRefChecker<'_, 'tcx> {
         for bound in bounds {
             if let Some(bound_ty) = self.is_pointer_trait(&bound.kind().skip_binder()) {
                 // Get the argument types as they appear in the function signature.
-                let arg_defs = self.tcx.fn_sig(def_id).skip_binder().inputs();
+                let arg_defs = self.tcx.fn_sig(def_id).subst_identity().skip_binder().inputs();
                 for (arg_num, arg_def) in arg_defs.iter().enumerate() {
                     // For all types reachable from the argument type in the fn sig
                     for generic_inner_ty in arg_def.walk() {
@@ -111,11 +111,9 @@ impl<'tcx> FunctionItemRefChecker<'_, 'tcx> {
     /// If the given predicate is the trait `fmt::Pointer`, returns the bound parameter type.
     fn is_pointer_trait(&self, bound: &PredicateKind<'tcx>) -> Option<Ty<'tcx>> {
         if let ty::PredicateKind::Clause(ty::Clause::Trait(predicate)) = bound {
-            if self.tcx.is_diagnostic_item(sym::Pointer, predicate.def_id()) {
-                Some(predicate.trait_ref.self_ty())
-            } else {
-                None
-            }
+            self.tcx
+                .is_diagnostic_item(sym::Pointer, predicate.def_id())
+                .then(|| predicate.trait_ref.self_ty())
         } else {
             None
         }
@@ -161,7 +159,8 @@ impl<'tcx> FunctionItemRefChecker<'_, 'tcx> {
             .as_ref()
             .assert_crate_local()
             .lint_root;
-        let fn_sig = self.tcx.fn_sig(fn_id);
+        // FIXME: use existing printing routines to print the function signature
+        let fn_sig = self.tcx.fn_sig(fn_id).subst(self.tcx, fn_substs);
         let unsafety = fn_sig.unsafety().prefix_str();
         let abi = match fn_sig.abi() {
             Abi::Rust => String::from(""),
diff --git a/compiler/rustc_mir_transform/src/generator.rs b/compiler/rustc_mir_transform/src/generator.rs
index 39c61a34a..2e97312ee 100644
--- a/compiler/rustc_mir_transform/src/generator.rs
+++ b/compiler/rustc_mir_transform/src/generator.rs
@@ -52,9 +52,9 @@
 
 use crate::deref_separator::deref_finder;
 use crate::simplify;
-use crate::util::expand_aggregate;
 use crate::MirPass;
-use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_errors::pluralize;
 use rustc_hir as hir;
 use rustc_hir::lang_items::LangItem;
 use rustc_hir::GeneratorKind;
@@ -70,6 +70,9 @@ use rustc_mir_dataflow::impls::{
 };
 use rustc_mir_dataflow::storage::always_storage_live_locals;
 use rustc_mir_dataflow::{self, Analysis};
+use rustc_span::def_id::DefId;
+use rustc_span::symbol::sym;
+use rustc_span::Span;
 use rustc_target::abi::VariantIdx;
 use rustc_target::spec::PanicStrategy;
 use std::{iter, ops};
@@ -123,7 +126,7 @@ impl<'tcx> MutVisitor<'tcx> for DerefArgVisitor<'tcx> {
                 place,
                 Place {
                     local: SELF_ARG,
-                    projection: self.tcx().intern_place_elems(&[ProjectionElem::Deref]),
+                    projection: self.tcx().mk_place_elems(&[ProjectionElem::Deref]),
                 },
                 self.tcx,
             );
@@ -159,10 +162,9 @@ impl<'tcx> MutVisitor<'tcx> for PinArgVisitor<'tcx> {
                 place,
                 Place {
                     local: SELF_ARG,
-                    projection: self.tcx().intern_place_elems(&[ProjectionElem::Field(
-                        Field::new(0),
-                        self.ref_gen_ty,
-                    )]),
+                    projection: self
+                        .tcx()
+                        .mk_place_elems(&[ProjectionElem::Field(Field::new(0), self.ref_gen_ty)]),
                 },
                 self.tcx,
             );
@@ -184,7 +186,7 @@ fn replace_base<'tcx>(place: &mut Place<'tcx>, new_base: Place<'tcx>, tcx: TyCtx
     let mut new_projection = new_base.projection.to_vec();
     new_projection.append(&mut place.projection.to_vec());
 
-    place.projection = tcx.intern_place_elems(&new_projection);
+    place.projection = tcx.mk_place_elems(&new_projection);
 }
 
 const SELF_ARG: Local = Local::from_u32(1);
@@ -268,31 +270,26 @@ impl<'tcx> TransformVisitor<'tcx> {
             assert_eq!(self.state_adt_ref.variant(idx).fields.len(), 0);
 
             // FIXME(swatinem): assert that `val` is indeed unit?
-            statements.extend(expand_aggregate(
-                Place::return_place(),
-                std::iter::empty(),
-                kind,
+            statements.push(Statement {
+                kind: StatementKind::Assign(Box::new((
+                    Place::return_place(),
+                    Rvalue::Aggregate(Box::new(kind), vec![]),
+                ))),
                 source_info,
-                self.tcx,
-            ));
+            });
             return;
         }
 
         // else: `Poll::Ready(x)`, `GeneratorState::Yielded(x)` or `GeneratorState::Complete(x)`
         assert_eq!(self.state_adt_ref.variant(idx).fields.len(), 1);
 
-        let ty = self
-            .tcx
-            .bound_type_of(self.state_adt_ref.variant(idx).fields[0].did)
-            .subst(self.tcx, self.state_substs);
-
-        statements.extend(expand_aggregate(
-            Place::return_place(),
-            std::iter::once((val, ty)),
-            kind,
+        statements.push(Statement {
+            kind: StatementKind::Assign(Box::new((
+                Place::return_place(),
+                Rvalue::Aggregate(Box::new(kind), vec![val]),
+            ))),
             source_info,
-            self.tcx,
-        ));
+        });
     }
 
     // Create a Place referencing a generator struct field
@@ -302,7 +299,7 @@ impl<'tcx> TransformVisitor<'tcx> {
         let mut projection = base.projection.to_vec();
         projection.push(ProjectionElem::Field(Field::new(idx), ty));
 
-        Place { local: base.local, projection: self.tcx.intern_place_elems(&projection) }
+        Place { local: base.local, projection: self.tcx.mk_place_elems(&projection) }
     }
 
     // Create a statement which changes the discriminant
@@ -429,7 +426,7 @@ fn make_generator_state_argument_pinned<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body
 
     let pin_did = tcx.require_lang_item(LangItem::Pin, Some(body.span));
     let pin_adt_ref = tcx.adt_def(pin_did);
-    let substs = tcx.intern_substs(&[ref_gen_ty.into()]);
+    let substs = tcx.mk_substs(&[ref_gen_ty.into()]);
     let pin_ref_gen_ty = tcx.mk_adt(pin_adt_ref, substs);
 
     // Replace the by ref generator argument
@@ -489,7 +486,7 @@ fn transform_async_context<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
 
     let get_context_def_id = tcx.require_lang_item(LangItem::GetContext, None);
 
-    for bb in BasicBlock::new(0)..body.basic_blocks.next_index() {
+    for bb in START_BLOCK..body.basic_blocks.next_index() {
         let bb_data = &body[bb];
         if bb_data.is_cleanup {
             continue;
@@ -854,7 +851,7 @@ fn sanitize_witness<'tcx>(
     body: &Body<'tcx>,
     witness: Ty<'tcx>,
     upvars: Vec<Ty<'tcx>>,
-    saved_locals: &GeneratorSavedLocals,
+    layout: &GeneratorLayout<'tcx>,
 ) {
     let did = body.source.def_id();
     let param_env = tcx.param_env(did);
@@ -873,31 +870,36 @@ fn sanitize_witness<'tcx>(
         }
     };
 
-    for (local, decl) in body.local_decls.iter_enumerated() {
-        // Ignore locals which are internal or not saved between yields.
-        if !saved_locals.contains(local) || decl.internal {
+    let mut mismatches = Vec::new();
+    for fty in &layout.field_tys {
+        if fty.ignore_for_traits {
             continue;
         }
-        let decl_ty = tcx.normalize_erasing_regions(param_env, decl.ty);
+        let decl_ty = tcx.normalize_erasing_regions(param_env, fty.ty);
 
         // Sanity check that typeck knows about the type of locals which are
         // live across a suspension point
         if !allowed.contains(&decl_ty) && !allowed_upvars.contains(&decl_ty) {
-            span_bug!(
-                body.span,
-                "Broken MIR: generator contains type {} in MIR, \
-                       but typeck only knows about {} and {:?}",
-                decl_ty,
-                allowed,
-                allowed_upvars
-            );
+            mismatches.push(decl_ty);
         }
     }
+
+    if !mismatches.is_empty() {
+        span_bug!(
+            body.span,
+            "Broken MIR: generator contains type {:?} in MIR, \
+                       but typeck only knows about {} and {:?}",
+            mismatches,
+            allowed,
+            allowed_upvars
+        );
+    }
 }
 
 fn compute_layout<'tcx>(
+    tcx: TyCtxt<'tcx>,
     liveness: LivenessInfo,
-    body: &mut Body<'tcx>,
+    body: &Body<'tcx>,
 ) -> (
     FxHashMap<Local, (Ty<'tcx>, VariantIdx, usize)>,
     GeneratorLayout<'tcx>,
@@ -915,9 +917,33 @@ fn compute_layout<'tcx>(
     let mut locals = IndexVec::<GeneratorSavedLocal, _>::new();
     let mut tys = IndexVec::<GeneratorSavedLocal, _>::new();
     for (saved_local, local) in saved_locals.iter_enumerated() {
-        locals.push(local);
-        tys.push(body.local_decls[local].ty);
         debug!("generator saved local {:?} => {:?}", saved_local, local);
+
+        locals.push(local);
+        let decl = &body.local_decls[local];
+        debug!(?decl);
+
+        let ignore_for_traits = if tcx.sess.opts.unstable_opts.drop_tracking_mir {
+            match decl.local_info {
+                // Do not include raw pointers created from accessing `static` items, as those could
+                // well be re-created by another access to the same static.
+                Some(box LocalInfo::StaticRef { is_thread_local, .. }) => !is_thread_local,
+                // Fake borrows are only read by fake reads, so do not have any reality in
+                // post-analysis MIR.
+                Some(box LocalInfo::FakeBorrow) => true,
+                _ => false,
+            }
+        } else {
+            // FIXME(#105084) HIR-based drop tracking does not account for all the temporaries that
+            // MIR building may introduce. This leads to wrongly ignored types, but this is
+            // necessary for internal consistency and to avoid ICEs.
+            decl.internal
+        };
+        let decl =
+            GeneratorSavedTy { ty: decl.ty, source_info: decl.source_info, ignore_for_traits };
+        debug!(?decl);
+
+        tys.push(decl);
     }
 
     // Leave empty variants for the UNRESUMED, RETURNED, and POISONED states.
@@ -947,7 +973,7 @@ fn compute_layout<'tcx>(
             // just use the first one here. That's fine; fields do not move
             // around inside generators, so it doesn't matter which variant
             // index we access them by.
-            remap.entry(locals[saved_local]).or_insert((tys[saved_local], variant_index, idx));
+            remap.entry(locals[saved_local]).or_insert((tys[saved_local].ty, variant_index, idx));
         }
         variant_fields.push(fields);
         variant_source_info.push(source_info_at_suspension_points[suspension_point_idx]);
@@ -957,6 +983,7 @@ fn compute_layout<'tcx>(
 
     let layout =
         GeneratorLayout { field_tys: tys, variant_fields, variant_source_info, storage_conflicts };
+    debug!(?layout);
 
     (remap, layout, storage_liveness)
 }
@@ -1227,7 +1254,7 @@ fn create_generator_resume_function<'tcx>(
     use rustc_middle::mir::AssertKind::{ResumedAfterPanic, ResumedAfterReturn};
 
     // Jump to the entry point on the unresumed
-    cases.insert(0, (UNRESUMED, BasicBlock::new(0)));
+    cases.insert(0, (UNRESUMED, START_BLOCK));
 
     // Panic when resumed on the returned or poisoned state
     let generator_kind = body.generator_kind().unwrap();
@@ -1351,6 +1378,42 @@ fn create_cases<'tcx>(
         .collect()
 }
 
+#[instrument(level = "debug", skip(tcx), ret)]
+pub(crate) fn mir_generator_witnesses<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    def_id: DefId,
+) -> GeneratorLayout<'tcx> {
+    assert!(tcx.sess.opts.unstable_opts.drop_tracking_mir);
+    let def_id = def_id.expect_local();
+
+    let (body, _) = tcx.mir_promoted(ty::WithOptConstParam::unknown(def_id));
+    let body = body.borrow();
+    let body = &*body;
+
+    // The first argument is the generator type passed by value
+    let gen_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty;
+
+    // Get the interior types and substs which typeck computed
+    let movable = match *gen_ty.kind() {
+        ty::Generator(_, _, movability) => movability == hir::Movability::Movable,
+        _ => span_bug!(body.span, "unexpected generator type {}", gen_ty),
+    };
+
+    // When first entering the generator, move the resume argument into its new local.
+    let always_live_locals = always_storage_live_locals(&body);
+
+    let liveness_info = locals_live_across_suspend_points(tcx, body, &always_live_locals, movable);
+
+    // Extract locals which are live across suspension point into `layout`
+    // `remap` gives a mapping from local indices onto generator struct indices
+    // `storage_liveness` tells us which locals have live storage at suspension points
+    let (_, generator_layout, _) = compute_layout(tcx, liveness_info, body);
+
+    check_suspend_tys(tcx, &generator_layout, &body);
+
+    generator_layout
+}
+
 impl<'tcx> MirPass<'tcx> for StateTransform {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         let Some(yield_ty) = body.yield_ty() else {
@@ -1363,14 +1426,14 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
         // The first argument is the generator type passed by value
         let gen_ty = body.local_decls.raw[1].ty;
 
-        // Get the interior types and substs which typeck computed
-        let (upvars, interior, discr_ty, movable) = match *gen_ty.kind() {
+        // Get the discriminant type and substs which typeck computed
+        let (discr_ty, upvars, interior, movable) = match *gen_ty.kind() {
             ty::Generator(_, substs, movability) => {
                 let substs = substs.as_generator();
                 (
-                    substs.upvar_tys().collect(),
-                    substs.witness(),
                     substs.discr_ty(tcx),
+                    substs.upvar_tys().collect::<Vec<_>>(),
+                    substs.witness(),
                     movability == hir::Movability::Movable,
                 )
             }
@@ -1386,13 +1449,13 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
             // Compute Poll<return_ty>
             let poll_did = tcx.require_lang_item(LangItem::Poll, None);
             let poll_adt_ref = tcx.adt_def(poll_did);
-            let poll_substs = tcx.intern_substs(&[body.return_ty().into()]);
+            let poll_substs = tcx.mk_substs(&[body.return_ty().into()]);
             (poll_adt_ref, poll_substs)
         } else {
             // Compute GeneratorState<yield_ty, return_ty>
             let state_did = tcx.require_lang_item(LangItem::GeneratorState, None);
             let state_adt_ref = tcx.adt_def(state_did);
-            let state_substs = tcx.intern_substs(&[yield_ty.into(), body.return_ty().into()]);
+            let state_substs = tcx.mk_substs(&[yield_ty.into(), body.return_ty().into()]);
             (state_adt_ref, state_substs)
         };
         let ret_ty = tcx.mk_adt(state_adt_ref, state_substs);
@@ -1417,7 +1480,7 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
 
         // When first entering the generator, move the resume argument into its new local.
         let source_info = SourceInfo::outermost(body.span);
-        let stmts = &mut body.basic_blocks_mut()[BasicBlock::new(0)].statements;
+        let stmts = &mut body.basic_blocks_mut()[START_BLOCK].statements;
         stmts.insert(
             0,
             Statement {
@@ -1434,8 +1497,6 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
         let liveness_info =
             locals_live_across_suspend_points(tcx, body, &always_live_locals, movable);
 
-        sanitize_witness(tcx, body, interior, upvars, &liveness_info.saved_locals);
-
         if tcx.sess.opts.unstable_opts.validate_mir {
             let mut vis = EnsureGeneratorFieldAssignmentsNeverAlias {
                 assigned_local: None,
@@ -1449,7 +1510,13 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
         // Extract locals which are live across suspension point into `layout`
         // `remap` gives a mapping from local indices onto generator struct indices
         // `storage_liveness` tells us which locals have live storage at suspension points
-        let (remap, layout, storage_liveness) = compute_layout(liveness_info, body);
+        let (remap, layout, storage_liveness) = compute_layout(tcx, liveness_info, body);
+
+        if tcx.sess.opts.unstable_opts.validate_mir
+            && !tcx.sess.opts.unstable_opts.drop_tracking_mir
+        {
+            sanitize_witness(tcx, body, interior, upvars, &layout);
+        }
 
         let can_return = can_return(tcx, body, tcx.param_env(body.source.def_id()));
 
@@ -1583,6 +1650,7 @@ impl<'tcx> Visitor<'tcx> for EnsureGeneratorFieldAssignmentsNeverAlias<'_> {
             | StatementKind::AscribeUserType(..)
             | StatementKind::Coverage(..)
             | StatementKind::Intrinsic(..)
+            | StatementKind::ConstEvalCounter
             | StatementKind::Nop => {}
         }
     }
@@ -1631,3 +1699,212 @@ impl<'tcx> Visitor<'tcx> for EnsureGeneratorFieldAssignmentsNeverAlias<'_> {
         }
     }
 }
+
+fn check_suspend_tys<'tcx>(tcx: TyCtxt<'tcx>, layout: &GeneratorLayout<'tcx>, body: &Body<'tcx>) {
+    let mut linted_tys = FxHashSet::default();
+
+    // We want a user-facing param-env.
+    let param_env = tcx.param_env(body.source.def_id());
+
+    for (variant, yield_source_info) in
+        layout.variant_fields.iter().zip(&layout.variant_source_info)
+    {
+        debug!(?variant);
+        for &local in variant {
+            let decl = &layout.field_tys[local];
+            debug!(?decl);
+
+            if !decl.ignore_for_traits && linted_tys.insert(decl.ty) {
+                let Some(hir_id) = decl.source_info.scope.lint_root(&body.source_scopes) else { continue };
+
+                check_must_not_suspend_ty(
+                    tcx,
+                    decl.ty,
+                    hir_id,
+                    param_env,
+                    SuspendCheckData {
+                        source_span: decl.source_info.span,
+                        yield_span: yield_source_info.span,
+                        plural_len: 1,
+                        ..Default::default()
+                    },
+                );
+            }
+        }
+    }
+}
+
+#[derive(Default)]
+struct SuspendCheckData<'a> {
+    source_span: Span,
+    yield_span: Span,
+    descr_pre: &'a str,
+    descr_post: &'a str,
+    plural_len: usize,
+}
+
+// Returns whether it emitted a diagnostic or not
+// Note that this fn and the proceeding one are based on the code
+// for creating must_use diagnostics
+//
+// Note that this technique was chosen over things like a `Suspend` marker trait
+// as it is simpler and has precedent in the compiler
+fn check_must_not_suspend_ty<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    ty: Ty<'tcx>,
+    hir_id: hir::HirId,
+    param_env: ty::ParamEnv<'tcx>,
+    data: SuspendCheckData<'_>,
+) -> bool {
+    if ty.is_unit() {
+        return false;
+    }
+
+    let plural_suffix = pluralize!(data.plural_len);
+
+    debug!("Checking must_not_suspend for {}", ty);
+
+    match *ty.kind() {
+        ty::Adt(..) if ty.is_box() => {
+            let boxed_ty = ty.boxed_ty();
+            let descr_pre = &format!("{}boxed ", data.descr_pre);
+            check_must_not_suspend_ty(
+                tcx,
+                boxed_ty,
+                hir_id,
+                param_env,
+                SuspendCheckData { descr_pre, ..data },
+            )
+        }
+        ty::Adt(def, _) => check_must_not_suspend_def(tcx, def.did(), hir_id, data),
+        // FIXME: support adding the attribute to TAITs
+        ty::Alias(ty::Opaque, ty::AliasTy { def_id: def, .. }) => {
+            let mut has_emitted = false;
+            for &(predicate, _) in tcx.explicit_item_bounds(def) {
+                // We only look at the `DefId`, so it is safe to skip the binder here.
+                if let ty::PredicateKind::Clause(ty::Clause::Trait(ref poly_trait_predicate)) =
+                    predicate.kind().skip_binder()
+                {
+                    let def_id = poly_trait_predicate.trait_ref.def_id;
+                    let descr_pre = &format!("{}implementer{} of ", data.descr_pre, plural_suffix);
+                    if check_must_not_suspend_def(
+                        tcx,
+                        def_id,
+                        hir_id,
+                        SuspendCheckData { descr_pre, ..data },
+                    ) {
+                        has_emitted = true;
+                        break;
+                    }
+                }
+            }
+            has_emitted
+        }
+        ty::Dynamic(binder, _, _) => {
+            let mut has_emitted = false;
+            for predicate in binder.iter() {
+                if let ty::ExistentialPredicate::Trait(ref trait_ref) = predicate.skip_binder() {
+                    let def_id = trait_ref.def_id;
+                    let descr_post = &format!(" trait object{}{}", plural_suffix, data.descr_post);
+                    if check_must_not_suspend_def(
+                        tcx,
+                        def_id,
+                        hir_id,
+                        SuspendCheckData { descr_post, ..data },
+                    ) {
+                        has_emitted = true;
+                        break;
+                    }
+                }
+            }
+            has_emitted
+        }
+        ty::Tuple(fields) => {
+            let mut has_emitted = false;
+            for (i, ty) in fields.iter().enumerate() {
+                let descr_post = &format!(" in tuple element {i}");
+                if check_must_not_suspend_ty(
+                    tcx,
+                    ty,
+                    hir_id,
+                    param_env,
+                    SuspendCheckData { descr_post, ..data },
+                ) {
+                    has_emitted = true;
+                }
+            }
+            has_emitted
+        }
+        ty::Array(ty, len) => {
+            let descr_pre = &format!("{}array{} of ", data.descr_pre, plural_suffix);
+            check_must_not_suspend_ty(
+                tcx,
+                ty,
+                hir_id,
+                param_env,
+                SuspendCheckData {
+                    descr_pre,
+                    plural_len: len.try_eval_target_usize(tcx, param_env).unwrap_or(0) as usize + 1,
+                    ..data
+                },
+            )
+        }
+        // If drop tracking is enabled, we want to look through references, since the referrent
+        // may not be considered live across the await point.
+        ty::Ref(_region, ty, _mutability) => {
+            let descr_pre = &format!("{}reference{} to ", data.descr_pre, plural_suffix);
+            check_must_not_suspend_ty(
+                tcx,
+                ty,
+                hir_id,
+                param_env,
+                SuspendCheckData { descr_pre, ..data },
+            )
+        }
+        _ => false,
+    }
+}
+
+fn check_must_not_suspend_def(
+    tcx: TyCtxt<'_>,
+    def_id: DefId,
+    hir_id: hir::HirId,
+    data: SuspendCheckData<'_>,
+) -> bool {
+    if let Some(attr) = tcx.get_attr(def_id, sym::must_not_suspend) {
+        let msg = format!(
+            "{}`{}`{} held across a suspend point, but should not be",
+            data.descr_pre,
+            tcx.def_path_str(def_id),
+            data.descr_post,
+        );
+        tcx.struct_span_lint_hir(
+            rustc_session::lint::builtin::MUST_NOT_SUSPEND,
+            hir_id,
+            data.source_span,
+            msg,
+            |lint| {
+                // add span pointing to the offending yield/await
+                lint.span_label(data.yield_span, "the value is held across this suspend point");
+
+                // Add optional reason note
+                if let Some(note) = attr.value_str() {
+                    // FIXME(guswynn): consider formatting this better
+                    lint.span_note(data.source_span, note.as_str());
+                }
+
+                // Add some quick suggestions on what to do
+                // FIXME: can `drop` work as a suggestion here as well?
+                lint.span_help(
+                    data.source_span,
+                    "consider using a block (`{ ... }`) \
+                    to shrink the value's scope, ending before the suspend point",
+                )
+            },
+        );
+
+        true
+    } else {
+        false
+    }
+}
diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs
index 28c9080d3..6e6d6566f 100644
--- a/compiler/rustc_mir_transform/src/inline.rs
+++ b/compiler/rustc_mir_transform/src/inline.rs
@@ -96,7 +96,7 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
         history: Vec::new(),
         changed: false,
     };
-    let blocks = BasicBlock::new(0)..body.basic_blocks.next_index();
+    let blocks = START_BLOCK..body.basic_blocks.next_index();
     this.process_blocks(body, blocks);
     this.changed
 }
@@ -331,7 +331,7 @@ impl<'tcx> Inliner<'tcx> {
                     return None;
                 }
 
-                let fn_sig = self.tcx.bound_fn_sig(def_id).subst(self.tcx, substs);
+                let fn_sig = self.tcx.fn_sig(def_id).subst(self.tcx, substs);
                 let source_info = SourceInfo { span: fn_span, ..terminator.source_info };
 
                 return Some(CallSite { callee, fn_sig, block: bb, target, source_info });
@@ -888,7 +888,7 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
         location: Location,
     ) {
         if let ProjectionElem::Field(f, ty) = elem {
-            let parent = Place { local, projection: self.tcx.intern_place_elems(proj_base) };
+            let parent = Place { local, projection: self.tcx.mk_place_elems(proj_base) };
             let parent_ty = parent.ty(&self.callee_body.local_decls, self.tcx);
             let check_equal = |this: &mut Self, f_ty| {
                 if !util::is_equal_up_to_subtyping(this.tcx, this.param_env, ty, f_ty) {
@@ -900,7 +900,7 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
 
             let kind = match parent_ty.ty.kind() {
                 &ty::Alias(ty::Opaque, ty::AliasTy { def_id, substs, .. }) => {
-                    self.tcx.bound_type_of(def_id).subst(self.tcx, substs).kind()
+                    self.tcx.type_of(def_id).subst(self.tcx, substs).kind()
                 }
                 kind => kind,
             };
@@ -947,12 +947,12 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
                             return;
                         };
 
-                        let Some(&f_ty) = layout.field_tys.get(local) else {
+                        let Some(f_ty) = layout.field_tys.get(local) else {
                             self.validation = Err("malformed MIR");
                             return;
                         };
 
-                        f_ty
+                        f_ty.ty
                     } else {
                         let Some(f_ty) = substs.as_generator().prefix_tys().nth(f.index()) else {
                             self.validation = Err("malformed MIR");
diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs
index b027f9492..792457c80 100644
--- a/compiler/rustc_mir_transform/src/inline/cycle.rs
+++ b/compiler/rustc_mir_transform/src/inline/cycle.rs
@@ -2,7 +2,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_middle::mir::TerminatorKind;
-use rustc_middle::ty::TypeVisitable;
+use rustc_middle::ty::TypeVisitableExt;
 use rustc_middle::ty::{self, subst::SubstsRef, InstanceDef, TyCtxt};
 use rustc_session::Limit;
 
diff --git a/compiler/rustc_mir_transform/src/instcombine.rs b/compiler/rustc_mir_transform/src/instcombine.rs
index 2f3c65869..4182da195 100644
--- a/compiler/rustc_mir_transform/src/instcombine.rs
+++ b/compiler/rustc_mir_transform/src/instcombine.rs
@@ -6,7 +6,9 @@ use rustc_middle::mir::{
     BinOp, Body, Constant, ConstantKind, LocalDecls, Operand, Place, ProjectionElem, Rvalue,
     SourceInfo, Statement, StatementKind, Terminator, TerminatorKind, UnOp,
 };
-use rustc_middle::ty::{self, TyCtxt};
+use rustc_middle::ty::layout::ValidityRequirement;
+use rustc_middle::ty::{self, ParamEnv, SubstsRef, Ty, TyCtxt};
+use rustc_span::symbol::Symbol;
 
 pub struct InstCombine;
 
@@ -16,7 +18,11 @@ impl<'tcx> MirPass<'tcx> for InstCombine {
     }
 
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        let ctx = InstCombineContext { tcx, local_decls: &body.local_decls };
+        let ctx = InstCombineContext {
+            tcx,
+            local_decls: &body.local_decls,
+            param_env: tcx.param_env_reveal_all_normalized(body.source.def_id()),
+        };
         for block in body.basic_blocks.as_mut() {
             for statement in block.statements.iter_mut() {
                 match statement.kind {
@@ -24,6 +30,7 @@ impl<'tcx> MirPass<'tcx> for InstCombine {
                         ctx.combine_bool_cmp(&statement.source_info, rvalue);
                         ctx.combine_ref_deref(&statement.source_info, rvalue);
                         ctx.combine_len(&statement.source_info, rvalue);
+                        ctx.combine_cast(&statement.source_info, rvalue);
                     }
                     _ => {}
                 }
@@ -33,6 +40,10 @@ impl<'tcx> MirPass<'tcx> for InstCombine {
                 &mut block.terminator.as_mut().unwrap(),
                 &mut block.statements,
             );
+            ctx.combine_intrinsic_assert(
+                &mut block.terminator.as_mut().unwrap(),
+                &mut block.statements,
+            );
         }
     }
 }
@@ -40,6 +51,7 @@ impl<'tcx> MirPass<'tcx> for InstCombine {
 struct InstCombineContext<'tcx, 'a> {
     tcx: TyCtxt<'tcx>,
     local_decls: &'a LocalDecls<'tcx>,
+    param_env: ParamEnv<'tcx>,
 }
 
 impl<'tcx> InstCombineContext<'tcx, '_> {
@@ -99,11 +111,7 @@ impl<'tcx> InstCombineContext<'tcx, '_> {
     fn combine_ref_deref(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) {
         if let Rvalue::Ref(_, _, place) = rvalue {
             if let Some((base, ProjectionElem::Deref)) = place.as_ref().last_projection() {
-                if let ty::Ref(_, _, Mutability::Not) =
-                    base.ty(self.local_decls, self.tcx).ty.kind()
-                {
-                    // The dereferenced place must have type `&_`, so that we don't copy `&mut _`.
-                } else {
+                if rvalue.ty(self.local_decls, self.tcx) != base.ty(self.local_decls, self.tcx).ty {
                     return;
                 }
 
@@ -113,7 +121,7 @@ impl<'tcx> InstCombineContext<'tcx, '_> {
 
                 *rvalue = Rvalue::Use(Operand::Copy(Place {
                     local: base.local,
-                    projection: self.tcx.intern_place_elems(base.projection),
+                    projection: self.tcx.mk_place_elems(base.projection),
                 }));
             }
         }
@@ -135,6 +143,14 @@ impl<'tcx> InstCombineContext<'tcx, '_> {
         }
     }
 
+    fn combine_cast(&self, _source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) {
+        if let Rvalue::Cast(_kind, operand, ty) = rvalue {
+            if operand.ty(self.local_decls, self.tcx) == *ty {
+                *rvalue = Rvalue::Use(operand.clone());
+            }
+        }
+    }
+
     fn combine_primitive_clone(
         &self,
         terminator: &mut Terminator<'tcx>,
@@ -200,4 +216,58 @@ impl<'tcx> InstCombineContext<'tcx, '_> {
         });
         terminator.kind = TerminatorKind::Goto { target: destination_block };
     }
+
+    fn combine_intrinsic_assert(
+        &self,
+        terminator: &mut Terminator<'tcx>,
+        _statements: &mut Vec<Statement<'tcx>>,
+    ) {
+        let TerminatorKind::Call { func, target, .. } = &mut terminator.kind  else { return; };
+        let Some(target_block) = target else { return; };
+        let func_ty = func.ty(self.local_decls, self.tcx);
+        let Some((intrinsic_name, substs)) = resolve_rust_intrinsic(self.tcx, func_ty) else {
+            return;
+        };
+        // The intrinsics we are interested in have one generic parameter
+        if substs.is_empty() {
+            return;
+        }
+        let ty = substs.type_at(0);
+
+        let known_is_valid = intrinsic_assert_panics(self.tcx, self.param_env, ty, intrinsic_name);
+        match known_is_valid {
+            // We don't know the layout or it's not validity assertion at all, don't touch it
+            None => {}
+            Some(true) => {
+                // If we know the assert panics, indicate to later opts that the call diverges
+                *target = None;
+            }
+            Some(false) => {
+                // If we know the assert does not panic, turn the call into a Goto
+                terminator.kind = TerminatorKind::Goto { target: *target_block };
+            }
+        }
+    }
+}
+
+fn intrinsic_assert_panics<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
+    ty: Ty<'tcx>,
+    intrinsic_name: Symbol,
+) -> Option<bool> {
+    let requirement = ValidityRequirement::from_intrinsic(intrinsic_name)?;
+    Some(!tcx.check_validity_requirement((requirement, param_env.and(ty))).ok()?)
+}
+
+fn resolve_rust_intrinsic<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    func_ty: Ty<'tcx>,
+) -> Option<(Symbol, SubstsRef<'tcx>)> {
+    if let ty::FnDef(def_id, substs) = *func_ty.kind() {
+        if tcx.is_intrinsic(def_id) {
+            return Some((tcx.item_name(def_id), substs));
+        }
+    }
+    None
 }
diff --git a/compiler/rustc_mir_transform/src/large_enums.rs b/compiler/rustc_mir_transform/src/large_enums.rs
new file mode 100644
index 000000000..89e0a007d
--- /dev/null
+++ b/compiler/rustc_mir_transform/src/large_enums.rs
@@ -0,0 +1,294 @@
+use crate::rustc_middle::ty::util::IntTypeExt;
+use crate::MirPass;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_middle::mir::interpret::AllocId;
+use rustc_middle::mir::*;
+use rustc_middle::ty::{self, AdtDef, ParamEnv, Ty, TyCtxt};
+use rustc_session::Session;
+use rustc_target::abi::{HasDataLayout, Size, TagEncoding, Variants};
+
+/// A pass that seeks to optimize unnecessary moves of large enum types, if there is a large
+/// enough discrepancy between them.
+///
+/// i.e. If there is are two variants:
+/// ```
+/// enum Example {
+///   Small,
+///   Large([u32; 1024]),
+/// }
+/// ```
+/// Instead of emitting moves of the large variant,
+/// Perform a memcpy instead.
+/// Based off of [this HackMD](https://hackmd.io/@ft4bxUsFT5CEUBmRKYHr7w/rJM8BBPzD).
+///
+/// In summary, what this does is at runtime determine which enum variant is active,
+/// and instead of copying all the bytes of the largest possible variant,
+/// copy only the bytes for the currently active variant.
+pub struct EnumSizeOpt {
+    pub(crate) discrepancy: u64,
+}
+
+impl<'tcx> MirPass<'tcx> for EnumSizeOpt {
+    fn is_enabled(&self, sess: &Session) -> bool {
+        sess.opts.unstable_opts.unsound_mir_opts || sess.mir_opt_level() >= 3
+    }
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+        // NOTE: This pass may produce different MIR based on the alignment of the target
+        // platform, but it will still be valid.
+        self.optim(tcx, body);
+    }
+}
+
+impl EnumSizeOpt {
+    fn candidate<'tcx>(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        param_env: ParamEnv<'tcx>,
+        ty: Ty<'tcx>,
+        alloc_cache: &mut FxHashMap<Ty<'tcx>, AllocId>,
+    ) -> Option<(AdtDef<'tcx>, usize, AllocId)> {
+        let adt_def = match ty.kind() {
+            ty::Adt(adt_def, _substs) if adt_def.is_enum() => adt_def,
+            _ => return None,
+        };
+        let layout = tcx.layout_of(param_env.and(ty)).ok()?;
+        let variants = match &layout.variants {
+            Variants::Single { .. } => return None,
+            Variants::Multiple { tag_encoding, .. }
+                if matches!(tag_encoding, TagEncoding::Niche { .. }) =>
+            {
+                return None;
+            }
+            Variants::Multiple { variants, .. } if variants.len() <= 1 => return None,
+            Variants::Multiple { variants, .. } => variants,
+        };
+        let min = variants.iter().map(|v| v.size).min().unwrap();
+        let max = variants.iter().map(|v| v.size).max().unwrap();
+        if max.bytes() - min.bytes() < self.discrepancy {
+            return None;
+        }
+
+        let num_discrs = adt_def.discriminants(tcx).count();
+        if variants.iter_enumerated().any(|(var_idx, _)| {
+            let discr_for_var = adt_def.discriminant_for_variant(tcx, var_idx).val;
+            (discr_for_var > usize::MAX as u128) || (discr_for_var as usize >= num_discrs)
+        }) {
+            return None;
+        }
+        if let Some(alloc_id) = alloc_cache.get(&ty) {
+            return Some((*adt_def, num_discrs, *alloc_id));
+        }
+
+        let data_layout = tcx.data_layout();
+        let ptr_sized_int = data_layout.ptr_sized_integer();
+        let target_bytes = ptr_sized_int.size().bytes() as usize;
+        let mut data = vec![0; target_bytes * num_discrs];
+        macro_rules! encode_store {
+            ($curr_idx: expr, $endian: expr, $bytes: expr) => {
+                let bytes = match $endian {
+                    rustc_target::abi::Endian::Little => $bytes.to_le_bytes(),
+                    rustc_target::abi::Endian::Big => $bytes.to_be_bytes(),
+                };
+                for (i, b) in bytes.into_iter().enumerate() {
+                    data[$curr_idx + i] = b;
+                }
+            };
+        }
+
+        for (var_idx, layout) in variants.iter_enumerated() {
+            let curr_idx =
+                target_bytes * adt_def.discriminant_for_variant(tcx, var_idx).val as usize;
+            let sz = layout.size;
+            match ptr_sized_int {
+                rustc_target::abi::Integer::I32 => {
+                    encode_store!(curr_idx, data_layout.endian, sz.bytes() as u32);
+                }
+                rustc_target::abi::Integer::I64 => {
+                    encode_store!(curr_idx, data_layout.endian, sz.bytes());
+                }
+                _ => unreachable!(),
+            };
+        }
+        let alloc = interpret::Allocation::from_bytes(
+            data,
+            tcx.data_layout.ptr_sized_integer().align(&tcx.data_layout).abi,
+            Mutability::Not,
+        );
+        let alloc = tcx.create_memory_alloc(tcx.mk_const_alloc(alloc));
+        Some((*adt_def, num_discrs, *alloc_cache.entry(ty).or_insert(alloc)))
+    }
+    fn optim<'tcx>(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+        let mut alloc_cache = FxHashMap::default();
+        let body_did = body.source.def_id();
+        let param_env = tcx.param_env(body_did);
+
+        let blocks = body.basic_blocks.as_mut();
+        let local_decls = &mut body.local_decls;
+
+        for bb in blocks {
+            bb.expand_statements(|st| {
+                if let StatementKind::Assign(box (
+                    lhs,
+                    Rvalue::Use(Operand::Copy(rhs) | Operand::Move(rhs)),
+                )) = &st.kind
+                {
+                    let ty = lhs.ty(local_decls, tcx).ty;
+
+                    let source_info = st.source_info;
+                    let span = source_info.span;
+
+                    let (adt_def, num_variants, alloc_id) =
+                        self.candidate(tcx, param_env, ty, &mut alloc_cache)?;
+                    let alloc = tcx.global_alloc(alloc_id).unwrap_memory();
+
+                    let tmp_ty = tcx.mk_array(tcx.types.usize, num_variants as u64);
+
+                    let size_array_local = local_decls.push(LocalDecl::new(tmp_ty, span));
+                    let store_live = Statement {
+                        source_info,
+                        kind: StatementKind::StorageLive(size_array_local),
+                    };
+
+                    let place = Place::from(size_array_local);
+                    let constant_vals = Constant {
+                        span,
+                        user_ty: None,
+                        literal: ConstantKind::Val(
+                            interpret::ConstValue::ByRef { alloc, offset: Size::ZERO },
+                            tmp_ty,
+                        ),
+                    };
+                    let rval = Rvalue::Use(Operand::Constant(box (constant_vals)));
+
+                    let const_assign =
+                        Statement { source_info, kind: StatementKind::Assign(box (place, rval)) };
+
+                    let discr_place = Place::from(
+                        local_decls
+                            .push(LocalDecl::new(adt_def.repr().discr_type().to_ty(tcx), span)),
+                    );
+
+                    let store_discr = Statement {
+                        source_info,
+                        kind: StatementKind::Assign(box (discr_place, Rvalue::Discriminant(*rhs))),
+                    };
+
+                    let discr_cast_place =
+                        Place::from(local_decls.push(LocalDecl::new(tcx.types.usize, span)));
+
+                    let cast_discr = Statement {
+                        source_info,
+                        kind: StatementKind::Assign(box (
+                            discr_cast_place,
+                            Rvalue::Cast(
+                                CastKind::IntToInt,
+                                Operand::Copy(discr_place),
+                                tcx.types.usize,
+                            ),
+                        )),
+                    };
+
+                    let size_place =
+                        Place::from(local_decls.push(LocalDecl::new(tcx.types.usize, span)));
+
+                    let store_size = Statement {
+                        source_info,
+                        kind: StatementKind::Assign(box (
+                            size_place,
+                            Rvalue::Use(Operand::Copy(Place {
+                                local: size_array_local,
+                                projection: tcx
+                                    .mk_place_elems(&[PlaceElem::Index(discr_cast_place.local)]),
+                            })),
+                        )),
+                    };
+
+                    let dst =
+                        Place::from(local_decls.push(LocalDecl::new(tcx.mk_mut_ptr(ty), span)));
+
+                    let dst_ptr = Statement {
+                        source_info,
+                        kind: StatementKind::Assign(box (
+                            dst,
+                            Rvalue::AddressOf(Mutability::Mut, *lhs),
+                        )),
+                    };
+
+                    let dst_cast_ty = tcx.mk_mut_ptr(tcx.types.u8);
+                    let dst_cast_place =
+                        Place::from(local_decls.push(LocalDecl::new(dst_cast_ty, span)));
+
+                    let dst_cast = Statement {
+                        source_info,
+                        kind: StatementKind::Assign(box (
+                            dst_cast_place,
+                            Rvalue::Cast(CastKind::PtrToPtr, Operand::Copy(dst), dst_cast_ty),
+                        )),
+                    };
+
+                    let src =
+                        Place::from(local_decls.push(LocalDecl::new(tcx.mk_imm_ptr(ty), span)));
+
+                    let src_ptr = Statement {
+                        source_info,
+                        kind: StatementKind::Assign(box (
+                            src,
+                            Rvalue::AddressOf(Mutability::Not, *rhs),
+                        )),
+                    };
+
+                    let src_cast_ty = tcx.mk_imm_ptr(tcx.types.u8);
+                    let src_cast_place =
+                        Place::from(local_decls.push(LocalDecl::new(src_cast_ty, span)));
+
+                    let src_cast = Statement {
+                        source_info,
+                        kind: StatementKind::Assign(box (
+                            src_cast_place,
+                            Rvalue::Cast(CastKind::PtrToPtr, Operand::Copy(src), src_cast_ty),
+                        )),
+                    };
+
+                    let deinit_old =
+                        Statement { source_info, kind: StatementKind::Deinit(box dst) };
+
+                    let copy_bytes = Statement {
+                        source_info,
+                        kind: StatementKind::Intrinsic(
+                            box NonDivergingIntrinsic::CopyNonOverlapping(CopyNonOverlapping {
+                                src: Operand::Copy(src_cast_place),
+                                dst: Operand::Copy(dst_cast_place),
+                                count: Operand::Copy(size_place),
+                            }),
+                        ),
+                    };
+
+                    let store_dead = Statement {
+                        source_info,
+                        kind: StatementKind::StorageDead(size_array_local),
+                    };
+                    let iter = [
+                        store_live,
+                        const_assign,
+                        store_discr,
+                        cast_discr,
+                        store_size,
+                        dst_ptr,
+                        dst_cast,
+                        src_ptr,
+                        src_cast,
+                        deinit_old,
+                        copy_bytes,
+                        store_dead,
+                    ]
+                    .into_iter();
+
+                    st.make_nop();
+                    Some(iter)
+                } else {
+                    None
+                }
+            });
+        }
+    }
+}
diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs
index 20b7fdcfe..cdd28ae0c 100644
--- a/compiler/rustc_mir_transform/src/lib.rs
+++ b/compiler/rustc_mir_transform/src/lib.rs
@@ -1,6 +1,7 @@
 #![allow(rustc::potential_query_instability)]
 #![feature(box_patterns)]
 #![feature(drain_filter)]
+#![feature(box_syntax)]
 #![feature(let_chains)]
 #![feature(map_try_insert)]
 #![feature(min_specialization)]
@@ -23,6 +24,7 @@ use rustc_const_eval::util;
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_data_structures::steal::Steal;
 use rustc_hir as hir;
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_index::vec::IndexVec;
@@ -33,7 +35,7 @@ use rustc_middle::mir::{
     TerminatorKind,
 };
 use rustc_middle::ty::query::Providers;
-use rustc_middle::ty::{self, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, TyCtxt, TypeVisitableExt};
 use rustc_span::sym;
 
 #[macro_use]
@@ -54,10 +56,11 @@ mod const_debuginfo;
 mod const_goto;
 mod const_prop;
 mod const_prop_lint;
+mod copy_prop;
 mod coverage;
+mod ctfe_limit;
 mod dataflow_const_prop;
 mod dead_store_elimination;
-mod deaggregator;
 mod deduce_param_attrs;
 mod deduplicate_blocks;
 mod deref_separator;
@@ -71,6 +74,7 @@ mod function_item_references;
 mod generator;
 mod inline;
 mod instcombine;
+mod large_enums;
 mod lower_intrinsics;
 mod lower_slice_len;
 mod match_branches;
@@ -86,11 +90,11 @@ mod required_consts;
 mod reveal_all;
 mod separate_const_switch;
 mod shim;
+mod ssa;
 // This pass is public to allow external drivers to perform MIR cleanup
 pub mod simplify;
 mod simplify_branches;
 mod simplify_comparison_integral;
-mod simplify_try;
 mod sroa;
 mod uninhabited_enum_branching;
 mod unreachable_prop;
@@ -102,7 +106,6 @@ use rustc_mir_dataflow::rustc_peek;
 
 pub fn provide(providers: &mut Providers) {
     check_unsafety::provide(providers);
-    check_packed_ref::provide(providers);
     coverage::query::provide(providers);
     ffi_unwind_calls::provide(providers);
     shim::provide(providers);
@@ -124,6 +127,7 @@ pub fn provide(providers: &mut Providers) {
         mir_drops_elaborated_and_const_checked,
         mir_for_ctfe,
         mir_for_ctfe_of_const_arg,
+        mir_generator_witnesses: generator::mir_generator_witnesses,
         optimized_mir,
         is_mir_available,
         is_ctfe_mir_available: |tcx, did| is_mir_available(tcx, did),
@@ -188,7 +192,7 @@ fn remap_mir_for_const_eval_select<'tcx>(
                 let arguments = (0..num_args).map(|x| {
                     let mut place_elems = place_elems.to_vec();
                     place_elems.push(ProjectionElem::Field(x.into(), fields[x]));
-                    let projection = tcx.intern_place_elems(&place_elems);
+                    let projection = tcx.mk_place_elems(&place_elems);
                     let place = Place {
                         local: place.local,
                         projection,
@@ -244,7 +248,7 @@ fn mir_const_qualif(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) ->
 
     // N.B., this `borrow()` is guaranteed to be valid (i.e., the value
     // cannot yet be stolen), because `mir_promoted()`, which steals
-    // from `mir_const(), forces this query to execute before
+    // from `mir_const()`, forces this query to execute before
     // performing the steal.
     let body = &tcx.mir_const(def).borrow();
 
@@ -410,6 +414,8 @@ fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -
         }
     }
 
+    pm::run_passes(tcx, &mut body, &[&ctfe_limit::CtfeLimit], None);
+
     debug_assert!(!body.has_free_regions(), "Free regions in MIR for CTFE");
 
     body
@@ -426,6 +432,11 @@ fn mir_drops_elaborated_and_const_checked(
         return tcx.mir_drops_elaborated_and_const_checked(def);
     }
 
+    if tcx.sess.opts.unstable_opts.drop_tracking_mir
+        && let DefKind::Generator = tcx.def_kind(def.did)
+    {
+        tcx.ensure().mir_generator_witnesses(def.did);
+    }
     let mir_borrowck = tcx.mir_borrowck_opt_const_arg(def);
 
     let is_fn_like = tcx.def_kind(def.did).is_fn_like();
@@ -513,9 +524,6 @@ fn run_runtime_lowering_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         &elaborate_box_derefs::ElaborateBoxDerefs,
         &generator::StateTransform,
         &add_retag::AddRetag,
-        // Deaggregator is necessary for const prop. We may want to consider implementing
-        // CTFE support for aggregates.
-        &deaggregator::Deaggregator,
         &Lint(const_prop_lint::ConstProp),
     ];
     pm::run_passes_no_validate(tcx, body, passes, Some(MirPhase::Runtime(RuntimePhase::Initial)));
@@ -541,13 +549,13 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         &[
             &reveal_all::RevealAll, // has to be done before inlining, since inlined code is in RevealAll mode.
             &lower_slice_len::LowerSliceLenCalls, // has to be done before inlining, otherwise actual call will be almost always inlined. Also simple, so can just do first
-            &normalize_array_len::NormalizeArrayLen, // has to run after `slice::len` lowering
             &unreachable_prop::UnreachablePropagation,
             &uninhabited_enum_branching::UninhabitedEnumBranching,
             &o1(simplify::SimplifyCfg::new("after-uninhabited-enum-branching")),
             &inline::Inline,
             &remove_storage_markers::RemoveStorageMarkers,
             &remove_zsts::RemoveZsts,
+            &normalize_array_len::NormalizeArrayLen, // has to run after `slice::len` lowering
             &const_goto::ConstGoto,
             &remove_unneeded_drops::RemoveUnneededDrops,
             &sroa::ScalarReplacementOfAggregates,
@@ -557,6 +565,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
             &instcombine::InstCombine,
             &separate_const_switch::SeparateConstSwitch,
             &simplify::SimplifyLocals::new("before-const-prop"),
+            &copy_prop::CopyProp,
             //
             // FIXME(#70073): This pass is responsible for both optimization as well as some lints.
             &const_prop::ConstProp,
@@ -567,8 +576,6 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
             &o1(simplify_branches::SimplifyConstCondition::new("after-const-prop")),
             &early_otherwise_branch::EarlyOtherwiseBranch,
             &simplify_comparison_integral::SimplifyComparisonIntegral,
-            &simplify_try::SimplifyArmIdentity,
-            &simplify_try::SimplifyBranchSame,
             &dead_store_elimination::DeadStoreElimination,
             &dest_prop::DestinationPropagation,
             &o1(simplify_branches::SimplifyConstCondition::new("final")),
@@ -578,6 +585,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
             &simplify::SimplifyLocals::new("final"),
             &multiple_return_terminators::MultipleReturnTerminators,
             &deduplicate_blocks::DeduplicateBlocks,
+            &large_enums::EnumSizeOpt { discrepancy: 128 },
             // Some cleanup necessary at least for LLVM and potentially other codegen backends.
             &add_call_guards::CriticalCallEdges,
             // Dump the end result for testing and debugging purposes.
diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs
index 9892580e6..f596cc180 100644
--- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs
+++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs
@@ -107,9 +107,29 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
                         }
                     }
                     sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
-                        // The checked binary operations are not suitable target for lowering here,
-                        // since their semantics depend on the value of overflow-checks flag used
-                        // during codegen. Issue #35310.
+                        if let Some(target) = *target {
+                            let lhs;
+                            let rhs;
+                            {
+                                let mut args = args.drain(..);
+                                lhs = args.next().unwrap();
+                                rhs = args.next().unwrap();
+                            }
+                            let bin_op = match intrinsic_name {
+                                sym::add_with_overflow => BinOp::Add,
+                                sym::sub_with_overflow => BinOp::Sub,
+                                sym::mul_with_overflow => BinOp::Mul,
+                                _ => bug!("unexpected intrinsic"),
+                            };
+                            block.statements.push(Statement {
+                                source_info: terminator.source_info,
+                                kind: StatementKind::Assign(Box::new((
+                                    *destination,
+                                    Rvalue::CheckedBinaryOp(bin_op, Box::new((lhs, rhs))),
+                                ))),
+                            });
+                            terminator.kind = TerminatorKind::Goto { target };
+                        }
                     }
                     sym::size_of | sym::min_align_of => {
                         if let Some(target) = *target {
diff --git a/compiler/rustc_mir_transform/src/lower_slice_len.rs b/compiler/rustc_mir_transform/src/lower_slice_len.rs
index 2f02d00ec..c6e7468aa 100644
--- a/compiler/rustc_mir_transform/src/lower_slice_len.rs
+++ b/compiler/rustc_mir_transform/src/lower_slice_len.rs
@@ -68,8 +68,11 @@ fn lower_slice_len_call<'tcx>(
                 ty::FnDef(fn_def_id, _) if fn_def_id == &slice_len_fn_item_def_id => {
                     // perform modifications
                     // from something like `_5 = core::slice::<impl [u8]>::len(move _6) -> bb1`
-                    // into `_5 = Len(*_6)
+                    // into:
+                    // ```
+                    // _5 = Len(*_6)
                     // goto bb1
+                    // ```
 
                     // make new RValue for Len
                     let deref_arg = tcx.mk_place_deref(arg);
diff --git a/compiler/rustc_mir_transform/src/normalize_array_len.rs b/compiler/rustc_mir_transform/src/normalize_array_len.rs
index 1708b287e..b36c8a0bd 100644
--- a/compiler/rustc_mir_transform/src/normalize_array_len.rs
+++ b/compiler/rustc_mir_transform/src/normalize_array_len.rs
@@ -1,288 +1,104 @@
 //! This pass eliminates casting of arrays into slices when their length
 //! is taken using `.len()` method. Handy to preserve information in MIR for const prop
 
+use crate::ssa::SsaLocals;
 use crate::MirPass;
-use rustc_data_structures::fx::FxIndexMap;
-use rustc_data_structures::intern::Interned;
-use rustc_index::bit_set::BitSet;
 use rustc_index::vec::IndexVec;
+use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
-use rustc_middle::ty::{self, ReErased, Region, TyCtxt};
-
-const MAX_NUM_BLOCKS: usize = 800;
-const MAX_NUM_LOCALS: usize = 3000;
+use rustc_middle::ty::{self, TyCtxt};
+use rustc_mir_dataflow::impls::borrowed_locals;
 
 pub struct NormalizeArrayLen;
 
 impl<'tcx> MirPass<'tcx> for NormalizeArrayLen {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
-        // See #105929
-        sess.mir_opt_level() >= 4 && sess.opts.unstable_opts.unsound_mir_opts
+        sess.mir_opt_level() >= 3
     }
 
+    #[instrument(level = "trace", skip(self, tcx, body))]
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        // early returns for edge cases of highly unrolled functions
-        if body.basic_blocks.len() > MAX_NUM_BLOCKS {
-            return;
-        }
-        if body.local_decls.len() > MAX_NUM_LOCALS {
-            return;
-        }
+        debug!(def_id = ?body.source.def_id());
         normalize_array_len_calls(tcx, body)
     }
 }
 
-pub fn normalize_array_len_calls<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-    // We don't ever touch terminators, so no need to invalidate the CFG cache
-    let basic_blocks = body.basic_blocks.as_mut_preserves_cfg();
-    let local_decls = &mut body.local_decls;
+fn normalize_array_len_calls<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id());
+    let borrowed_locals = borrowed_locals(body);
+    let ssa = SsaLocals::new(tcx, param_env, body, &borrowed_locals);
 
-    // do a preliminary analysis to see if we ever have locals of type `[T;N]` or `&[T;N]`
-    let mut interesting_locals = BitSet::new_empty(local_decls.len());
-    for (local, decl) in local_decls.iter_enumerated() {
-        match decl.ty.kind() {
-            ty::Array(..) => {
-                interesting_locals.insert(local);
-            }
-            ty::Ref(.., ty, Mutability::Not) => match ty.kind() {
-                ty::Array(..) => {
-                    interesting_locals.insert(local);
-                }
-                _ => {}
-            },
-            _ => {}
-        }
-    }
-    if interesting_locals.is_empty() {
-        // we have found nothing to analyze
-        return;
-    }
-    let num_intesting_locals = interesting_locals.count();
-    let mut state = FxIndexMap::with_capacity_and_hasher(num_intesting_locals, Default::default());
-    let mut patches_scratchpad =
-        FxIndexMap::with_capacity_and_hasher(num_intesting_locals, Default::default());
-    let mut replacements_scratchpad =
-        FxIndexMap::with_capacity_and_hasher(num_intesting_locals, Default::default());
-    for block in basic_blocks {
-        // make length calls for arrays [T; N] not to decay into length calls for &[T]
-        // that forbids constant propagation
-        normalize_array_len_call(
-            tcx,
-            block,
-            local_decls,
-            &interesting_locals,
-            &mut state,
-            &mut patches_scratchpad,
-            &mut replacements_scratchpad,
-        );
-        state.clear();
-        patches_scratchpad.clear();
-        replacements_scratchpad.clear();
-    }
-}
+    let slice_lengths = compute_slice_length(tcx, &ssa, body);
+    debug!(?slice_lengths);
 
-struct Patcher<'a, 'tcx> {
-    tcx: TyCtxt<'tcx>,
-    patches_scratchpad: &'a FxIndexMap<usize, usize>,
-    replacements_scratchpad: &'a mut FxIndexMap<usize, Local>,
-    local_decls: &'a mut IndexVec<Local, LocalDecl<'tcx>>,
-    statement_idx: usize,
+    Replacer { tcx, slice_lengths }.visit_body_preserves_cfg(body);
 }
 
-impl<'tcx> Patcher<'_, 'tcx> {
-    fn patch_expand_statement(
-        &mut self,
-        statement: &mut Statement<'tcx>,
-    ) -> Option<std::vec::IntoIter<Statement<'tcx>>> {
-        let idx = self.statement_idx;
-        if let Some(len_statemnt_idx) = self.patches_scratchpad.get(&idx).copied() {
-            let mut statements = Vec::with_capacity(2);
-
-            // we are at statement that performs a cast. The only sound way is
-            // to create another local that performs a similar copy without a cast and then
-            // use this copy in the Len operation
-
-            match &statement.kind {
-                StatementKind::Assign(box (
-                    ..,
-                    Rvalue::Cast(
-                        CastKind::Pointer(ty::adjustment::PointerCast::Unsize),
-                        operand,
-                        _,
-                    ),
-                )) => {
-                    match operand {
-                        Operand::Copy(place) | Operand::Move(place) => {
-                            // create new local
-                            let ty = operand.ty(self.local_decls, self.tcx);
-                            let local_decl = LocalDecl::with_source_info(ty, statement.source_info);
-                            let local = self.local_decls.push(local_decl);
-                            // make it live
-                            let mut make_live_statement = statement.clone();
-                            make_live_statement.kind = StatementKind::StorageLive(local);
-                            statements.push(make_live_statement);
-                            // copy into it
-
-                            let operand = Operand::Copy(*place);
-                            let mut make_copy_statement = statement.clone();
-                            let assign_to = Place::from(local);
-                            let rvalue = Rvalue::Use(operand);
-                            make_copy_statement.kind =
-                                StatementKind::Assign(Box::new((assign_to, rvalue)));
-                            statements.push(make_copy_statement);
-
-                            // to reorder we have to copy and make NOP
-                            statements.push(statement.clone());
-                            statement.make_nop();
-
-                            self.replacements_scratchpad.insert(len_statemnt_idx, local);
-                        }
-                        _ => {
-                            unreachable!("it's a bug in the implementation")
-                        }
-                    }
-                }
-                _ => {
-                    unreachable!("it's a bug in the implementation")
+fn compute_slice_length<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    ssa: &SsaLocals,
+    body: &Body<'tcx>,
+) -> IndexVec<Local, Option<ty::Const<'tcx>>> {
+    let mut slice_lengths = IndexVec::from_elem(None, &body.local_decls);
+
+    for (local, rvalue) in ssa.assignments(body) {
+        match rvalue {
+            Rvalue::Cast(
+                CastKind::Pointer(ty::adjustment::PointerCast::Unsize),
+                operand,
+                cast_ty,
+            ) => {
+                let operand_ty = operand.ty(body, tcx);
+                debug!(?operand_ty);
+                if let Some(operand_ty) = operand_ty.builtin_deref(true)
+                    && let ty::Array(_, len) = operand_ty.ty.kind()
+                    && let Some(cast_ty) = cast_ty.builtin_deref(true)
+                    && let ty::Slice(..) = cast_ty.ty.kind()
+                {
+                    slice_lengths[local] = Some(*len);
                 }
             }
-
-            self.statement_idx += 1;
-
-            Some(statements.into_iter())
-        } else if let Some(local) = self.replacements_scratchpad.get(&idx).copied() {
-            let mut statements = Vec::with_capacity(2);
-
-            match &statement.kind {
-                StatementKind::Assign(box (into, Rvalue::Len(place))) => {
-                    let add_deref = if let Some(..) = place.as_local() {
-                        false
-                    } else if let Some(..) = place.local_or_deref_local() {
-                        true
-                    } else {
-                        unreachable!("it's a bug in the implementation")
-                    };
-                    // replace len statement
-                    let mut len_statement = statement.clone();
-                    let mut place = Place::from(local);
-                    if add_deref {
-                        place = self.tcx.mk_place_deref(place);
-                    }
-                    len_statement.kind =
-                        StatementKind::Assign(Box::new((*into, Rvalue::Len(place))));
-                    statements.push(len_statement);
-
-                    // make temporary dead
-                    let mut make_dead_statement = statement.clone();
-                    make_dead_statement.kind = StatementKind::StorageDead(local);
-                    statements.push(make_dead_statement);
-
-                    // make original statement NOP
-                    statement.make_nop();
+            // The length information is stored in the fat pointer, so we treat `operand` as a value.
+            Rvalue::Use(operand) => {
+                if let Some(rhs) = operand.place() && let Some(rhs) = rhs.as_local() {
+                    slice_lengths[local] = slice_lengths[rhs];
                 }
-                _ => {
-                    unreachable!("it's a bug in the implementation")
+            }
+            // The length information is stored in the fat pointer.
+            // Reborrowing copies length information from one pointer to the other.
+            Rvalue::Ref(_, _, rhs) | Rvalue::AddressOf(_, rhs) => {
+                if let [PlaceElem::Deref] = rhs.projection[..] {
+                    slice_lengths[local] = slice_lengths[rhs.local];
                 }
             }
-
-            self.statement_idx += 1;
-
-            Some(statements.into_iter())
-        } else {
-            self.statement_idx += 1;
-            None
+            _ => {}
         }
     }
+
+    slice_lengths
 }
 
-fn normalize_array_len_call<'tcx>(
+struct Replacer<'tcx> {
     tcx: TyCtxt<'tcx>,
-    block: &mut BasicBlockData<'tcx>,
-    local_decls: &mut IndexVec<Local, LocalDecl<'tcx>>,
-    interesting_locals: &BitSet<Local>,
-    state: &mut FxIndexMap<Local, usize>,
-    patches_scratchpad: &mut FxIndexMap<usize, usize>,
-    replacements_scratchpad: &mut FxIndexMap<usize, Local>,
-) {
-    for (statement_idx, statement) in block.statements.iter_mut().enumerate() {
-        match &mut statement.kind {
-            StatementKind::Assign(box (place, rvalue)) => {
-                match rvalue {
-                    Rvalue::Cast(
-                        CastKind::Pointer(ty::adjustment::PointerCast::Unsize),
-                        operand,
-                        cast_ty,
-                    ) => {
-                        let Some(local) = place.as_local() else { return };
-                        match operand {
-                            Operand::Copy(place) | Operand::Move(place) => {
-                                let Some(operand_local) = place.local_or_deref_local() else { return; };
-                                if !interesting_locals.contains(operand_local) {
-                                    return;
-                                }
-                                let operand_ty = local_decls[operand_local].ty;
-                                match (operand_ty.kind(), cast_ty.kind()) {
-                                    (ty::Array(of_ty_src, ..), ty::Slice(of_ty_dst)) => {
-                                        if of_ty_src == of_ty_dst {
-                                            // this is a cast from [T; N] into [T], so we are good
-                                            state.insert(local, statement_idx);
-                                        }
-                                    }
-                                    // current way of patching doesn't allow to work with `mut`
-                                    (
-                                        ty::Ref(
-                                            Region(Interned(ReErased, _)),
-                                            operand_ty,
-                                            Mutability::Not,
-                                        ),
-                                        ty::Ref(
-                                            Region(Interned(ReErased, _)),
-                                            cast_ty,
-                                            Mutability::Not,
-                                        ),
-                                    ) => {
-                                        match (operand_ty.kind(), cast_ty.kind()) {
-                                            // current way of patching doesn't allow to work with `mut`
-                                            (ty::Array(of_ty_src, ..), ty::Slice(of_ty_dst)) => {
-                                                if of_ty_src == of_ty_dst {
-                                                    // this is a cast from [T; N] into [T], so we are good
-                                                    state.insert(local, statement_idx);
-                                                }
-                                            }
-                                            _ => {}
-                                        }
-                                    }
-                                    _ => {}
-                                }
-                            }
-                            _ => {}
-                        }
-                    }
-                    Rvalue::Len(place) => {
-                        let Some(local) = place.local_or_deref_local() else {
-                            return;
-                        };
-                        if let Some(cast_statement_idx) = state.get(&local).copied() {
-                            patches_scratchpad.insert(cast_statement_idx, statement_idx);
-                        }
-                    }
-                    _ => {
-                        // invalidate
-                        state.remove(&place.local);
-                    }
-                }
-            }
-            _ => {}
-        }
-    }
+    slice_lengths: IndexVec<Local, Option<ty::Const<'tcx>>>,
+}
 
-    let mut patcher = Patcher {
-        tcx,
-        patches_scratchpad: &*patches_scratchpad,
-        replacements_scratchpad,
-        local_decls,
-        statement_idx: 0,
-    };
+impl<'tcx> MutVisitor<'tcx> for Replacer<'tcx> {
+    fn tcx(&self) -> TyCtxt<'tcx> {
+        self.tcx
+    }
 
-    block.expand_statements(|st| patcher.patch_expand_statement(st));
+    fn visit_rvalue(&mut self, rvalue: &mut Rvalue<'tcx>, loc: Location) {
+        if let Rvalue::Len(place) = rvalue
+            && let [PlaceElem::Deref] = &place.projection[..]
+            && let Some(len) = self.slice_lengths[place.local]
+        {
+            *rvalue = Rvalue::Use(Operand::Constant(Box::new(Constant {
+                span: rustc_span::DUMMY_SP,
+                user_ty: None,
+                literal: ConstantKind::from_const(len, self.tcx),
+            })));
+        }
+        self.super_rvalue(rvalue, loc);
+    }
 }
diff --git a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs
index f1bbf2ea7..e3a03aa08 100644
--- a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs
+++ b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs
@@ -35,6 +35,7 @@ impl RemoveNoopLandingPads {
                 | StatementKind::StorageDead(_)
                 | StatementKind::AscribeUserType(..)
                 | StatementKind::Coverage(..)
+                | StatementKind::ConstEvalCounter
                 | StatementKind::Nop => {
                     // These are all noops in a landing pad
                 }
diff --git a/compiler/rustc_mir_transform/src/remove_zsts.rs b/compiler/rustc_mir_transform/src/remove_zsts.rs
index 6cabef92d..1becfddb2 100644
--- a/compiler/rustc_mir_transform/src/remove_zsts.rs
+++ b/compiler/rustc_mir_transform/src/remove_zsts.rs
@@ -13,7 +13,7 @@ impl<'tcx> MirPass<'tcx> for RemoveZsts {
 
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         // Avoid query cycles (generators require optimized MIR for layout).
-        if tcx.type_of(body.source.def_id()).is_generator() {
+        if tcx.type_of(body.source.def_id()).subst_identity().is_generator() {
             return;
         }
         let param_env = tcx.param_env(body.source.def_id());
diff --git a/compiler/rustc_mir_transform/src/separate_const_switch.rs b/compiler/rustc_mir_transform/src/separate_const_switch.rs
index 2f116aaa9..a24d2d34d 100644
--- a/compiler/rustc_mir_transform/src/separate_const_switch.rs
+++ b/compiler/rustc_mir_transform/src/separate_const_switch.rs
@@ -250,6 +250,7 @@ fn is_likely_const<'tcx>(mut tracked_place: Place<'tcx>, block: &BasicBlockData<
             | StatementKind::Coverage(_)
             | StatementKind::StorageDead(_)
             | StatementKind::Intrinsic(_)
+            | StatementKind::ConstEvalCounter
             | StatementKind::Nop => {}
         }
     }
@@ -318,6 +319,7 @@ fn find_determining_place<'tcx>(
             | StatementKind::AscribeUserType(_, _)
             | StatementKind::Coverage(_)
             | StatementKind::Intrinsic(_)
+            | StatementKind::ConstEvalCounter
             | StatementKind::Nop => {}
 
             // If the discriminant is set, it is always set
diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs
index dace540fa..ebe63d6cb 100644
--- a/compiler/rustc_mir_transform/src/shim.rs
+++ b/compiler/rustc_mir_transform/src/shim.rs
@@ -15,7 +15,6 @@ use rustc_target::spec::abi::Abi;
 use std::fmt;
 use std::iter;
 
-use crate::util::expand_aggregate;
 use crate::{
     abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, deref_separator,
     pass_manager as pm, remove_noop_landing_pads, simplify,
@@ -148,11 +147,11 @@ fn build_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option<Ty<'tcx>>)
     assert!(!matches!(ty, Some(ty) if ty.is_generator()));
 
     let substs = if let Some(ty) = ty {
-        tcx.intern_substs(&[ty.into()])
+        tcx.mk_substs(&[ty.into()])
     } else {
         InternalSubsts::identity_for_item(tcx, def_id)
     };
-    let sig = tcx.bound_fn_sig(def_id).subst(tcx, substs);
+    let sig = tcx.fn_sig(def_id).subst(tcx, substs);
     let sig = tcx.erase_late_bound_regions(sig);
     let span = tcx.def_span(def_id);
 
@@ -363,7 +362,7 @@ impl<'tcx> CloneShimBuilder<'tcx> {
         // we must subst the self_ty because it's
         // otherwise going to be TySelf and we can't index
         // or access fields of a Place of type TySelf.
-        let sig = tcx.bound_fn_sig(def_id).subst(tcx, &[self_ty.into()]);
+        let sig = tcx.fn_sig(def_id).subst(tcx, &[self_ty.into()]);
         let sig = tcx.erase_late_bound_regions(sig);
         let span = tcx.def_span(def_id);
 
@@ -427,7 +426,7 @@ impl<'tcx> CloneShimBuilder<'tcx> {
     fn make_place(&mut self, mutability: Mutability, ty: Ty<'tcx>) -> Place<'tcx> {
         let span = self.span;
         let mut local = LocalDecl::new(ty, span);
-        if mutability == Mutability::Not {
+        if mutability.is_not() {
             local = local.immutable();
         }
         Place::from(self.local_decls.push(local))
@@ -598,7 +597,7 @@ fn build_call_shim<'tcx>(
         let untuple_args = sig.inputs();
 
         // Create substitutions for the `Self` and `Args` generic parameters of the shim body.
-        let arg_tup = tcx.mk_tup(untuple_args.iter());
+        let arg_tup = tcx.mk_tup(untuple_args);
 
         (Some([ty.into(), arg_tup.into()]), Some(untuple_args))
     } else {
@@ -606,7 +605,7 @@ fn build_call_shim<'tcx>(
     };
 
     let def_id = instance.def_id();
-    let sig = tcx.bound_fn_sig(def_id);
+    let sig = tcx.fn_sig(def_id);
     let sig = sig.map_bound(|sig| tcx.erase_late_bound_regions(sig));
 
     assert_eq!(sig_substs.is_some(), !instance.has_polymorphic_mir_body());
@@ -633,7 +632,7 @@ fn build_call_shim<'tcx>(
             Adjustment::Deref => tcx.mk_imm_ptr(fnty),
             Adjustment::RefMut => tcx.mk_mut_ptr(fnty),
         };
-        sig.inputs_and_output = tcx.intern_type_list(&inputs_and_output);
+        sig.inputs_and_output = tcx.mk_type_list(&inputs_and_output);
     }
 
     // FIXME(eddyb) avoid having this snippet both here and in
@@ -644,7 +643,7 @@ fn build_call_shim<'tcx>(
         let self_arg = &mut inputs_and_output[0];
         debug_assert!(tcx.generics_of(def_id).has_self && *self_arg == tcx.types.self_param);
         *self_arg = tcx.mk_mut_ptr(*self_arg);
-        sig.inputs_and_output = tcx.intern_type_list(&inputs_and_output);
+        sig.inputs_and_output = tcx.mk_type_list(&inputs_and_output);
     }
 
     let span = tcx.def_span(def_id);
@@ -693,7 +692,7 @@ fn build_call_shim<'tcx>(
 
         // `FnDef` call with optional receiver.
         CallKind::Direct(def_id) => {
-            let ty = tcx.type_of(def_id);
+            let ty = tcx.type_of(def_id).subst_identity();
             (
                 Operand::Constant(Box::new(Constant {
                     span,
@@ -798,7 +797,11 @@ pub fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> {
     let param_env = tcx.param_env(ctor_id);
 
     // Normalize the sig.
-    let sig = tcx.fn_sig(ctor_id).no_bound_vars().expect("LBR in ADT constructor signature");
+    let sig = tcx
+        .fn_sig(ctor_id)
+        .subst_identity()
+        .no_bound_vars()
+        .expect("LBR in ADT constructor signature");
     let sig = tcx.normalize_erasing_regions(param_env, sig);
 
     let ty::Adt(adt_def, substs) = sig.output().kind() else {
@@ -827,19 +830,23 @@ pub fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> {
     // return;
     debug!("build_ctor: variant_index={:?}", variant_index);
 
-    let statements = expand_aggregate(
-        Place::return_place(),
-        adt_def.variant(variant_index).fields.iter().enumerate().map(|(idx, field_def)| {
-            (Operand::Move(Place::from(Local::new(idx + 1))), field_def.ty(tcx, substs))
-        }),
-        AggregateKind::Adt(adt_def.did(), variant_index, substs, None, None),
+    let kind = AggregateKind::Adt(adt_def.did(), variant_index, substs, None, None);
+    let variant = adt_def.variant(variant_index);
+    let statement = Statement {
+        kind: StatementKind::Assign(Box::new((
+            Place::return_place(),
+            Rvalue::Aggregate(
+                Box::new(kind),
+                (0..variant.fields.len())
+                    .map(|idx| Operand::Move(Place::from(Local::new(idx + 1))))
+                    .collect(),
+            ),
+        ))),
         source_info,
-        tcx,
-    )
-    .collect();
+    };
 
     let start_block = BasicBlockData {
-        statements,
+        statements: vec![statement],
         terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }),
         is_cleanup: false,
     };
diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs
index 8f6abe7a9..9ef55c558 100644
--- a/compiler/rustc_mir_transform/src/simplify.rs
+++ b/compiler/rustc_mir_transform/src/simplify.rs
@@ -404,6 +404,18 @@ impl<'tcx> MirPass<'tcx> for SimplifyLocals {
     }
 }
 
+pub fn remove_unused_definitions<'tcx>(body: &mut Body<'tcx>) {
+    // First, we're going to get a count of *actual* uses for every `Local`.
+    let mut used_locals = UsedLocals::new(body);
+
+    // Next, we're going to remove any `Local` with zero actual uses. When we remove those
+    // `Locals`, we're also going to subtract any uses of other `Locals` from the `used_locals`
+    // count. For example, if we removed `_2 = discriminant(_1)`, then we'll subtract one from
+    // `use_counts[_1]`. That in turn might make `_1` unused, so we loop until we hit a
+    // fixedpoint where there are no more unused locals.
+    remove_unused_definitions_helper(&mut used_locals, body);
+}
+
 pub fn simplify_locals<'tcx>(body: &mut Body<'tcx>, tcx: TyCtxt<'tcx>) {
     // First, we're going to get a count of *actual* uses for every `Local`.
     let mut used_locals = UsedLocals::new(body);
@@ -413,7 +425,7 @@ pub fn simplify_locals<'tcx>(body: &mut Body<'tcx>, tcx: TyCtxt<'tcx>) {
     // count. For example, if we removed `_2 = discriminant(_1)`, then we'll subtract one from
     // `use_counts[_1]`. That in turn might make `_1` unused, so we loop until we hit a
     // fixedpoint where there are no more unused locals.
-    remove_unused_definitions(&mut used_locals, body);
+    remove_unused_definitions_helper(&mut used_locals, body);
 
     // Finally, we'll actually do the work of shrinking `body.local_decls` and remapping the `Local`s.
     let map = make_local_map(&mut body.local_decls, &used_locals);
@@ -484,7 +496,7 @@ impl UsedLocals {
         self.increment = false;
 
         // The location of the statement is irrelevant.
-        let location = Location { block: START_BLOCK, statement_index: 0 };
+        let location = Location::START;
         self.visit_statement(statement, location);
     }
 
@@ -517,7 +529,7 @@ impl<'tcx> Visitor<'tcx> for UsedLocals {
                 self.super_statement(statement, location);
             }
 
-            StatementKind::Nop => {}
+            StatementKind::ConstEvalCounter | StatementKind::Nop => {}
 
             StatementKind::StorageLive(_local) | StatementKind::StorageDead(_local) => {}
 
@@ -548,7 +560,7 @@ impl<'tcx> Visitor<'tcx> for UsedLocals {
 }
 
 /// Removes unused definitions. Updates the used locals to reflect the changes made.
-fn remove_unused_definitions(used_locals: &mut UsedLocals, body: &mut Body<'_>) {
+fn remove_unused_definitions_helper(used_locals: &mut UsedLocals, body: &mut Body<'_>) {
     // The use counts are updated as we remove the statements. A local might become unused
     // during the retain operation, leading to a temporary inconsistency (storage statements or
     // definitions referencing the local might remain). For correctness it is crucial that this
diff --git a/compiler/rustc_mir_transform/src/simplify_try.rs b/compiler/rustc_mir_transform/src/simplify_try.rs
deleted file mode 100644
index e4f3ace9a..000000000
--- a/compiler/rustc_mir_transform/src/simplify_try.rs
+++ /dev/null
@@ -1,822 +0,0 @@
-//! The general point of the optimizations provided here is to simplify something like:
-//!
-//! ```rust
-//! # fn foo<T, E>(x: Result<T, E>) -> Result<T, E> {
-//! match x {
-//!     Ok(x) => Ok(x),
-//!     Err(x) => Err(x)
-//! }
-//! # }
-//! ```
-//!
-//! into just `x`.
-
-use crate::{simplify, MirPass};
-use itertools::Itertools as _;
-use rustc_index::{bit_set::BitSet, vec::IndexVec};
-use rustc_middle::mir::visit::{NonUseContext, PlaceContext, Visitor};
-use rustc_middle::mir::*;
-use rustc_middle::ty::{self, List, Ty, TyCtxt};
-use rustc_target::abi::VariantIdx;
-use std::iter::{once, Enumerate, Peekable};
-use std::slice::Iter;
-
-/// Simplifies arms of form `Variant(x) => Variant(x)` to just a move.
-///
-/// This is done by transforming basic blocks where the statements match:
-///
-/// ```ignore (MIR)
-/// _LOCAL_TMP = ((_LOCAL_1 as Variant ).FIELD: TY );
-/// _TMP_2 = _LOCAL_TMP;
-/// ((_LOCAL_0 as Variant).FIELD: TY) = move _TMP_2;
-/// discriminant(_LOCAL_0) = VAR_IDX;
-/// ```
-///
-/// into:
-///
-/// ```ignore (MIR)
-/// _LOCAL_0 = move _LOCAL_1
-/// ```
-pub struct SimplifyArmIdentity;
-
-#[derive(Debug)]
-struct ArmIdentityInfo<'tcx> {
-    /// Storage location for the variant's field
-    local_temp_0: Local,
-    /// Storage location holding the variant being read from
-    local_1: Local,
-    /// The variant field being read from
-    vf_s0: VarField<'tcx>,
-    /// Index of the statement which loads the variant being read
-    get_variant_field_stmt: usize,
-
-    /// Tracks each assignment to a temporary of the variant's field
-    field_tmp_assignments: Vec<(Local, Local)>,
-
-    /// Storage location holding the variant's field that was read from
-    local_tmp_s1: Local,
-    /// Storage location holding the enum that we are writing to
-    local_0: Local,
-    /// The variant field being written to
-    vf_s1: VarField<'tcx>,
-
-    /// Storage location that the discriminant is being written to
-    set_discr_local: Local,
-    /// The variant being written
-    set_discr_var_idx: VariantIdx,
-
-    /// Index of the statement that should be overwritten as a move
-    stmt_to_overwrite: usize,
-    /// SourceInfo for the new move
-    source_info: SourceInfo,
-
-    /// Indices of matching Storage{Live,Dead} statements encountered.
-    /// (StorageLive index,, StorageDead index, Local)
-    storage_stmts: Vec<(usize, usize, Local)>,
-
-    /// The statements that should be removed (turned into nops)
-    stmts_to_remove: Vec<usize>,
-
-    /// Indices of debug variables that need to be adjusted to point to
-    // `{local_0}.{dbg_projection}`.
-    dbg_info_to_adjust: Vec<usize>,
-
-    /// The projection used to rewrite debug info.
-    dbg_projection: &'tcx List<PlaceElem<'tcx>>,
-}
-
-fn get_arm_identity_info<'a, 'tcx>(
-    stmts: &'a [Statement<'tcx>],
-    locals_count: usize,
-    debug_info: &'a [VarDebugInfo<'tcx>],
-) -> Option<ArmIdentityInfo<'tcx>> {
-    // This can't possibly match unless there are at least 3 statements in the block
-    // so fail fast on tiny blocks.
-    if stmts.len() < 3 {
-        return None;
-    }
-
-    let mut tmp_assigns = Vec::new();
-    let mut nop_stmts = Vec::new();
-    let mut storage_stmts = Vec::new();
-    let mut storage_live_stmts = Vec::new();
-    let mut storage_dead_stmts = Vec::new();
-
-    type StmtIter<'a, 'tcx> = Peekable<Enumerate<Iter<'a, Statement<'tcx>>>>;
-
-    fn is_storage_stmt(stmt: &Statement<'_>) -> bool {
-        matches!(stmt.kind, StatementKind::StorageLive(_) | StatementKind::StorageDead(_))
-    }
-
-    /// Eats consecutive Statements which match `test`, performing the specified `action` for each.
-    /// The iterator `stmt_iter` is not advanced if none were matched.
-    fn try_eat<'a, 'tcx>(
-        stmt_iter: &mut StmtIter<'a, 'tcx>,
-        test: impl Fn(&'a Statement<'tcx>) -> bool,
-        mut action: impl FnMut(usize, &'a Statement<'tcx>),
-    ) {
-        while stmt_iter.peek().map_or(false, |(_, stmt)| test(stmt)) {
-            let (idx, stmt) = stmt_iter.next().unwrap();
-
-            action(idx, stmt);
-        }
-    }
-
-    /// Eats consecutive `StorageLive` and `StorageDead` Statements.
-    /// The iterator `stmt_iter` is not advanced if none were found.
-    fn try_eat_storage_stmts(
-        stmt_iter: &mut StmtIter<'_, '_>,
-        storage_live_stmts: &mut Vec<(usize, Local)>,
-        storage_dead_stmts: &mut Vec<(usize, Local)>,
-    ) {
-        try_eat(stmt_iter, is_storage_stmt, |idx, stmt| {
-            if let StatementKind::StorageLive(l) = stmt.kind {
-                storage_live_stmts.push((idx, l));
-            } else if let StatementKind::StorageDead(l) = stmt.kind {
-                storage_dead_stmts.push((idx, l));
-            }
-        })
-    }
-
-    fn is_tmp_storage_stmt(stmt: &Statement<'_>) -> bool {
-        use rustc_middle::mir::StatementKind::Assign;
-        if let Assign(box (place, Rvalue::Use(Operand::Copy(p) | Operand::Move(p)))) = &stmt.kind {
-            place.as_local().is_some() && p.as_local().is_some()
-        } else {
-            false
-        }
-    }
-
-    /// Eats consecutive `Assign` Statements.
-    // The iterator `stmt_iter` is not advanced if none were found.
-    fn try_eat_assign_tmp_stmts(
-        stmt_iter: &mut StmtIter<'_, '_>,
-        tmp_assigns: &mut Vec<(Local, Local)>,
-        nop_stmts: &mut Vec<usize>,
-    ) {
-        try_eat(stmt_iter, is_tmp_storage_stmt, |idx, stmt| {
-            use rustc_middle::mir::StatementKind::Assign;
-            if let Assign(box (place, Rvalue::Use(Operand::Copy(p) | Operand::Move(p)))) =
-                &stmt.kind
-            {
-                tmp_assigns.push((place.as_local().unwrap(), p.as_local().unwrap()));
-                nop_stmts.push(idx);
-            }
-        })
-    }
-
-    fn find_storage_live_dead_stmts_for_local(
-        local: Local,
-        stmts: &[Statement<'_>],
-    ) -> Option<(usize, usize)> {
-        trace!("looking for {:?}", local);
-        let mut storage_live_stmt = None;
-        let mut storage_dead_stmt = None;
-        for (idx, stmt) in stmts.iter().enumerate() {
-            if stmt.kind == StatementKind::StorageLive(local) {
-                storage_live_stmt = Some(idx);
-            } else if stmt.kind == StatementKind::StorageDead(local) {
-                storage_dead_stmt = Some(idx);
-            }
-        }
-
-        Some((storage_live_stmt?, storage_dead_stmt.unwrap_or(usize::MAX)))
-    }
-
-    // Try to match the expected MIR structure with the basic block we're processing.
-    // We want to see something that looks like:
-    // ```
-    // (StorageLive(_) | StorageDead(_));*
-    // _LOCAL_INTO = ((_LOCAL_FROM as Variant).FIELD: TY);
-    // (StorageLive(_) | StorageDead(_));*
-    // (tmp_n+1 = tmp_n);*
-    // (StorageLive(_) | StorageDead(_));*
-    // (tmp_n+1 = tmp_n);*
-    // ((LOCAL_FROM as Variant).FIELD: TY) = move tmp;
-    // discriminant(LOCAL_FROM) = VariantIdx;
-    // (StorageLive(_) | StorageDead(_));*
-    // ```
-    let mut stmt_iter = stmts.iter().enumerate().peekable();
-
-    try_eat_storage_stmts(&mut stmt_iter, &mut storage_live_stmts, &mut storage_dead_stmts);
-
-    let (get_variant_field_stmt, stmt) = stmt_iter.next()?;
-    let (local_tmp_s0, local_1, vf_s0, dbg_projection) = match_get_variant_field(stmt)?;
-
-    try_eat_storage_stmts(&mut stmt_iter, &mut storage_live_stmts, &mut storage_dead_stmts);
-
-    try_eat_assign_tmp_stmts(&mut stmt_iter, &mut tmp_assigns, &mut nop_stmts);
-
-    try_eat_storage_stmts(&mut stmt_iter, &mut storage_live_stmts, &mut storage_dead_stmts);
-
-    try_eat_assign_tmp_stmts(&mut stmt_iter, &mut tmp_assigns, &mut nop_stmts);
-
-    let (idx, stmt) = stmt_iter.next()?;
-    let (local_tmp_s1, local_0, vf_s1) = match_set_variant_field(stmt)?;
-    nop_stmts.push(idx);
-
-    let (idx, stmt) = stmt_iter.next()?;
-    let (set_discr_local, set_discr_var_idx) = match_set_discr(stmt)?;
-    let discr_stmt_source_info = stmt.source_info;
-    nop_stmts.push(idx);
-
-    try_eat_storage_stmts(&mut stmt_iter, &mut storage_live_stmts, &mut storage_dead_stmts);
-
-    for (live_idx, live_local) in storage_live_stmts {
-        if let Some(i) = storage_dead_stmts.iter().rposition(|(_, l)| *l == live_local) {
-            let (dead_idx, _) = storage_dead_stmts.swap_remove(i);
-            storage_stmts.push((live_idx, dead_idx, live_local));
-
-            if live_local == local_tmp_s0 {
-                nop_stmts.push(get_variant_field_stmt);
-            }
-        }
-    }
-    // We sort primitive usize here so we can use unstable sort
-    nop_stmts.sort_unstable();
-
-    // Use one of the statements we're going to discard between the point
-    // where the storage location for the variant field becomes live and
-    // is killed.
-    let (live_idx, dead_idx) = find_storage_live_dead_stmts_for_local(local_tmp_s0, stmts)?;
-    let stmt_to_overwrite =
-        nop_stmts.iter().find(|stmt_idx| live_idx < **stmt_idx && **stmt_idx < dead_idx);
-
-    let mut tmp_assigned_vars = BitSet::new_empty(locals_count);
-    for (l, r) in &tmp_assigns {
-        tmp_assigned_vars.insert(*l);
-        tmp_assigned_vars.insert(*r);
-    }
-
-    let dbg_info_to_adjust: Vec<_> = debug_info
-        .iter()
-        .enumerate()
-        .filter_map(|(i, var_info)| {
-            if let VarDebugInfoContents::Place(p) = var_info.value {
-                if tmp_assigned_vars.contains(p.local) {
-                    return Some(i);
-                }
-            }
-
-            None
-        })
-        .collect();
-
-    Some(ArmIdentityInfo {
-        local_temp_0: local_tmp_s0,
-        local_1,
-        vf_s0,
-        get_variant_field_stmt,
-        field_tmp_assignments: tmp_assigns,
-        local_tmp_s1,
-        local_0,
-        vf_s1,
-        set_discr_local,
-        set_discr_var_idx,
-        stmt_to_overwrite: *stmt_to_overwrite?,
-        source_info: discr_stmt_source_info,
-        storage_stmts,
-        stmts_to_remove: nop_stmts,
-        dbg_info_to_adjust,
-        dbg_projection,
-    })
-}
-
-fn optimization_applies<'tcx>(
-    opt_info: &ArmIdentityInfo<'tcx>,
-    local_decls: &IndexVec<Local, LocalDecl<'tcx>>,
-    local_uses: &IndexVec<Local, usize>,
-    var_debug_info: &[VarDebugInfo<'tcx>],
-) -> bool {
-    trace!("testing if optimization applies...");
-
-    // FIXME(wesleywiser): possibly relax this restriction?
-    if opt_info.local_0 == opt_info.local_1 {
-        trace!("NO: moving into ourselves");
-        return false;
-    } else if opt_info.vf_s0 != opt_info.vf_s1 {
-        trace!("NO: the field-and-variant information do not match");
-        return false;
-    } else if local_decls[opt_info.local_0].ty != local_decls[opt_info.local_1].ty {
-        // FIXME(Centril,oli-obk): possibly relax to same layout?
-        trace!("NO: source and target locals have different types");
-        return false;
-    } else if (opt_info.local_0, opt_info.vf_s0.var_idx)
-        != (opt_info.set_discr_local, opt_info.set_discr_var_idx)
-    {
-        trace!("NO: the discriminants do not match");
-        return false;
-    }
-
-    // Verify the assignment chain consists of the form b = a; c = b; d = c; etc...
-    if opt_info.field_tmp_assignments.is_empty() {
-        trace!("NO: no assignments found");
-        return false;
-    }
-    let mut last_assigned_to = opt_info.field_tmp_assignments[0].1;
-    let source_local = last_assigned_to;
-    for (l, r) in &opt_info.field_tmp_assignments {
-        if *r != last_assigned_to {
-            trace!("NO: found unexpected assignment {:?} = {:?}", l, r);
-            return false;
-        }
-
-        last_assigned_to = *l;
-    }
-
-    // Check that the first and last used locals are only used twice
-    // since they are of the form:
-    //
-    // ```
-    // _first = ((_x as Variant).n: ty);
-    // _n = _first;
-    // ...
-    // ((_y as Variant).n: ty) = _n;
-    // discriminant(_y) = z;
-    // ```
-    for (l, r) in &opt_info.field_tmp_assignments {
-        if local_uses[*l] != 2 {
-            warn!("NO: FAILED assignment chain local {:?} was used more than twice", l);
-            return false;
-        } else if local_uses[*r] != 2 {
-            warn!("NO: FAILED assignment chain local {:?} was used more than twice", r);
-            return false;
-        }
-    }
-
-    // Check that debug info only points to full Locals and not projections.
-    for dbg_idx in &opt_info.dbg_info_to_adjust {
-        let dbg_info = &var_debug_info[*dbg_idx];
-        if let VarDebugInfoContents::Place(p) = dbg_info.value {
-            if !p.projection.is_empty() {
-                trace!("NO: debug info for {:?} had a projection {:?}", dbg_info.name, p);
-                return false;
-            }
-        }
-    }
-
-    if source_local != opt_info.local_temp_0 {
-        trace!(
-            "NO: start of assignment chain does not match enum variant temp: {:?} != {:?}",
-            source_local,
-            opt_info.local_temp_0
-        );
-        return false;
-    } else if last_assigned_to != opt_info.local_tmp_s1 {
-        trace!(
-            "NO: end of assignment chain does not match written enum temp: {:?} != {:?}",
-            last_assigned_to,
-            opt_info.local_tmp_s1
-        );
-        return false;
-    }
-
-    trace!("SUCCESS: optimization applies!");
-    true
-}
-
-impl<'tcx> MirPass<'tcx> for SimplifyArmIdentity {
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        // FIXME(77359): This optimization can result in unsoundness.
-        if !tcx.sess.opts.unstable_opts.unsound_mir_opts {
-            return;
-        }
-
-        let source = body.source;
-        trace!("running SimplifyArmIdentity on {:?}", source);
-
-        let local_uses = LocalUseCounter::get_local_uses(body);
-        for bb in body.basic_blocks.as_mut() {
-            if let Some(opt_info) =
-                get_arm_identity_info(&bb.statements, body.local_decls.len(), &body.var_debug_info)
-            {
-                trace!("got opt_info = {:#?}", opt_info);
-                if !optimization_applies(
-                    &opt_info,
-                    &body.local_decls,
-                    &local_uses,
-                    &body.var_debug_info,
-                ) {
-                    debug!("optimization skipped for {:?}", source);
-                    continue;
-                }
-
-                // Also remove unused Storage{Live,Dead} statements which correspond
-                // to temps used previously.
-                for (live_idx, dead_idx, local) in &opt_info.storage_stmts {
-                    // The temporary that we've read the variant field into is scoped to this block,
-                    // so we can remove the assignment.
-                    if *local == opt_info.local_temp_0 {
-                        bb.statements[opt_info.get_variant_field_stmt].make_nop();
-                    }
-
-                    for (left, right) in &opt_info.field_tmp_assignments {
-                        if local == left || local == right {
-                            bb.statements[*live_idx].make_nop();
-                            bb.statements[*dead_idx].make_nop();
-                        }
-                    }
-                }
-
-                // Right shape; transform
-                for stmt_idx in opt_info.stmts_to_remove {
-                    bb.statements[stmt_idx].make_nop();
-                }
-
-                let stmt = &mut bb.statements[opt_info.stmt_to_overwrite];
-                stmt.source_info = opt_info.source_info;
-                stmt.kind = StatementKind::Assign(Box::new((
-                    opt_info.local_0.into(),
-                    Rvalue::Use(Operand::Move(opt_info.local_1.into())),
-                )));
-
-                bb.statements.retain(|stmt| stmt.kind != StatementKind::Nop);
-
-                // Fix the debug info to point to the right local
-                for dbg_index in opt_info.dbg_info_to_adjust {
-                    let dbg_info = &mut body.var_debug_info[dbg_index];
-                    assert!(
-                        matches!(dbg_info.value, VarDebugInfoContents::Place(_)),
-                        "value was not a Place"
-                    );
-                    if let VarDebugInfoContents::Place(p) = &mut dbg_info.value {
-                        assert!(p.projection.is_empty());
-                        p.local = opt_info.local_0;
-                        p.projection = opt_info.dbg_projection;
-                    }
-                }
-
-                trace!("block is now {:?}", bb.statements);
-            }
-        }
-    }
-}
-
-struct LocalUseCounter {
-    local_uses: IndexVec<Local, usize>,
-}
-
-impl LocalUseCounter {
-    fn get_local_uses(body: &Body<'_>) -> IndexVec<Local, usize> {
-        let mut counter = LocalUseCounter { local_uses: IndexVec::from_elem(0, &body.local_decls) };
-        counter.visit_body(body);
-        counter.local_uses
-    }
-}
-
-impl Visitor<'_> for LocalUseCounter {
-    fn visit_local(&mut self, local: Local, context: PlaceContext, _location: Location) {
-        if context.is_storage_marker()
-            || context == PlaceContext::NonUse(NonUseContext::VarDebugInfo)
-        {
-            return;
-        }
-
-        self.local_uses[local] += 1;
-    }
-}
-
-/// Match on:
-/// ```ignore (MIR)
-/// _LOCAL_INTO = ((_LOCAL_FROM as Variant).FIELD: TY);
-/// ```
-fn match_get_variant_field<'tcx>(
-    stmt: &Statement<'tcx>,
-) -> Option<(Local, Local, VarField<'tcx>, &'tcx List<PlaceElem<'tcx>>)> {
-    match &stmt.kind {
-        StatementKind::Assign(box (
-            place_into,
-            Rvalue::Use(Operand::Copy(pf) | Operand::Move(pf)),
-        )) => {
-            let local_into = place_into.as_local()?;
-            let (local_from, vf) = match_variant_field_place(*pf)?;
-            Some((local_into, local_from, vf, pf.projection))
-        }
-        _ => None,
-    }
-}
-
-/// Match on:
-/// ```ignore (MIR)
-/// ((_LOCAL_FROM as Variant).FIELD: TY) = move _LOCAL_INTO;
-/// ```
-fn match_set_variant_field<'tcx>(stmt: &Statement<'tcx>) -> Option<(Local, Local, VarField<'tcx>)> {
-    match &stmt.kind {
-        StatementKind::Assign(box (place_from, Rvalue::Use(Operand::Move(place_into)))) => {
-            let local_into = place_into.as_local()?;
-            let (local_from, vf) = match_variant_field_place(*place_from)?;
-            Some((local_into, local_from, vf))
-        }
-        _ => None,
-    }
-}
-
-/// Match on:
-/// ```ignore (MIR)
-/// discriminant(_LOCAL_TO_SET) = VAR_IDX;
-/// ```
-fn match_set_discr(stmt: &Statement<'_>) -> Option<(Local, VariantIdx)> {
-    match &stmt.kind {
-        StatementKind::SetDiscriminant { place, variant_index } => {
-            Some((place.as_local()?, *variant_index))
-        }
-        _ => None,
-    }
-}
-
-#[derive(PartialEq, Debug)]
-struct VarField<'tcx> {
-    field: Field,
-    field_ty: Ty<'tcx>,
-    var_idx: VariantIdx,
-}
-
-/// Match on `((_LOCAL as Variant).FIELD: TY)`.
-fn match_variant_field_place(place: Place<'_>) -> Option<(Local, VarField<'_>)> {
-    match place.as_ref() {
-        PlaceRef {
-            local,
-            projection: &[ProjectionElem::Downcast(_, var_idx), ProjectionElem::Field(field, ty)],
-        } => Some((local, VarField { field, field_ty: ty, var_idx })),
-        _ => None,
-    }
-}
-
-/// Simplifies `SwitchInt(_) -> [targets]`,
-/// where all the `targets` have the same form,
-/// into `goto -> target_first`.
-pub struct SimplifyBranchSame;
-
-impl<'tcx> MirPass<'tcx> for SimplifyBranchSame {
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        // This optimization is disabled by default for now due to
-        // soundness concerns; see issue #89485 and PR #89489.
-        if !tcx.sess.opts.unstable_opts.unsound_mir_opts {
-            return;
-        }
-
-        trace!("Running SimplifyBranchSame on {:?}", body.source);
-        let finder = SimplifyBranchSameOptimizationFinder { body, tcx };
-        let opts = finder.find();
-
-        let did_remove_blocks = opts.len() > 0;
-        for opt in opts.iter() {
-            trace!("SUCCESS: Applying optimization {:?}", opt);
-            // Replace `SwitchInt(..) -> [bb_first, ..];` with a `goto -> bb_first;`.
-            body.basic_blocks_mut()[opt.bb_to_opt_terminator].terminator_mut().kind =
-                TerminatorKind::Goto { target: opt.bb_to_goto };
-        }
-
-        if did_remove_blocks {
-            // We have dead blocks now, so remove those.
-            simplify::remove_dead_blocks(tcx, body);
-        }
-    }
-}
-
-#[derive(Debug)]
-struct SimplifyBranchSameOptimization {
-    /// All basic blocks are equal so go to this one
-    bb_to_goto: BasicBlock,
-    /// Basic block where the terminator can be simplified to a goto
-    bb_to_opt_terminator: BasicBlock,
-}
-
-struct SwitchTargetAndValue {
-    target: BasicBlock,
-    // None in case of the `otherwise` case
-    value: Option<u128>,
-}
-
-struct SimplifyBranchSameOptimizationFinder<'a, 'tcx> {
-    body: &'a Body<'tcx>,
-    tcx: TyCtxt<'tcx>,
-}
-
-impl<'tcx> SimplifyBranchSameOptimizationFinder<'_, 'tcx> {
-    fn find(&self) -> Vec<SimplifyBranchSameOptimization> {
-        self.body
-            .basic_blocks
-            .iter_enumerated()
-            .filter_map(|(bb_idx, bb)| {
-                let (discr_switched_on, targets_and_values) = match &bb.terminator().kind {
-                    TerminatorKind::SwitchInt { targets, discr, .. } => {
-                        let targets_and_values: Vec<_> = targets.iter()
-                            .map(|(val, target)| SwitchTargetAndValue { target, value: Some(val) })
-                            .chain(once(SwitchTargetAndValue { target: targets.otherwise(), value: None }))
-                            .collect();
-                        (discr, targets_and_values)
-                    },
-                    _ => return None,
-                };
-
-                // find the adt that has its discriminant read
-                // assuming this must be the last statement of the block
-                let adt_matched_on = match &bb.statements.last()?.kind {
-                    StatementKind::Assign(box (place, rhs))
-                        if Some(*place) == discr_switched_on.place() =>
-                    {
-                        match rhs {
-                            Rvalue::Discriminant(adt_place) if adt_place.ty(self.body, self.tcx).ty.is_enum() => adt_place,
-                            _ => {
-                                trace!("NO: expected a discriminant read of an enum instead of: {:?}", rhs);
-                                return None;
-                            }
-                        }
-                    }
-                    other => {
-                        trace!("NO: expected an assignment of a discriminant read to a place. Found: {:?}", other);
-                        return None
-                    },
-                };
-
-                let mut iter_bbs_reachable = targets_and_values
-                    .iter()
-                    .map(|target_and_value| (target_and_value, &self.body.basic_blocks[target_and_value.target]))
-                    .filter(|(_, bb)| {
-                        // Reaching `unreachable` is UB so assume it doesn't happen.
-                        bb.terminator().kind != TerminatorKind::Unreachable
-                    })
-                    .peekable();
-
-                let bb_first = iter_bbs_reachable.peek().map_or(&targets_and_values[0], |(idx, _)| *idx);
-                let mut all_successors_equivalent = StatementEquality::TrivialEqual;
-
-                // All successor basic blocks must be equal or contain statements that are pairwise considered equal.
-                for ((target_and_value_l,bb_l), (target_and_value_r,bb_r)) in iter_bbs_reachable.tuple_windows() {
-                    let trivial_checks = bb_l.is_cleanup == bb_r.is_cleanup
-                                            && bb_l.terminator().kind == bb_r.terminator().kind
-                                            && bb_l.statements.len() == bb_r.statements.len();
-                    let statement_check = || {
-                        bb_l.statements.iter().zip(&bb_r.statements).try_fold(StatementEquality::TrivialEqual, |acc,(l,r)| {
-                            let stmt_equality = self.statement_equality(*adt_matched_on, &l, target_and_value_l, &r, target_and_value_r);
-                            if matches!(stmt_equality, StatementEquality::NotEqual) {
-                                // short circuit
-                                None
-                            } else {
-                                Some(acc.combine(&stmt_equality))
-                            }
-                        })
-                        .unwrap_or(StatementEquality::NotEqual)
-                    };
-                    if !trivial_checks {
-                        all_successors_equivalent = StatementEquality::NotEqual;
-                        break;
-                    }
-                    all_successors_equivalent = all_successors_equivalent.combine(&statement_check());
-                };
-
-                match all_successors_equivalent{
-                    StatementEquality::TrivialEqual => {
-                        // statements are trivially equal, so just take first
-                        trace!("Statements are trivially equal");
-                        Some(SimplifyBranchSameOptimization {
-                            bb_to_goto: bb_first.target,
-                            bb_to_opt_terminator: bb_idx,
-                        })
-                    }
-                    StatementEquality::ConsideredEqual(bb_to_choose) => {
-                        trace!("Statements are considered equal");
-                        Some(SimplifyBranchSameOptimization {
-                            bb_to_goto: bb_to_choose,
-                            bb_to_opt_terminator: bb_idx,
-                        })
-                    }
-                    StatementEquality::NotEqual => {
-                        trace!("NO: not all successors of basic block {:?} were equivalent", bb_idx);
-                        None
-                    }
-                }
-            })
-            .collect()
-    }
-
-    /// Tests if two statements can be considered equal
-    ///
-    /// Statements can be trivially equal if the kinds match.
-    /// But they can also be considered equal in the following case A:
-    /// ```ignore (MIR)
-    /// discriminant(_0) = 0;   // bb1
-    /// _0 = move _1;           // bb2
-    /// ```
-    /// In this case the two statements are equal iff
-    /// - `_0` is an enum where the variant index 0 is fieldless, and
-    /// -  bb1 was targeted by a switch where the discriminant of `_1` was switched on
-    fn statement_equality(
-        &self,
-        adt_matched_on: Place<'tcx>,
-        x: &Statement<'tcx>,
-        x_target_and_value: &SwitchTargetAndValue,
-        y: &Statement<'tcx>,
-        y_target_and_value: &SwitchTargetAndValue,
-    ) -> StatementEquality {
-        let helper = |rhs: &Rvalue<'tcx>,
-                      place: &Place<'tcx>,
-                      variant_index: VariantIdx,
-                      switch_value: u128,
-                      side_to_choose| {
-            let place_type = place.ty(self.body, self.tcx).ty;
-            let adt = match *place_type.kind() {
-                ty::Adt(adt, _) if adt.is_enum() => adt,
-                _ => return StatementEquality::NotEqual,
-            };
-            // We need to make sure that the switch value that targets the bb with
-            // SetDiscriminant is the same as the variant discriminant.
-            let variant_discr = adt.discriminant_for_variant(self.tcx, variant_index).val;
-            if variant_discr != switch_value {
-                trace!(
-                    "NO: variant discriminant {} does not equal switch value {}",
-                    variant_discr,
-                    switch_value
-                );
-                return StatementEquality::NotEqual;
-            }
-            let variant_is_fieldless = adt.variant(variant_index).fields.is_empty();
-            if !variant_is_fieldless {
-                trace!("NO: variant {:?} was not fieldless", variant_index);
-                return StatementEquality::NotEqual;
-            }
-
-            match rhs {
-                Rvalue::Use(operand) if operand.place() == Some(adt_matched_on) => {
-                    StatementEquality::ConsideredEqual(side_to_choose)
-                }
-                _ => {
-                    trace!(
-                        "NO: RHS of assignment was {:?}, but expected it to match the adt being matched on in the switch, which is {:?}",
-                        rhs,
-                        adt_matched_on
-                    );
-                    StatementEquality::NotEqual
-                }
-            }
-        };
-        match (&x.kind, &y.kind) {
-            // trivial case
-            (x, y) if x == y => StatementEquality::TrivialEqual,
-
-            // check for case A
-            (
-                StatementKind::Assign(box (_, rhs)),
-                &StatementKind::SetDiscriminant { ref place, variant_index },
-            ) if y_target_and_value.value.is_some() => {
-                // choose basic block of x, as that has the assign
-                helper(
-                    rhs,
-                    place,
-                    variant_index,
-                    y_target_and_value.value.unwrap(),
-                    x_target_and_value.target,
-                )
-            }
-            (
-                &StatementKind::SetDiscriminant { ref place, variant_index },
-                &StatementKind::Assign(box (_, ref rhs)),
-            ) if x_target_and_value.value.is_some() => {
-                // choose basic block of y, as that has the assign
-                helper(
-                    rhs,
-                    place,
-                    variant_index,
-                    x_target_and_value.value.unwrap(),
-                    y_target_and_value.target,
-                )
-            }
-            _ => {
-                trace!("NO: statements `{:?}` and `{:?}` not considered equal", x, y);
-                StatementEquality::NotEqual
-            }
-        }
-    }
-}
-
-#[derive(Copy, Clone, Eq, PartialEq)]
-enum StatementEquality {
-    /// The two statements are trivially equal; same kind
-    TrivialEqual,
-    /// The two statements are considered equal, but may be of different kinds. The BasicBlock field is the basic block to jump to when performing the branch-same optimization.
-    /// For example, `_0 = _1` and `discriminant(_0) = discriminant(0)` are considered equal if 0 is a fieldless variant of an enum. But we don't want to jump to the basic block with the SetDiscriminant, as that is not legal if _1 is not the 0 variant index
-    ConsideredEqual(BasicBlock),
-    /// The two statements are not equal
-    NotEqual,
-}
-
-impl StatementEquality {
-    fn combine(&self, other: &StatementEquality) -> StatementEquality {
-        use StatementEquality::*;
-        match (self, other) {
-            (TrivialEqual, TrivialEqual) => TrivialEqual,
-            (TrivialEqual, ConsideredEqual(b)) | (ConsideredEqual(b), TrivialEqual) => {
-                ConsideredEqual(*b)
-            }
-            (ConsideredEqual(b1), ConsideredEqual(b2)) => {
-                if b1 == b2 {
-                    ConsideredEqual(*b1)
-                } else {
-                    NotEqual
-                }
-            }
-            (_, NotEqual) | (NotEqual, _) => NotEqual,
-        }
-    }
-}
diff --git a/compiler/rustc_mir_transform/src/sroa.rs b/compiler/rustc_mir_transform/src/sroa.rs
index 42124f5a4..13168e9a2 100644
--- a/compiler/rustc_mir_transform/src/sroa.rs
+++ b/compiler/rustc_mir_transform/src/sroa.rs
@@ -1,10 +1,11 @@
 use crate::MirPass;
-use rustc_data_structures::fx::{FxIndexMap, IndexEntry};
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::{BitSet, GrowableBitSet};
 use rustc_index::vec::IndexVec;
+use rustc_middle::mir::patch::MirPatch;
 use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
-use rustc_middle::ty::TyCtxt;
+use rustc_middle::ty::{Ty, TyCtxt};
+use rustc_mir_dataflow::value_analysis::{excluded_locals, iter_fields};
 
 pub struct ScalarReplacementOfAggregates;
 
@@ -13,27 +14,43 @@ impl<'tcx> MirPass<'tcx> for ScalarReplacementOfAggregates {
         sess.mir_opt_level() >= 3
     }
 
+    #[instrument(level = "debug", skip(self, tcx, body))]
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        let escaping = escaping_locals(&*body);
-        debug!(?escaping);
-        let replacements = compute_flattening(tcx, body, escaping);
-        debug!(?replacements);
-        replace_flattened_locals(tcx, body, replacements);
+        debug!(def_id = ?body.source.def_id());
+        let mut excluded = excluded_locals(body);
+        loop {
+            debug!(?excluded);
+            let escaping = escaping_locals(&excluded, body);
+            debug!(?escaping);
+            let replacements = compute_flattening(tcx, body, escaping);
+            debug!(?replacements);
+            let all_dead_locals = replace_flattened_locals(tcx, body, replacements);
+            if !all_dead_locals.is_empty() {
+                excluded.union(&all_dead_locals);
+                excluded = {
+                    let mut growable = GrowableBitSet::from(excluded);
+                    growable.ensure(body.local_decls.len());
+                    growable.into()
+                };
+            } else {
+                break;
+            }
+        }
     }
 }
 
 /// Identify all locals that are not eligible for SROA.
 ///
 /// There are 3 cases:
-/// - the aggegated local is used or passed to other code (function parameters and arguments);
+/// - the aggregated local is used or passed to other code (function parameters and arguments);
 /// - the locals is a union or an enum;
 /// - the local's address is taken, and thus the relative addresses of the fields are observable to
 ///   client code.
-fn escaping_locals(body: &Body<'_>) -> BitSet<Local> {
+fn escaping_locals(excluded: &BitSet<Local>, body: &Body<'_>) -> BitSet<Local> {
     let mut set = BitSet::new_empty(body.local_decls.len());
     set.insert_range(RETURN_PLACE..=Local::from_usize(body.arg_count));
     for (local, decl) in body.local_decls().iter_enumerated() {
-        if decl.ty.is_union() || decl.ty.is_enum() {
+        if decl.ty.is_union() || decl.ty.is_enum() || excluded.contains(local) {
             set.insert(local);
         }
     }
@@ -58,41 +75,33 @@ fn escaping_locals(body: &Body<'_>) -> BitSet<Local> {
             self.super_place(place, context, location);
         }
 
-        fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, location: Location) {
-            if let Rvalue::AddressOf(.., place) | Rvalue::Ref(.., place) = rvalue {
-                if !place.is_indirect() {
-                    // Raw pointers may be used to access anything inside the enclosing place.
-                    self.set.insert(place.local);
-                    return;
+        fn visit_assign(
+            &mut self,
+            lvalue: &Place<'tcx>,
+            rvalue: &Rvalue<'tcx>,
+            location: Location,
+        ) {
+            if lvalue.as_local().is_some() {
+                match rvalue {
+                    // Aggregate assignments are expanded in run_pass.
+                    Rvalue::Aggregate(..) | Rvalue::Use(..) => {
+                        self.visit_rvalue(rvalue, location);
+                        return;
+                    }
+                    _ => {}
                 }
             }
-            self.super_rvalue(rvalue, location)
+            self.super_assign(lvalue, rvalue, location)
         }
 
         fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) {
-            if let StatementKind::StorageLive(..)
-            | StatementKind::StorageDead(..)
-            | StatementKind::Deinit(..) = statement.kind
-            {
+            match statement.kind {
                 // Storage statements are expanded in run_pass.
-                return;
+                StatementKind::StorageLive(..)
+                | StatementKind::StorageDead(..)
+                | StatementKind::Deinit(..) => return,
+                _ => self.super_statement(statement, location),
             }
-            self.super_statement(statement, location)
-        }
-
-        fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
-            // Drop implicitly calls `drop_in_place`, which takes a `&mut`.
-            // This implies that `Drop` implicitly takes the address of the place.
-            if let TerminatorKind::Drop { place, .. }
-            | TerminatorKind::DropAndReplace { place, .. } = terminator.kind
-            {
-                if !place.is_indirect() {
-                    // Raw pointers may be used to access anything inside the enclosing place.
-                    self.set.insert(place.local);
-                    return;
-                }
-            }
-            self.super_terminator(terminator, location);
         }
 
         // We ignore anything that happens in debuginfo, since we expand it using
@@ -103,7 +112,30 @@ fn escaping_locals(body: &Body<'_>) -> BitSet<Local> {
 
 #[derive(Default, Debug)]
 struct ReplacementMap<'tcx> {
-    fields: FxIndexMap<PlaceRef<'tcx>, Local>,
+    /// Pre-computed list of all "new" locals for each "old" local. This is used to expand storage
+    /// and deinit statement and debuginfo.
+    fragments: IndexVec<Local, Option<IndexVec<Field, Option<(Ty<'tcx>, Local)>>>>,
+}
+
+impl<'tcx> ReplacementMap<'tcx> {
+    fn replace_place(&self, tcx: TyCtxt<'tcx>, place: PlaceRef<'tcx>) -> Option<Place<'tcx>> {
+        let &[PlaceElem::Field(f, _), ref rest @ ..] = place.projection else { return None; };
+        let fields = self.fragments[place.local].as_ref()?;
+        let (_, new_local) = fields[f]?;
+        Some(Place { local: new_local, projection: tcx.mk_place_elems(&rest) })
+    }
+
+    fn place_fragments(
+        &self,
+        place: Place<'tcx>,
+    ) -> Option<impl Iterator<Item = (Field, Ty<'tcx>, Local)> + '_> {
+        let local = place.as_local()?;
+        let fields = self.fragments[local].as_ref()?;
+        Some(fields.iter_enumerated().filter_map(|(field, &opt_ty_local)| {
+            let (ty, local) = opt_ty_local?;
+            Some((field, ty, local))
+        }))
+    }
 }
 
 /// Compute the replacement of flattened places into locals.
@@ -115,53 +147,25 @@ fn compute_flattening<'tcx>(
     body: &mut Body<'tcx>,
     escaping: BitSet<Local>,
 ) -> ReplacementMap<'tcx> {
-    let mut visitor = PreFlattenVisitor {
-        tcx,
-        escaping,
-        local_decls: &mut body.local_decls,
-        map: Default::default(),
-    };
-    for (block, bbdata) in body.basic_blocks.iter_enumerated() {
-        visitor.visit_basic_block_data(block, bbdata);
-    }
-    return visitor.map;
-
-    struct PreFlattenVisitor<'tcx, 'll> {
-        tcx: TyCtxt<'tcx>,
-        local_decls: &'ll mut LocalDecls<'tcx>,
-        escaping: BitSet<Local>,
-        map: ReplacementMap<'tcx>,
-    }
-
-    impl<'tcx, 'll> PreFlattenVisitor<'tcx, 'll> {
-        fn create_place(&mut self, place: PlaceRef<'tcx>) {
-            if self.escaping.contains(place.local) {
-                return;
-            }
+    let mut fragments = IndexVec::from_elem(None, &body.local_decls);
 
-            match self.map.fields.entry(place) {
-                IndexEntry::Occupied(_) => {}
-                IndexEntry::Vacant(v) => {
-                    let ty = place.ty(&*self.local_decls, self.tcx).ty;
-                    let local = self.local_decls.push(LocalDecl {
-                        ty,
-                        user_ty: None,
-                        ..self.local_decls[place.local].clone()
-                    });
-                    v.insert(local);
-                }
-            }
-        }
-    }
-
-    impl<'tcx, 'll> Visitor<'tcx> for PreFlattenVisitor<'tcx, 'll> {
-        fn visit_place(&mut self, place: &Place<'tcx>, _: PlaceContext, _: Location) {
-            if let &[PlaceElem::Field(..), ..] = &place.projection[..] {
-                let pr = PlaceRef { local: place.local, projection: &place.projection[..1] };
-                self.create_place(pr)
-            }
+    for local in body.local_decls.indices() {
+        if escaping.contains(local) {
+            continue;
         }
+        let decl = body.local_decls[local].clone();
+        let ty = decl.ty;
+        iter_fields(ty, tcx, |variant, field, field_ty| {
+            if variant.is_some() {
+                // Downcasts are currently not supported.
+                return;
+            };
+            let new_local =
+                body.local_decls.push(LocalDecl { ty: field_ty, user_ty: None, ..decl.clone() });
+            fragments.get_or_insert_with(local, IndexVec::new).insert(field, (field_ty, new_local));
+        });
     }
+    ReplacementMap { fragments }
 }
 
 /// Perform the replacement computed by `compute_flattening`.
@@ -169,29 +173,24 @@ fn replace_flattened_locals<'tcx>(
     tcx: TyCtxt<'tcx>,
     body: &mut Body<'tcx>,
     replacements: ReplacementMap<'tcx>,
-) {
-    let mut all_dead_locals = BitSet::new_empty(body.local_decls.len());
-    for p in replacements.fields.keys() {
-        all_dead_locals.insert(p.local);
+) -> BitSet<Local> {
+    let mut all_dead_locals = BitSet::new_empty(replacements.fragments.len());
+    for (local, replacements) in replacements.fragments.iter_enumerated() {
+        if replacements.is_some() {
+            all_dead_locals.insert(local);
+        }
     }
     debug!(?all_dead_locals);
     if all_dead_locals.is_empty() {
-        return;
+        return all_dead_locals;
     }
 
-    let mut fragments = IndexVec::new();
-    for (k, v) in &replacements.fields {
-        fragments.ensure_contains_elem(k.local, || Vec::new());
-        fragments[k.local].push((k.projection, *v));
-    }
-    debug!(?fragments);
-
     let mut visitor = ReplacementVisitor {
         tcx,
         local_decls: &body.local_decls,
-        replacements,
+        replacements: &replacements,
         all_dead_locals,
-        fragments,
+        patch: MirPatch::new(body),
     };
     for (bb, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() {
         visitor.visit_basic_block_data(bb, data);
@@ -205,6 +204,9 @@ fn replace_flattened_locals<'tcx>(
     for var_debug_info in &mut body.var_debug_info {
         visitor.visit_var_debug_info(var_debug_info);
     }
+    let ReplacementVisitor { patch, all_dead_locals, .. } = visitor;
+    patch.apply(body);
+    all_dead_locals
 }
 
 struct ReplacementVisitor<'tcx, 'll> {
@@ -212,40 +214,23 @@ struct ReplacementVisitor<'tcx, 'll> {
     /// This is only used to compute the type for `VarDebugInfoContents::Composite`.
     local_decls: &'ll LocalDecls<'tcx>,
     /// Work to do.
-    replacements: ReplacementMap<'tcx>,
+    replacements: &'ll ReplacementMap<'tcx>,
     /// This is used to check that we are not leaving references to replaced locals behind.
     all_dead_locals: BitSet<Local>,
-    /// Pre-computed list of all "new" locals for each "old" local. This is used to expand storage
-    /// and deinit statement and debuginfo.
-    fragments: IndexVec<Local, Vec<(&'tcx [PlaceElem<'tcx>], Local)>>,
+    patch: MirPatch<'tcx>,
 }
 
-impl<'tcx, 'll> ReplacementVisitor<'tcx, 'll> {
-    fn gather_debug_info_fragments(
-        &self,
-        place: PlaceRef<'tcx>,
-    ) -> Vec<VarDebugInfoFragment<'tcx>> {
+impl<'tcx> ReplacementVisitor<'tcx, '_> {
+    fn gather_debug_info_fragments(&self, local: Local) -> Option<Vec<VarDebugInfoFragment<'tcx>>> {
         let mut fragments = Vec::new();
-        let parts = &self.fragments[place.local];
-        for (proj, replacement_local) in parts {
-            if proj.starts_with(place.projection) {
-                fragments.push(VarDebugInfoFragment {
-                    projection: proj[place.projection.len()..].to_vec(),
-                    contents: Place::from(*replacement_local),
-                });
-            }
-        }
-        fragments
-    }
-
-    fn replace_place(&self, place: PlaceRef<'tcx>) -> Option<Place<'tcx>> {
-        if let &[PlaceElem::Field(..), ref rest @ ..] = place.projection {
-            let pr = PlaceRef { local: place.local, projection: &place.projection[..1] };
-            let local = self.replacements.fields.get(&pr)?;
-            Some(Place { local: *local, projection: self.tcx.intern_place_elems(&rest) })
-        } else {
-            None
+        let parts = self.replacements.place_fragments(local.into())?;
+        for (field, ty, replacement_local) in parts {
+            fragments.push(VarDebugInfoFragment {
+                projection: vec![PlaceElem::Field(field, ty)],
+                contents: Place::from(replacement_local),
+            });
         }
+        Some(fragments)
     }
 }
 
@@ -254,94 +239,188 @@ impl<'tcx, 'll> MutVisitor<'tcx> for ReplacementVisitor<'tcx, 'll> {
         self.tcx
     }
 
-    fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) {
-        if let StatementKind::StorageLive(..)
-        | StatementKind::StorageDead(..)
-        | StatementKind::Deinit(..) = statement.kind
-        {
-            // Storage statements are expanded in run_pass.
-            return;
-        }
-        self.super_statement(statement, location)
-    }
-
     fn visit_place(&mut self, place: &mut Place<'tcx>, context: PlaceContext, location: Location) {
-        if let Some(repl) = self.replace_place(place.as_ref()) {
+        if let Some(repl) = self.replacements.replace_place(self.tcx, place.as_ref()) {
             *place = repl
         } else {
             self.super_place(place, context, location)
         }
     }
 
+    #[instrument(level = "trace", skip(self))]
+    fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) {
+        match statement.kind {
+            // Duplicate storage and deinit statements, as they pretty much apply to all fields.
+            StatementKind::StorageLive(l) => {
+                if let Some(final_locals) = self.replacements.place_fragments(l.into()) {
+                    for (_, _, fl) in final_locals {
+                        self.patch.add_statement(location, StatementKind::StorageLive(fl));
+                    }
+                    statement.make_nop();
+                }
+                return;
+            }
+            StatementKind::StorageDead(l) => {
+                if let Some(final_locals) = self.replacements.place_fragments(l.into()) {
+                    for (_, _, fl) in final_locals {
+                        self.patch.add_statement(location, StatementKind::StorageDead(fl));
+                    }
+                    statement.make_nop();
+                }
+                return;
+            }
+            StatementKind::Deinit(box place) => {
+                if let Some(final_locals) = self.replacements.place_fragments(place) {
+                    for (_, _, fl) in final_locals {
+                        self.patch
+                            .add_statement(location, StatementKind::Deinit(Box::new(fl.into())));
+                    }
+                    statement.make_nop();
+                    return;
+                }
+            }
+
+            // We have `a = Struct { 0: x, 1: y, .. }`.
+            // We replace it by
+            // ```
+            // a_0 = x
+            // a_1 = y
+            // ...
+            // ```
+            StatementKind::Assign(box (place, Rvalue::Aggregate(_, ref mut operands))) => {
+                if let Some(local) = place.as_local()
+                    && let Some(final_locals) = &self.replacements.fragments[local]
+                {
+                    // This is ok as we delete the statement later.
+                    let operands = std::mem::take(operands);
+                    for (&opt_ty_local, mut operand) in final_locals.iter().zip(operands) {
+                        if let Some((_, new_local)) = opt_ty_local {
+                            // Replace mentions of SROA'd locals that appear in the operand.
+                            self.visit_operand(&mut operand, location);
+
+                            let rvalue = Rvalue::Use(operand);
+                            self.patch.add_statement(
+                                location,
+                                StatementKind::Assign(Box::new((new_local.into(), rvalue))),
+                            );
+                        }
+                    }
+                    statement.make_nop();
+                    return;
+                }
+            }
+
+            // We have `a = some constant`
+            // We add the projections.
+            // ```
+            // a_0 = a.0
+            // a_1 = a.1
+            // ...
+            // ```
+            // ConstProp will pick up the pieces and replace them by actual constants.
+            StatementKind::Assign(box (place, Rvalue::Use(Operand::Constant(_)))) => {
+                if let Some(final_locals) = self.replacements.place_fragments(place) {
+                    // Put the deaggregated statements *after* the original one.
+                    let location = location.successor_within_block();
+                    for (field, ty, new_local) in final_locals {
+                        let rplace = self.tcx.mk_place_field(place, field, ty);
+                        let rvalue = Rvalue::Use(Operand::Move(rplace));
+                        self.patch.add_statement(
+                            location,
+                            StatementKind::Assign(Box::new((new_local.into(), rvalue))),
+                        );
+                    }
+                    // We still need `place.local` to exist, so don't make it nop.
+                    return;
+                }
+            }
+
+            // We have `a = move? place`
+            // We replace it by
+            // ```
+            // a_0 = move? place.0
+            // a_1 = move? place.1
+            // ...
+            // ```
+            StatementKind::Assign(box (lhs, Rvalue::Use(ref op))) => {
+                let (rplace, copy) = match *op {
+                    Operand::Copy(rplace) => (rplace, true),
+                    Operand::Move(rplace) => (rplace, false),
+                    Operand::Constant(_) => bug!(),
+                };
+                if let Some(final_locals) = self.replacements.place_fragments(lhs) {
+                    for (field, ty, new_local) in final_locals {
+                        let rplace = self.tcx.mk_place_field(rplace, field, ty);
+                        debug!(?rplace);
+                        let rplace = self
+                            .replacements
+                            .replace_place(self.tcx, rplace.as_ref())
+                            .unwrap_or(rplace);
+                        debug!(?rplace);
+                        let rvalue = if copy {
+                            Rvalue::Use(Operand::Copy(rplace))
+                        } else {
+                            Rvalue::Use(Operand::Move(rplace))
+                        };
+                        self.patch.add_statement(
+                            location,
+                            StatementKind::Assign(Box::new((new_local.into(), rvalue))),
+                        );
+                    }
+                    statement.make_nop();
+                    return;
+                }
+            }
+
+            _ => {}
+        }
+        self.super_statement(statement, location)
+    }
+
+    #[instrument(level = "trace", skip(self))]
     fn visit_var_debug_info(&mut self, var_debug_info: &mut VarDebugInfo<'tcx>) {
         match &mut var_debug_info.value {
             VarDebugInfoContents::Place(ref mut place) => {
-                if let Some(repl) = self.replace_place(place.as_ref()) {
+                if let Some(repl) = self.replacements.replace_place(self.tcx, place.as_ref()) {
                     *place = repl;
-                } else if self.all_dead_locals.contains(place.local) {
+                } else if let Some(local) = place.as_local()
+                    && let Some(fragments) = self.gather_debug_info_fragments(local)
+                {
                     let ty = place.ty(self.local_decls, self.tcx).ty;
-                    let fragments = self.gather_debug_info_fragments(place.as_ref());
                     var_debug_info.value = VarDebugInfoContents::Composite { ty, fragments };
                 }
             }
             VarDebugInfoContents::Composite { ty: _, ref mut fragments } => {
                 let mut new_fragments = Vec::new();
+                debug!(?fragments);
                 fragments
                     .drain_filter(|fragment| {
-                        if let Some(repl) = self.replace_place(fragment.contents.as_ref()) {
+                        if let Some(repl) =
+                            self.replacements.replace_place(self.tcx, fragment.contents.as_ref())
+                        {
                             fragment.contents = repl;
-                            true
-                        } else if self.all_dead_locals.contains(fragment.contents.local) {
-                            let frg = self.gather_debug_info_fragments(fragment.contents.as_ref());
+                            false
+                        } else if let Some(local) = fragment.contents.as_local()
+                            && let Some(frg) = self.gather_debug_info_fragments(local)
+                        {
                             new_fragments.extend(frg.into_iter().map(|mut f| {
                                 f.projection.splice(0..0, fragment.projection.iter().copied());
                                 f
                             }));
-                            false
-                        } else {
                             true
+                        } else {
+                            false
                         }
                     })
                     .for_each(drop);
+                debug!(?fragments);
+                debug!(?new_fragments);
                 fragments.extend(new_fragments);
             }
             VarDebugInfoContents::Const(_) => {}
         }
     }
 
-    fn visit_basic_block_data(&mut self, bb: BasicBlock, bbdata: &mut BasicBlockData<'tcx>) {
-        self.super_basic_block_data(bb, bbdata);
-
-        #[derive(Debug)]
-        enum Stmt {
-            StorageLive,
-            StorageDead,
-            Deinit,
-        }
-
-        bbdata.expand_statements(|stmt| {
-            let source_info = stmt.source_info;
-            let (stmt, origin_local) = match &stmt.kind {
-                StatementKind::StorageLive(l) => (Stmt::StorageLive, *l),
-                StatementKind::StorageDead(l) => (Stmt::StorageDead, *l),
-                StatementKind::Deinit(p) if let Some(l) = p.as_local() => (Stmt::Deinit, l),
-                _ => return None,
-            };
-            if !self.all_dead_locals.contains(origin_local) {
-                return None;
-            }
-            let final_locals = self.fragments.get(origin_local)?;
-            Some(final_locals.iter().map(move |&(_, l)| {
-                let kind = match stmt {
-                    Stmt::StorageLive => StatementKind::StorageLive(l),
-                    Stmt::StorageDead => StatementKind::StorageDead(l),
-                    Stmt::Deinit => StatementKind::Deinit(Box::new(l.into())),
-                };
-                Statement { source_info, kind }
-            }))
-        });
-    }
-
     fn visit_local(&mut self, local: &mut Local, _: PlaceContext, _: Location) {
         assert!(!self.all_dead_locals.contains(*local));
     }
diff --git a/compiler/rustc_mir_transform/src/ssa.rs b/compiler/rustc_mir_transform/src/ssa.rs
new file mode 100644
index 000000000..c1e7f62de
--- /dev/null
+++ b/compiler/rustc_mir_transform/src/ssa.rs
@@ -0,0 +1,257 @@
+use either::Either;
+use rustc_data_structures::graph::dominators::Dominators;
+use rustc_index::bit_set::BitSet;
+use rustc_index::vec::IndexVec;
+use rustc_middle::middle::resolve_bound_vars::Set1;
+use rustc_middle::mir::visit::*;
+use rustc_middle::mir::*;
+use rustc_middle::ty::{ParamEnv, TyCtxt};
+
+#[derive(Debug)]
+pub struct SsaLocals {
+    /// Assignments to each local. This defines whether the local is SSA.
+    assignments: IndexVec<Local, Set1<LocationExtended>>,
+    /// We visit the body in reverse postorder, to ensure each local is assigned before it is used.
+    /// We remember the order in which we saw the assignments to compute the SSA values in a single
+    /// pass.
+    assignment_order: Vec<Local>,
+    /// Copy equivalence classes between locals. See `copy_classes` for documentation.
+    copy_classes: IndexVec<Local, Local>,
+}
+
+/// We often encounter MIR bodies with 1 or 2 basic blocks. In those cases, it's unnecessary to
+/// actually compute dominators, we can just compare block indices because bb0 is always the first
+/// block, and in any body all other blocks are always always dominated by bb0.
+struct SmallDominators {
+    inner: Option<Dominators<BasicBlock>>,
+}
+
+trait DomExt {
+    fn dominates(self, _other: Self, dominators: &SmallDominators) -> bool;
+}
+
+impl DomExt for Location {
+    fn dominates(self, other: Location, dominators: &SmallDominators) -> bool {
+        if self.block == other.block {
+            self.statement_index <= other.statement_index
+        } else {
+            dominators.dominates(self.block, other.block)
+        }
+    }
+}
+
+impl SmallDominators {
+    fn dominates(&self, dom: BasicBlock, node: BasicBlock) -> bool {
+        if let Some(inner) = &self.inner { inner.dominates(dom, node) } else { dom < node }
+    }
+}
+
+impl SsaLocals {
+    pub fn new<'tcx>(
+        tcx: TyCtxt<'tcx>,
+        param_env: ParamEnv<'tcx>,
+        body: &Body<'tcx>,
+        borrowed_locals: &BitSet<Local>,
+    ) -> SsaLocals {
+        let assignment_order = Vec::new();
+
+        let assignments = IndexVec::from_elem(Set1::Empty, &body.local_decls);
+        let dominators =
+            if body.basic_blocks.len() > 2 { Some(body.basic_blocks.dominators()) } else { None };
+        let dominators = SmallDominators { inner: dominators };
+        let mut visitor = SsaVisitor { assignments, assignment_order, dominators };
+
+        for (local, decl) in body.local_decls.iter_enumerated() {
+            if matches!(body.local_kind(local), LocalKind::Arg) {
+                visitor.assignments[local] = Set1::One(LocationExtended::Arg);
+            }
+            if borrowed_locals.contains(local) && !decl.ty.is_freeze(tcx, param_env) {
+                visitor.assignments[local] = Set1::Many;
+            }
+        }
+
+        if body.basic_blocks.len() > 2 {
+            for (bb, data) in traversal::reverse_postorder(body) {
+                visitor.visit_basic_block_data(bb, data);
+            }
+        } else {
+            for (bb, data) in body.basic_blocks.iter_enumerated() {
+                visitor.visit_basic_block_data(bb, data);
+            }
+        }
+
+        for var_debug_info in &body.var_debug_info {
+            visitor.visit_var_debug_info(var_debug_info);
+        }
+
+        debug!(?visitor.assignments);
+
+        visitor
+            .assignment_order
+            .retain(|&local| matches!(visitor.assignments[local], Set1::One(_)));
+        debug!(?visitor.assignment_order);
+
+        let copy_classes = compute_copy_classes(&visitor, body);
+
+        SsaLocals {
+            assignments: visitor.assignments,
+            assignment_order: visitor.assignment_order,
+            copy_classes,
+        }
+    }
+
+    pub fn is_ssa(&self, local: Local) -> bool {
+        matches!(self.assignments[local], Set1::One(_))
+    }
+
+    pub fn assignments<'a, 'tcx>(
+        &'a self,
+        body: &'a Body<'tcx>,
+    ) -> impl Iterator<Item = (Local, &'a Rvalue<'tcx>)> + 'a {
+        self.assignment_order.iter().filter_map(|&local| {
+            if let Set1::One(LocationExtended::Plain(loc)) = self.assignments[local] {
+                // `loc` must point to a direct assignment to `local`.
+                let Either::Left(stmt) = body.stmt_at(loc) else { bug!() };
+                let Some((target, rvalue)) = stmt.kind.as_assign() else { bug!() };
+                assert_eq!(target.as_local(), Some(local));
+                Some((local, rvalue))
+            } else {
+                None
+            }
+        })
+    }
+
+    /// Compute the equivalence classes for locals, based on copy statements.
+    ///
+    /// The returned vector maps each local to the one it copies. In the following case:
+    ///   _a = &mut _0
+    ///   _b = move? _a
+    ///   _c = move? _a
+    ///   _d = move? _c
+    /// We return the mapping
+    ///   _a => _a // not a copy so, represented by itself
+    ///   _b => _a
+    ///   _c => _a
+    ///   _d => _a // transitively through _c
+    ///
+    /// Exception: we do not see through the return place, as it cannot be substituted.
+    pub fn copy_classes(&self) -> &IndexVec<Local, Local> {
+        &self.copy_classes
+    }
+
+    /// Make a property uniform on a copy equivalence class by removing elements.
+    pub fn meet_copy_equivalence(&self, property: &mut BitSet<Local>) {
+        // Consolidate to have a local iff all its copies are.
+        //
+        // `copy_classes` defines equivalence classes between locals. The `local`s that recursively
+        // move/copy the same local all have the same `head`.
+        for (local, &head) in self.copy_classes.iter_enumerated() {
+            // If any copy does not have `property`, then the head is not.
+            if !property.contains(local) {
+                property.remove(head);
+            }
+        }
+        for (local, &head) in self.copy_classes.iter_enumerated() {
+            // If any copy does not have `property`, then the head doesn't either,
+            // then no copy has `property`.
+            if !property.contains(head) {
+                property.remove(local);
+            }
+        }
+
+        // Verify that we correctly computed equivalence classes.
+        #[cfg(debug_assertions)]
+        for (local, &head) in self.copy_classes.iter_enumerated() {
+            assert_eq!(property.contains(local), property.contains(head));
+        }
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+enum LocationExtended {
+    Plain(Location),
+    Arg,
+}
+
+struct SsaVisitor {
+    dominators: SmallDominators,
+    assignments: IndexVec<Local, Set1<LocationExtended>>,
+    assignment_order: Vec<Local>,
+}
+
+impl<'tcx> Visitor<'tcx> for SsaVisitor {
+    fn visit_local(&mut self, local: Local, ctxt: PlaceContext, loc: Location) {
+        match ctxt {
+            PlaceContext::MutatingUse(MutatingUseContext::Store) => {
+                self.assignments[local].insert(LocationExtended::Plain(loc));
+                self.assignment_order.push(local);
+            }
+            // Anything can happen with raw pointers, so remove them.
+            PlaceContext::NonMutatingUse(NonMutatingUseContext::AddressOf)
+            | PlaceContext::MutatingUse(_) => self.assignments[local] = Set1::Many,
+            // Immutable borrows are taken into account in `SsaLocals::new` by
+            // removing non-freeze locals.
+            PlaceContext::NonMutatingUse(_) => {
+                let set = &mut self.assignments[local];
+                let assign_dominates = match *set {
+                    Set1::Empty | Set1::Many => false,
+                    Set1::One(LocationExtended::Arg) => true,
+                    Set1::One(LocationExtended::Plain(assign)) => {
+                        assign.successor_within_block().dominates(loc, &self.dominators)
+                    }
+                };
+                // We are visiting a use that is not dominated by an assignment.
+                // Either there is a cycle involved, or we are reading for uninitialized local.
+                // Bail out.
+                if !assign_dominates {
+                    *set = Set1::Many;
+                }
+            }
+            PlaceContext::NonUse(_) => {}
+        }
+    }
+}
+
+#[instrument(level = "trace", skip(ssa, body))]
+fn compute_copy_classes(ssa: &SsaVisitor, body: &Body<'_>) -> IndexVec<Local, Local> {
+    let mut copies = IndexVec::from_fn_n(|l| l, body.local_decls.len());
+
+    for &local in &ssa.assignment_order {
+        debug!(?local);
+
+        if local == RETURN_PLACE {
+            // `_0` is special, we cannot rename it.
+            continue;
+        }
+
+        // This is not SSA: mark that we don't know the value.
+        debug!(assignments = ?ssa.assignments[local]);
+        let Set1::One(LocationExtended::Plain(loc)) = ssa.assignments[local] else { continue };
+
+        // `loc` must point to a direct assignment to `local`.
+        let Either::Left(stmt) = body.stmt_at(loc) else { bug!() };
+        let Some((_target, rvalue)) = stmt.kind.as_assign() else { bug!() };
+        assert_eq!(_target.as_local(), Some(local));
+
+        let (Rvalue::Use(Operand::Copy(place) | Operand::Move(place)) | Rvalue::CopyForDeref(place))
+            = rvalue
+        else { continue };
+
+        let Some(rhs) = place.as_local() else { continue };
+        let Set1::One(_) = ssa.assignments[rhs] else { continue };
+
+        // We visit in `assignment_order`, ie. reverse post-order, so `rhs` has been
+        // visited before `local`, and we just have to copy the representing local.
+        copies[local] = copies[rhs];
+    }
+
+    debug!(?copies);
+
+    // Invariant: `copies` must point to the head of an equivalence class.
+    #[cfg(debug_assertions)]
+    for &head in copies.iter() {
+        assert_eq!(copies[head], head);
+    }
+
+    copies
+}
diff --git a/compiler/rustc_error_messages/locales/en-US/monomorphize.ftl b/compiler/rustc_monomorphize/locales/en-US.ftl
index 243d10bfa..6cea6a603 100644
--- a/compiler/rustc_error_messages/locales/en-US/monomorphize.ftl
+++ b/compiler/rustc_monomorphize/locales/en-US.ftl
@@ -24,3 +24,9 @@ monomorphize_large_assignments =
 
 monomorphize_couldnt_dump_mono_stats =
     unexpected error occurred while dumping monomorphization stats: {$error}
+
+monomorphize_encountered_error_while_instantiating =
+    the above error was encountered while instantiating `{$formatted_item}`
+
+monomorphize_unknown_cgu_collection_mode =
+    unknown codegen-item collection mode '{$mode}', falling back to 'lazy' mode
diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs
index ec1de3056..45e659eab 100644
--- a/compiler/rustc_monomorphize/src/collector.rs
+++ b/compiler/rustc_monomorphize/src/collector.rs
@@ -190,7 +190,7 @@ use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::query::TyCtxtAt;
 use rustc_middle::ty::subst::{GenericArgKind, InternalSubsts};
 use rustc_middle::ty::{
-    self, GenericParamDefKind, Instance, Ty, TyCtxt, TypeFoldable, TypeVisitable, VtblEntry,
+    self, GenericParamDefKind, Instance, Ty, TyCtxt, TypeFoldable, TypeVisitableExt, VtblEntry,
 };
 use rustc_middle::{middle::codegen_fn_attrs::CodegenFnAttrFlags, mir::visit::TyContext};
 use rustc_session::config::EntryFnType;
@@ -201,7 +201,9 @@ use rustc_target::abi::Size;
 use std::ops::Range;
 use std::path::PathBuf;
 
-use crate::errors::{LargeAssignmentsLint, RecursionLimit, TypeLengthLimit};
+use crate::errors::{
+    EncounteredErrorWhileInstantiating, LargeAssignmentsLint, RecursionLimit, TypeLengthLimit,
+};
 
 #[derive(PartialEq)]
 pub enum MonoItemCollectionMode {
@@ -524,10 +526,10 @@ fn collect_items_rec<'tcx>(
         && starting_point.node.is_user_defined()
     {
         let formatted_item = with_no_trimmed_paths!(starting_point.node.to_string());
-        tcx.sess.span_note_without_error(
-            starting_point.span,
-            &format!("the above error was encountered while instantiating `{formatted_item}`"),
-        );
+        tcx.sess.emit_note(EncounteredErrorWhileInstantiating {
+            span: starting_point.span,
+            formatted_item,
+        });
     }
     inlining_map.lock_mut().record_accesses(starting_point.node, &neighbors.items);
 
@@ -658,7 +660,7 @@ struct MirNeighborCollector<'a, 'tcx> {
 impl<'a, 'tcx> MirNeighborCollector<'a, 'tcx> {
     pub fn monomorphize<T>(&self, value: T) -> T
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         debug!("monomorphize: self.instance={:?}", self.instance);
         self.instance.subst_mir_and_normalize_erasing_regions(
@@ -1191,28 +1193,13 @@ impl<'v> RootCollector<'_, 'v> {
     fn process_item(&mut self, id: hir::ItemId) {
         match self.tcx.def_kind(id.owner_id) {
             DefKind::Enum | DefKind::Struct | DefKind::Union => {
-                let item = self.tcx.hir().item(id);
-                match item.kind {
-                    hir::ItemKind::Enum(_, ref generics)
-                    | hir::ItemKind::Struct(_, ref generics)
-                    | hir::ItemKind::Union(_, ref generics) => {
-                        if generics.params.is_empty() {
-                            if self.mode == MonoItemCollectionMode::Eager {
-                                debug!(
-                                    "RootCollector: ADT drop-glue for {}",
-                                    self.tcx.def_path_str(item.owner_id.to_def_id())
-                                );
-
-                                let ty = Instance::new(
-                                    item.owner_id.to_def_id(),
-                                    InternalSubsts::empty(),
-                                )
-                                .ty(self.tcx, ty::ParamEnv::reveal_all());
-                                visit_drop_use(self.tcx, ty, true, DUMMY_SP, self.output);
-                            }
-                        }
-                    }
-                    _ => bug!(),
+                if self.mode == MonoItemCollectionMode::Eager
+                    && self.tcx.generics_of(id.owner_id).count() == 0
+                {
+                    debug!("RootCollector: ADT drop-glue for `{id:?}`",);
+
+                    let ty = self.tcx.type_of(id.owner_id.to_def_id()).no_bound_vars().unwrap();
+                    visit_drop_use(self.tcx, ty, true, DUMMY_SP, self.output);
                 }
             }
             DefKind::GlobalAsm => {
@@ -1238,10 +1225,9 @@ impl<'v> RootCollector<'_, 'v> {
                     collect_const_value(self.tcx, val, &mut self.output);
                 }
             }
-            DefKind::Impl => {
+            DefKind::Impl { .. } => {
                 if self.mode == MonoItemCollectionMode::Eager {
-                    let item = self.tcx.hir().item(id);
-                    create_mono_items_for_default_impls(self.tcx, item, self.output);
+                    create_mono_items_for_default_impls(self.tcx, id, self.output);
                 }
             }
             DefKind::Fn => {
@@ -1296,7 +1282,7 @@ impl<'v> RootCollector<'_, 'v> {
         };
 
         let start_def_id = self.tcx.require_lang_item(LangItem::Start, None);
-        let main_ret_ty = self.tcx.fn_sig(main_def_id).output();
+        let main_ret_ty = self.tcx.fn_sig(main_def_id).no_bound_vars().unwrap().output();
 
         // Given that `main()` has no arguments,
         // then its return type cannot have
@@ -1312,7 +1298,7 @@ impl<'v> RootCollector<'_, 'v> {
             self.tcx,
             ty::ParamEnv::reveal_all(),
             start_def_id,
-            self.tcx.intern_substs(&[main_ret_ty.into()]),
+            self.tcx.mk_substs(&[main_ret_ty.into()]),
         )
         .unwrap()
         .unwrap();
@@ -1326,66 +1312,51 @@ fn item_requires_monomorphization(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     generics.requires_monomorphization(tcx)
 }
 
+#[instrument(level = "debug", skip(tcx, output))]
 fn create_mono_items_for_default_impls<'tcx>(
     tcx: TyCtxt<'tcx>,
-    item: &'tcx hir::Item<'tcx>,
+    item: hir::ItemId,
     output: &mut MonoItems<'tcx>,
 ) {
-    match item.kind {
-        hir::ItemKind::Impl(ref impl_) => {
-            if matches!(impl_.polarity, hir::ImplPolarity::Negative(_)) {
-                return;
-            }
+    let polarity = tcx.impl_polarity(item.owner_id);
+    if matches!(polarity, ty::ImplPolarity::Negative) {
+        return;
+    }
 
-            for param in impl_.generics.params {
-                match param.kind {
-                    hir::GenericParamKind::Lifetime { .. } => {}
-                    hir::GenericParamKind::Type { .. } | hir::GenericParamKind::Const { .. } => {
-                        return;
-                    }
-                }
-            }
+    if tcx.generics_of(item.owner_id).own_requires_monomorphization() {
+        return;
+    }
 
-            debug!(
-                "create_mono_items_for_default_impls(item={})",
-                tcx.def_path_str(item.owner_id.to_def_id())
-            );
+    let Some(trait_ref) = tcx.impl_trait_ref(item.owner_id) else {
+        return;
+    };
 
-            if let Some(trait_ref) = tcx.impl_trait_ref(item.owner_id) {
-                let trait_ref = trait_ref.subst_identity();
+    let trait_ref = trait_ref.subst_identity();
 
-                let param_env = ty::ParamEnv::reveal_all();
-                let trait_ref = tcx.normalize_erasing_regions(param_env, trait_ref);
-                let overridden_methods = tcx.impl_item_implementor_ids(item.owner_id);
-                for method in tcx.provided_trait_methods(trait_ref.def_id) {
-                    if overridden_methods.contains_key(&method.def_id) {
-                        continue;
-                    }
+    let param_env = ty::ParamEnv::reveal_all();
+    let trait_ref = tcx.normalize_erasing_regions(param_env, trait_ref);
+    let overridden_methods = tcx.impl_item_implementor_ids(item.owner_id);
+    for method in tcx.provided_trait_methods(trait_ref.def_id) {
+        if overridden_methods.contains_key(&method.def_id) {
+            continue;
+        }
 
-                    if tcx.generics_of(method.def_id).own_requires_monomorphization() {
-                        continue;
-                    }
+        if tcx.generics_of(method.def_id).own_requires_monomorphization() {
+            continue;
+        }
 
-                    let substs =
-                        InternalSubsts::for_item(tcx, method.def_id, |param, _| match param.kind {
-                            GenericParamDefKind::Lifetime => tcx.lifetimes.re_erased.into(),
-                            GenericParamDefKind::Type { .. }
-                            | GenericParamDefKind::Const { .. } => {
-                                trait_ref.substs[param.index as usize]
-                            }
-                        });
-                    let instance =
-                        ty::Instance::expect_resolve(tcx, param_env, method.def_id, substs);
-
-                    let mono_item = create_fn_mono_item(tcx, instance, DUMMY_SP);
-                    if mono_item.node.is_instantiable(tcx) && should_codegen_locally(tcx, &instance)
-                    {
-                        output.push(mono_item);
-                    }
-                }
+        let substs = InternalSubsts::for_item(tcx, method.def_id, |param, _| match param.kind {
+            GenericParamDefKind::Lifetime => tcx.lifetimes.re_erased.into(),
+            GenericParamDefKind::Type { .. } | GenericParamDefKind::Const { .. } => {
+                trait_ref.substs[param.index as usize]
             }
+        });
+        let instance = ty::Instance::expect_resolve(tcx, param_env, method.def_id, substs);
+
+        let mono_item = create_fn_mono_item(tcx, instance, DUMMY_SP);
+        if mono_item.node.is_instantiable(tcx) && should_codegen_locally(tcx, &instance) {
+            output.push(mono_item);
         }
-        _ => bug!(),
     }
 }
 
diff --git a/compiler/rustc_monomorphize/src/errors.rs b/compiler/rustc_monomorphize/src/errors.rs
index 5233cfb21..495a73490 100644
--- a/compiler/rustc_monomorphize/src/errors.rs
+++ b/compiler/rustc_monomorphize/src/errors.rs
@@ -1,5 +1,6 @@
 use std::path::PathBuf;
 
+use crate::fluent_generated as fluent;
 use rustc_errors::ErrorGuaranteed;
 use rustc_errors::IntoDiagnostic;
 use rustc_macros::{Diagnostic, LintDiagnostic};
@@ -44,7 +45,7 @@ impl IntoDiagnostic<'_> for UnusedGenericParamsHint {
         self,
         handler: &'_ rustc_errors::Handler,
     ) -> rustc_errors::DiagnosticBuilder<'_, ErrorGuaranteed> {
-        let mut diag = handler.struct_err(rustc_errors::fluent::monomorphize_unused_generic_params);
+        let mut diag = handler.struct_err(fluent::monomorphize_unused_generic_params);
         diag.set_span(self.span);
         for (span, name) in self.param_spans.into_iter().zip(self.param_names) {
             // FIXME: I can figure out how to do a label with a fluent string with a fixed message,
@@ -83,3 +84,17 @@ pub struct SymbolAlreadyDefined {
 pub struct CouldntDumpMonoStats {
     pub error: String,
 }
+
+#[derive(Diagnostic)]
+#[diag(monomorphize_encountered_error_while_instantiating)]
+pub struct EncounteredErrorWhileInstantiating {
+    #[primary_span]
+    pub span: Span,
+    pub formatted_item: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(monomorphize_unknown_cgu_collection_mode)]
+pub struct UnknownCguCollectionMode<'a> {
+    pub mode: &'a str,
+}
diff --git a/compiler/rustc_monomorphize/src/lib.rs b/compiler/rustc_monomorphize/src/lib.rs
index f88155e4f..f6b791f29 100644
--- a/compiler/rustc_monomorphize/src/lib.rs
+++ b/compiler/rustc_monomorphize/src/lib.rs
@@ -9,7 +9,9 @@ extern crate tracing;
 #[macro_use]
 extern crate rustc_middle;
 
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
 use rustc_hir::lang_items::LangItem;
+use rustc_macros::fluent_messages;
 use rustc_middle::traits;
 use rustc_middle::ty::adjustment::CustomCoerceUnsized;
 use rustc_middle::ty::query::{Providers, TyCtxtAt};
@@ -21,6 +23,8 @@ mod partitioning;
 mod polymorphize;
 mod util;
 
+fluent_messages! { "../locales/en-US.ftl" }
+
 fn custom_coerce_unsize_info<'tcx>(
     tcx: TyCtxtAt<'tcx>,
     source_ty: Ty<'tcx>,
diff --git a/compiler/rustc_monomorphize/src/partitioning/default.rs b/compiler/rustc_monomorphize/src/partitioning/default.rs
index 29009c480..2c56edd89 100644
--- a/compiler/rustc_monomorphize/src/partitioning/default.rs
+++ b/compiler/rustc_monomorphize/src/partitioning/default.rs
@@ -9,7 +9,7 @@ use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel
 use rustc_middle::mir::mono::{CodegenUnit, CodegenUnitNameBuilder, Linkage, Visibility};
 use rustc_middle::mir::mono::{InstantiationMode, MonoItem};
 use rustc_middle::ty::print::characteristic_def_id_of_type;
-use rustc_middle::ty::{self, visit::TypeVisitable, DefIdTree, InstanceDef, TyCtxt};
+use rustc_middle::ty::{self, visit::TypeVisitableExt, DefIdTree, InstanceDef, TyCtxt};
 use rustc_span::symbol::Symbol;
 
 use super::PartitioningCx;
@@ -308,7 +308,7 @@ fn characteristic_def_id_of_mono_item<'tcx>(
                     let impl_self_ty = tcx.subst_and_normalize_erasing_regions(
                         instance.substs,
                         ty::ParamEnv::reveal_all(),
-                        tcx.type_of(impl_def_id),
+                        tcx.type_of(impl_def_id).skip_binder(),
                     );
                     if let Some(def_id) = characteristic_def_id_of_type(impl_self_ty) {
                         return Some(def_id);
diff --git a/compiler/rustc_monomorphize/src/partitioning/mod.rs b/compiler/rustc_monomorphize/src/partitioning/mod.rs
index fd6bcad18..524c51d88 100644
--- a/compiler/rustc_monomorphize/src/partitioning/mod.rs
+++ b/compiler/rustc_monomorphize/src/partitioning/mod.rs
@@ -114,7 +114,9 @@ use rustc_span::symbol::Symbol;
 
 use crate::collector::InliningMap;
 use crate::collector::{self, MonoItemCollectionMode};
-use crate::errors::{CouldntDumpMonoStats, SymbolAlreadyDefined, UnknownPartitionStrategy};
+use crate::errors::{
+    CouldntDumpMonoStats, SymbolAlreadyDefined, UnknownCguCollectionMode, UnknownPartitionStrategy,
+};
 
 pub struct PartitioningCx<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
@@ -180,7 +182,7 @@ pub fn partition<'tcx>(
         partitioner.place_root_mono_items(cx, mono_items)
     };
 
-    initial_partitioning.codegen_units.iter_mut().for_each(|cgu| cgu.estimate_size(tcx));
+    initial_partitioning.codegen_units.iter_mut().for_each(|cgu| cgu.create_size_estimate(tcx));
 
     debug_dump(tcx, "INITIAL PARTITIONING:", initial_partitioning.codegen_units.iter());
 
@@ -200,7 +202,7 @@ pub fn partition<'tcx>(
         partitioner.place_inlined_mono_items(cx, initial_partitioning)
     };
 
-    post_inlining.codegen_units.iter_mut().for_each(|cgu| cgu.estimate_size(tcx));
+    post_inlining.codegen_units.iter_mut().for_each(|cgu| cgu.create_size_estimate(tcx));
 
     debug_dump(tcx, "POST INLINING:", post_inlining.codegen_units.iter());
 
@@ -348,17 +350,13 @@ where
 fn collect_and_partition_mono_items(tcx: TyCtxt<'_>, (): ()) -> (&DefIdSet, &[CodegenUnit<'_>]) {
     let collection_mode = match tcx.sess.opts.unstable_opts.print_mono_items {
         Some(ref s) => {
-            let mode_string = s.to_lowercase();
-            let mode_string = mode_string.trim();
-            if mode_string == "eager" {
+            let mode = s.to_lowercase();
+            let mode = mode.trim();
+            if mode == "eager" {
                 MonoItemCollectionMode::Eager
             } else {
-                if mode_string != "lazy" {
-                    let message = format!(
-                        "Unknown codegen-item collection mode '{mode_string}'. \
-                                           Falling back to 'lazy' mode."
-                    );
-                    tcx.sess.warn(&message);
+                if mode != "lazy" {
+                    tcx.sess.emit_warning(UnknownCguCollectionMode { mode });
                 }
 
                 MonoItemCollectionMode::Lazy
diff --git a/compiler/rustc_monomorphize/src/polymorphize.rs b/compiler/rustc_monomorphize/src/polymorphize.rs
index cf13d4584..b7c3dbcc0 100644
--- a/compiler/rustc_monomorphize/src/polymorphize.rs
+++ b/compiler/rustc_monomorphize/src/polymorphize.rs
@@ -15,7 +15,7 @@ use rustc_middle::ty::{
     self,
     query::Providers,
     subst::SubstsRef,
-    visit::{TypeSuperVisitable, TypeVisitable, TypeVisitor},
+    visit::{TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor},
     Const, Ty, TyCtxt, UnusedGenericParams,
 };
 use rustc_span::symbol::sym;
@@ -172,7 +172,7 @@ fn mark_used_by_default_parameters<'tcx>(
         | DefKind::Field
         | DefKind::LifetimeParam
         | DefKind::GlobalAsm
-        | DefKind::Impl => {
+        | DefKind::Impl { .. } => {
             for param in &generics.params {
                 debug!(?param, "(other)");
                 if let ty::GenericParamDefKind::Lifetime = param.kind {
@@ -296,7 +296,7 @@ impl<'a, 'tcx> Visitor<'tcx> for MarkUsedGenericParams<'a, 'tcx> {
     }
 }
 
-impl<'a, 'tcx> TypeVisitor<'tcx> for MarkUsedGenericParams<'a, 'tcx> {
+impl<'a, 'tcx> TypeVisitor<TyCtxt<'tcx>> for MarkUsedGenericParams<'a, 'tcx> {
     #[instrument(level = "debug", skip(self))]
     fn visit_const(&mut self, c: Const<'tcx>) -> ControlFlow<Self::BreakTy> {
         if !c.has_non_region_param() {
diff --git a/compiler/rustc_parse/Cargo.toml b/compiler/rustc_parse/Cargo.toml
index dbcfb3903..3eb158c81 100644
--- a/compiler/rustc_parse/Cargo.toml
+++ b/compiler/rustc_parse/Cargo.toml
@@ -16,7 +16,7 @@ rustc_lexer = { path = "../rustc_lexer" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
-thin-vec = "0.2.8"
+thin-vec = "0.2.12"
 tracing = "0.1"
 unicode-normalization = "0.1.11"
 unicode-width = "0.1.4"
diff --git a/compiler/rustc_error_messages/locales/en-US/parse.ftl b/compiler/rustc_parse/locales/en-US.ftl
index 8f063f508..e76e91fc1 100644
--- a/compiler/rustc_error_messages/locales/en-US/parse.ftl
+++ b/compiler/rustc_parse/locales/en-US.ftl
@@ -93,6 +93,26 @@ parse_do_catch_syntax_removed = found removed `do catch` syntax
 parse_float_literal_requires_integer_part = float literals must have an integer part
     .suggestion = must have an integer part
 
+parse_invalid_int_literal_width = invalid width `{$width}` for integer literal
+    .help = valid widths are 8, 16, 32, 64 and 128
+
+parse_invalid_num_literal_base_prefix = invalid base prefix for number literal
+    .note = base prefixes (`0xff`, `0b1010`, `0o755`) are lowercase
+    .suggestion = try making the prefix lowercase
+
+parse_invalid_num_literal_suffix = invalid suffix `{$suffix}` for number literal
+    .label = invalid suffix `{$suffix}`
+    .help = the suffix must be one of the numeric types (`u32`, `isize`, `f32`, etc.)
+
+parse_invalid_float_literal_width = invalid width `{$width}` for float literal
+    .help = valid widths are 32 and 64
+
+parse_invalid_float_literal_suffix = invalid suffix `{$suffix}` for float literal
+    .label = invalid suffix `{$suffix}`
+    .help = valid suffixes are `f32` and `f64`
+
+parse_int_literal_too_large = integer literal is too large
+
 parse_missing_semicolon_before_array = expected `;`, found `[`
     .suggestion = consider adding `;` here
 
@@ -128,6 +148,13 @@ parse_missing_in_in_for_loop = missing `in` in `for` loop
     .use_in_not_of = try using `in` here instead
     .add_in = try adding `in` here
 
+parse_missing_expression_in_for_loop = missing expression to iterate on in `for` loop
+    .suggestion = try adding an expression to the `for` loop
+
+parse_loop_else = `{$loop_kind}...else` loops are not supported
+    .note = consider moving this `else` clause to a separate `if` statement and use a `bool` variable to control if it should run
+    .loop_keyword = `else` is attached to this loop
+
 parse_missing_comma_after_match_arm = expected `,` following `match` arm
     .suggestion = missing a comma here to end this `match` arm
 
@@ -197,13 +224,33 @@ parse_match_arm_body_without_braces = `match` arm body without braces
             [one] statement
            *[other] statements
         } with a body
-    .suggestion_use_comma_not_semicolon = use a comma to end a `match` arm expression
+    .suggestion_use_comma_not_semicolon = replace `;` with `,` to end a `match` arm expression
+
+parse_inclusive_range_extra_equals = unexpected `=` after inclusive range
+    .suggestion_remove_eq = use `..=` instead
+    .note = inclusive ranges end with a single equals sign (`..=`)
+
+parse_inclusive_range_match_arrow = unexpected `>` after inclusive range
+    .label = this is parsed as an inclusive range `..=`
+    .suggestion = add a space between the pattern and `=>`
+
+parse_inclusive_range_no_end = inclusive range with no end
+    .suggestion_open_range = use `..` instead
+    .note = inclusive ranges must be bounded at the end (`..=b` or `a..=b`)
 
 parse_struct_literal_not_allowed_here = struct literals are not allowed here
     .suggestion = surround the struct literal with parentheses
 
 parse_invalid_interpolated_expression = invalid interpolated expression
 
+parse_hexadecimal_float_literal_not_supported = hexadecimal float literal is not supported
+parse_octal_float_literal_not_supported = octal float literal is not supported
+parse_binary_float_literal_not_supported = binary float literal is not supported
+parse_not_supported = not supported
+
+parse_invalid_literal_suffix = suffixes on {$kind} literals are invalid
+    .label = invalid suffix `{$suffix}`
+
 parse_invalid_literal_suffix_on_tuple_index = suffixes on a tuple index are invalid
     .label = invalid suffix `{$suffix}`
     .tuple_exception_line_1 = `{$suffix}` is *temporarily* accepted on tuple index fields as it was incorrectly accepted on stable for a few releases
@@ -238,6 +285,7 @@ parse_const_let_mutually_exclusive = `const` and `let` are mutually exclusive
 
 parse_invalid_expression_in_let_else = a `{$operator}` expression cannot be directly assigned in `let...else`
 parse_invalid_curly_in_let_else = right curly brace `{"}"}` before `else` in a `let...else` statement not allowed
+parse_extra_if_in_let_else = remove the `if` if you meant to write a `let...else` statement
 
 parse_compound_assignment_expression_in_let = can't reassign to an uninitialized variable
     .suggestion = initialize the variable
@@ -378,3 +426,309 @@ parse_where_clause_before_tuple_struct_body = where clauses are not allowed befo
     .name_label = while parsing this tuple struct
     .body_label = the struct body
     .suggestion = move the body before the where clause
+
+parse_async_fn_in_2015 = `async fn` is not permitted in Rust 2015
+    .label = to use `async fn`, switch to Rust 2018 or later
+
+parse_async_block_in_2015 = `async` blocks are only allowed in Rust 2018 or later
+
+parse_self_argument_pointer = cannot pass `self` by raw pointer
+    .label = cannot pass `self` by raw pointer
+
+parse_visibility_not_followed_by_item = visibility `{$vis}` is not followed by an item
+    .label = the visibility
+    .help = you likely meant to define an item, e.g., `{$vis} fn foo() {"{}"}`
+
+parse_default_not_followed_by_item = `default` is not followed by an item
+    .label = the `default` qualifier
+    .note = only `fn`, `const`, `type`, or `impl` items may be prefixed by `default`
+
+parse_missing_struct_for_struct_definition = missing `struct` for struct definition
+    .suggestion = add `struct` here to parse `{$ident}` as a public struct
+
+parse_missing_fn_for_function_definition = missing `fn` for function definition
+    .suggestion = add `fn` here to parse `{$ident}` as a public function
+
+parse_missing_fn_for_method_definition = missing `fn` for method definition
+    .suggestion = add `fn` here to parse `{$ident}` as a public method
+
+parse_ambiguous_missing_keyword_for_item_definition = missing `fn` or `struct` for function or struct definition
+    .suggestion = if you meant to call a macro, try
+    .help = if you meant to call a macro, remove the `pub` and add a trailing `!` after the identifier
+
+parse_missing_trait_in_trait_impl = missing trait in a trait impl
+    .suggestion_add_trait = add a trait here
+    .suggestion_remove_for = for an inherent impl, drop this `for`
+
+parse_missing_for_in_trait_impl = missing `for` in a trait impl
+    .suggestion = add `for` here
+
+parse_expected_trait_in_trait_impl_found_type = expected a trait, found type
+
+parse_non_item_in_item_list = non-item in item list
+    .suggestion_use_const_not_let = consider using `const` instead of `let` for associated const
+    .label_list_start = item list starts here
+    .label_non_item = non-item starts here
+    .label_list_end = item list ends here
+    .suggestion_remove_semicolon = consider removing this semicolon
+
+parse_bounds_not_allowed_on_trait_aliases = bounds are not allowed on trait aliases
+
+parse_trait_alias_cannot_be_auto = trait aliases cannot be `auto`
+parse_trait_alias_cannot_be_unsafe = trait aliases cannot be `unsafe`
+
+parse_associated_static_item_not_allowed = associated `static` items are not allowed
+
+parse_extern_crate_name_with_dashes = crate name using dashes are not valid in `extern crate` statements
+    .label = dash-separated idents are not valid
+    .suggestion = if the original crate name uses dashes you need to use underscores in the code
+
+parse_extern_item_cannot_be_const = extern items cannot be `const`
+    .suggestion = try using a static value
+    .note = for more information, visit https://doc.rust-lang.org/std/keyword.extern.html
+
+parse_const_global_cannot_be_mutable = const globals cannot be mutable
+    .label = cannot be mutable
+    .suggestion = you might want to declare a static instead
+
+parse_missing_const_type = missing type for `{$kind}` item
+    .suggestion = provide a type for the item
+
+parse_enum_struct_mutually_exclusive = `enum` and `struct` are mutually exclusive
+    .suggestion = replace `enum struct` with
+
+parse_unexpected_token_after_struct_name = expected `where`, `{"{"}`, `(`, or `;` after struct name
+parse_unexpected_token_after_struct_name_found_reserved_identifier = expected `where`, `{"{"}`, `(`, or `;` after struct name, found reserved identifier `{$token}`
+parse_unexpected_token_after_struct_name_found_keyword = expected `where`, `{"{"}`, `(`, or `;` after struct name, found keyword `{$token}`
+parse_unexpected_token_after_struct_name_found_reserved_keyword = expected `where`, `{"{"}`, `(`, or `;` after struct name, found reserved keyword `{$token}`
+parse_unexpected_token_after_struct_name_found_doc_comment = expected `where`, `{"{"}`, `(`, or `;` after struct name, found doc comment `{$token}`
+parse_unexpected_token_after_struct_name_found_other = expected `where`, `{"{"}`, `(`, or `;` after struct name, found `{$token}`
+
+parse_unexpected_self_in_generic_parameters = unexpected keyword `Self` in generic parameters
+    .note = you cannot use `Self` as a generic parameter because it is reserved for associated items
+
+parse_unexpected_default_value_for_lifetime_in_generic_parameters = unexpected default lifetime parameter
+    .label = lifetime parameters cannot have default values
+
+parse_multiple_where_clauses = cannot define duplicate `where` clauses on an item
+    .label = previous `where` clause starts here
+    .suggestion = consider joining the two `where` clauses into one
+
+parse_nonterminal_expected_item_keyword = expected an item keyword
+parse_nonterminal_expected_statement = expected a statement
+parse_nonterminal_expected_ident = expected ident, found `{$token}`
+parse_nonterminal_expected_lifetime = expected a lifetime, found `{$token}`
+
+parse_or_pattern_not_allowed_in_let_binding = top-level or-patterns are not allowed in `let` bindings
+parse_or_pattern_not_allowed_in_fn_parameters = top-level or-patterns are not allowed in function parameters
+parse_sugg_remove_leading_vert_in_pattern = remove the `|`
+parse_sugg_wrap_pattern_in_parens = wrap the pattern in parentheses
+
+parse_note_pattern_alternatives_use_single_vert = alternatives in or-patterns are separated with `|`, not `||`
+
+parse_unexpected_vert_vert_before_function_parameter = unexpected `||` before function parameter
+    .suggestion = remove the `||`
+
+parse_label_while_parsing_or_pattern_here = while parsing this or-pattern starting here
+
+parse_unexpected_vert_vert_in_pattern = unexpected token `||` in pattern
+    .suggestion = use a single `|` to separate multiple alternative patterns
+
+parse_trailing_vert_not_allowed = a trailing `|` is not allowed in an or-pattern
+    .suggestion = remove the `{$token}`
+
+parse_dotdotdot_rest_pattern = unexpected `...`
+    .label = not a valid pattern
+    .suggestion = for a rest pattern, use `..` instead of `...`
+
+parse_pattern_on_wrong_side_of_at = pattern on wrong side of `@`
+    .label_pattern = pattern on the left, should be on the right
+    .label_binding = binding on the right, should be on the left
+    .suggestion = switch the order
+
+parse_expected_binding_left_of_at = left-hand side of `@` must be a binding
+    .label_lhs = interpreted as a pattern, not a binding
+    .label_rhs = also a pattern
+    .note = bindings are `x`, `mut x`, `ref x`, and `ref mut x`
+
+parse_ambiguous_range_pattern = the range pattern here has ambiguous interpretation
+    .suggestion = add parentheses to clarify the precedence
+
+parse_unexpected_lifetime_in_pattern = unexpected lifetime `{$symbol}` in pattern
+    .suggestion = remove the lifetime
+
+parse_ref_mut_order_incorrect = the order of `mut` and `ref` is incorrect
+    .suggestion = try switching the order
+
+parse_mut_on_nested_ident_pattern = `mut` must be attached to each individual binding
+    .suggestion = add `mut` to each binding
+parse_mut_on_non_ident_pattern = `mut` must be followed by a named binding
+    .suggestion = remove the `mut` prefix
+parse_note_mut_pattern_usage = `mut` may be followed by `variable` and `variable @ pattern`
+
+parse_repeated_mut_in_pattern = `mut` on a binding may not be repeated
+    .suggestion = remove the additional `mut`s
+
+parse_dot_dot_dot_range_to_pattern_not_allowed = range-to patterns with `...` are not allowed
+    .suggestion = use `..=` instead
+
+parse_enum_pattern_instead_of_identifier = expected identifier, found enum pattern
+
+parse_dot_dot_dot_for_remaining_fields = expected field pattern, found `{$token_str}`
+    .suggestion = to omit remaining fields, use `..`
+
+parse_expected_comma_after_pattern_field = expected `,`
+
+parse_return_types_use_thin_arrow = return types are denoted using `->`
+    .suggestion = use `->` instead
+
+parse_need_plus_after_trait_object_lifetime = lifetime in trait object type must be followed by `+`
+
+parse_expected_mut_or_const_in_raw_pointer_type = expected `mut` or `const` keyword in raw pointer type
+    .suggestion = add `mut` or `const` here
+
+parse_lifetime_after_mut = lifetime must precede `mut`
+    .suggestion = place the lifetime before `mut`
+
+parse_dyn_after_mut = `mut` must precede `dyn`
+    .suggestion = place `mut` before `dyn`
+
+parse_fn_pointer_cannot_be_const = an `fn` pointer type cannot be `const`
+    .label = `const` because of this
+    .suggestion = remove the `const` qualifier
+
+parse_fn_pointer_cannot_be_async = an `fn` pointer type cannot be `async`
+    .label = `async` because of this
+    .suggestion = remove the `async` qualifier
+
+parse_nested_c_variadic_type = C-variadic type `...` may not be nested inside another type
+
+parse_invalid_dyn_keyword = invalid `dyn` keyword
+    .help = `dyn` is only needed at the start of a trait `+`-separated list
+    .suggestion = remove this keyword
+
+parse_negative_bounds_not_supported = negative bounds are not supported
+    .label = negative bounds are not supported
+    .suggestion = {$num_bounds ->
+            [one] remove the bound
+           *[other] remove the bounds
+        }
+
+parse_help_set_edition_cargo = set `edition = "{$edition}"` in `Cargo.toml`
+parse_help_set_edition_standalone = pass `--edition {$edition}` to `rustc`
+parse_note_edition_guide = for more on editions, read https://doc.rust-lang.org/edition-guide
+
+parse_unexpected_token_after_dot = unexpected token: `{$actual}`
+
+parse_cannot_be_raw_ident = `{$ident}` cannot be a raw identifier
+
+parse_cr_doc_comment = bare CR not allowed in {$block ->
+    [true] block doc-comment
+    *[false] doc-comment
+}
+
+parse_no_digits_literal = no valid digits found for number
+
+parse_invalid_digit_literal = invalid digit for a base {$base} literal
+
+parse_empty_exponent_float = expected at least one digit in exponent
+
+parse_float_literal_unsupported_base = {$base} float literal is not supported
+
+parse_more_than_one_char = character literal may only contain one codepoint
+    .followed_by = this `{$chr}` is followed by the combining {$len ->
+        [one] mark
+        *[other] marks
+        } `{$escaped_marks}`
+    .non_printing = there are non-printing characters, the full sequence is `{$escaped}`
+    .consider_normalized = consider using the normalized form `{$ch}` of this character
+    .remove_non = consider removing the non-printing characters
+    .use_double_quotes = if you meant to write a {$is_byte ->
+        [true] byte string
+        *[false] `str`
+        } literal, use double quotes
+
+parse_no_brace_unicode_escape = incorrect unicode escape sequence
+    .label = {parse_no_brace_unicode_escape}
+    .use_braces = format of unicode escape sequences uses braces
+    .format_of_unicode = format of unicode escape sequences is `\u{"{...}"}`
+
+parse_invalid_unicode_escape = invalid unicode character escape
+    .label = invalid escape
+    .help = unicode escape must {$surrogate ->
+    [true] not be a surrogate
+    *[false] be at most 10FFFF
+    }
+
+parse_escape_only_char = {$byte ->
+    [true] byte
+    *[false] character
+    } constant must be escaped: `{$escaped_msg}`
+    .escape = escape the character
+
+parse_bare_cr = {$double_quotes ->
+    [true] bare CR not allowed in string, use `\r` instead
+    *[false] character constant must be escaped: `\r`
+    }
+    .escape = escape the character
+
+parse_bare_cr_in_raw_string = bare CR not allowed in raw string
+
+parse_too_short_hex_escape = numeric character escape is too short
+
+parse_invalid_char_in_escape = {parse_invalid_char_in_escape_msg}: `{$ch}`
+    .label = {parse_invalid_char_in_escape_msg}
+
+parse_invalid_char_in_escape_msg = invalid character in {$is_hex ->
+    [true] numeric character
+    *[false] unicode
+    } escape
+
+parse_out_of_range_hex_escape = out of range hex escape
+    .label = must be a character in the range [\x00-\x7f]
+
+parse_leading_underscore_unicode_escape = {parse_leading_underscore_unicode_escape_label}: `_`
+parse_leading_underscore_unicode_escape_label = invalid start of unicode escape
+
+parse_overlong_unicode_escape = overlong unicode escape
+    .label = must have at most 6 hex digits
+
+parse_unclosed_unicode_escape = unterminated unicode escape
+    .label = missing a closing `{"}"}`
+    .terminate = terminate the unicode escape
+
+parse_unicode_escape_in_byte = unicode escape in byte string
+    .label = {parse_unicode_escape_in_byte}
+    .help = unicode escape sequences cannot be used as a byte or in a byte string
+
+parse_empty_unicode_escape = empty unicode escape
+    .label = this escape must have at least 1 hex digit
+
+parse_zero_chars = empty character literal
+    .label = {parse_zero_chars}
+
+parse_lone_slash = invalid trailing slash in literal
+    .label = {parse_lone_slash}
+
+parse_unskipped_whitespace = non-ASCII whitespace symbol '{$ch}' is not skipped
+    .label = {parse_unskipped_whitespace}
+
+parse_multiple_skipped_lines = multiple lines skipped by escaped newline
+    .label = skipping everything up to and including this point
+
+parse_unknown_prefix = prefix `{$prefix}` is unknown
+    .label = unknown prefix
+    .note =  prefixed identifiers and literals are reserved since Rust 2021
+    .suggestion_br = use `br` for a raw byte string
+    .suggestion_whitespace = consider inserting whitespace here
+
+parse_too_many_hashes = too many `#` symbols: raw strings may be delimited by up to 255 `#` symbols, but found {$num}
+
+parse_unknown_start_of_token = unknown start of token: {$escaped}
+    .sugg_quotes = Unicode characters '“' (Left Double Quotation Mark) and '”' (Right Double Quotation Mark) look like '{$ascii_str}' ({$ascii_name}), but are not
+    .sugg_other = Unicode character '{$ch}' ({$u_name}) looks like '{$ascii_str}' ({$ascii_name}), but it is not
+    .help_null = source files must contain UTF-8 encoded text, unexpected null bytes might occur when a different encoding is used
+    .note_repeats = character appears {$repeats ->
+        [one] once more
+        *[other] {$repeats} more times
+    }
diff --git a/compiler/rustc_parse/src/errors.rs b/compiler/rustc_parse/src/errors.rs
index 06b970ad9..1662db36d 100644
--- a/compiler/rustc_parse/src/errors.rs
+++ b/compiler/rustc_parse/src/errors.rs
@@ -1,11 +1,15 @@
+use std::borrow::Cow;
+
 use rustc_ast::token::Token;
-use rustc_ast::Path;
-use rustc_errors::{fluent, AddToDiagnostic, Applicability, EmissionGuarantee, IntoDiagnostic};
+use rustc_ast::{Path, Visibility};
+use rustc_errors::{AddToDiagnostic, Applicability, EmissionGuarantee, IntoDiagnostic};
 use rustc_macros::{Diagnostic, Subdiagnostic};
 use rustc_session::errors::ExprParenthesesNeeded;
+use rustc_span::edition::{Edition, LATEST_STABLE_EDITION};
 use rustc_span::symbol::Ident;
 use rustc_span::{Span, Symbol};
 
+use crate::fluent_generated as fluent;
 use crate::parser::TokenDescription;
 
 #[derive(Diagnostic)]
@@ -75,7 +79,7 @@ pub(crate) struct IncorrectSemicolon<'a> {
 #[diag(parse_incorrect_use_of_await)]
 pub(crate) struct IncorrectUseOfAwait {
     #[primary_span]
-    #[suggestion(parentheses_suggestion, code = "", applicability = "machine-applicable")]
+    #[suggestion(parse_parentheses_suggestion, code = "", applicability = "machine-applicable")]
     pub span: Span,
 }
 
@@ -84,7 +88,7 @@ pub(crate) struct IncorrectUseOfAwait {
 pub(crate) struct IncorrectAwait {
     #[primary_span]
     pub span: Span,
-    #[suggestion(postfix_suggestion, code = "{expr}.await{question_mark}")]
+    #[suggestion(parse_postfix_suggestion, code = "{expr}.await{question_mark}")]
     pub sugg_span: (Span, Applicability),
     pub expr: String,
     pub question_mark: &'static str,
@@ -137,7 +141,7 @@ pub(crate) struct InvalidComparisonOperator {
 #[derive(Subdiagnostic)]
 pub(crate) enum InvalidComparisonOperatorSub {
     #[suggestion(
-        use_instead,
+        parse_use_instead,
         style = "short",
         applicability = "machine-applicable",
         code = "{correct}"
@@ -148,7 +152,7 @@ pub(crate) enum InvalidComparisonOperatorSub {
         invalid: String,
         correct: String,
     },
-    #[label(spaceship_operator_invalid)]
+    #[label(parse_spaceship_operator_invalid)]
     Spaceship(#[primary_span] Span),
 }
 
@@ -166,14 +170,14 @@ pub(crate) struct InvalidLogicalOperator {
 #[derive(Subdiagnostic)]
 pub(crate) enum InvalidLogicalOperatorSub {
     #[suggestion(
-        use_amp_amp_for_conjunction,
+        parse_use_amp_amp_for_conjunction,
         style = "short",
         applicability = "machine-applicable",
         code = "&&"
     )]
     Conjunction(#[primary_span] Span),
     #[suggestion(
-        use_pipe_pipe_for_disjunction,
+        parse_use_pipe_pipe_for_disjunction,
         style = "short",
         applicability = "machine-applicable",
         code = "||"
@@ -259,14 +263,14 @@ pub(crate) struct UnexpectedTokenAfterLabel {
     #[primary_span]
     #[label(parse_unexpected_token_after_label)]
     pub span: Span,
-    #[suggestion(suggestion_remove_label, style = "verbose", code = "")]
+    #[suggestion(parse_suggestion_remove_label, style = "verbose", code = "")]
     pub remove_label: Option<Span>,
     #[subdiagnostic]
     pub enclose_in_block: Option<UnexpectedTokenAfterLabelSugg>,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(suggestion_enclose_in_block, applicability = "machine-applicable")]
+#[multipart_suggestion(parse_suggestion_enclose_in_block, applicability = "machine-applicable")]
 pub(crate) struct UnexpectedTokenAfterLabelSugg {
     #[suggestion_part(code = "{{ ")]
     pub left: Span,
@@ -337,24 +341,33 @@ pub(crate) struct IfExpressionMissingThenBlock {
     #[primary_span]
     pub if_span: Span,
     #[subdiagnostic]
-    pub sub: IfExpressionMissingThenBlockSub,
+    pub missing_then_block_sub: IfExpressionMissingThenBlockSub,
+    #[subdiagnostic]
+    pub let_else_sub: Option<IfExpressionLetSomeSub>,
 }
 
 #[derive(Subdiagnostic)]
 pub(crate) enum IfExpressionMissingThenBlockSub {
-    #[help(condition_possibly_unfinished)]
+    #[help(parse_condition_possibly_unfinished)]
     UnfinishedCondition(#[primary_span] Span),
-    #[help(add_then_block)]
+    #[help(parse_add_then_block)]
     AddThenBlock(#[primary_span] Span),
 }
 
+#[derive(Subdiagnostic)]
+#[suggestion(parse_extra_if_in_let_else, applicability = "maybe-incorrect", code = "")]
+pub(crate) struct IfExpressionLetSomeSub {
+    #[primary_span]
+    pub if_span: Span,
+}
+
 #[derive(Diagnostic)]
 #[diag(parse_if_expression_missing_condition)]
 pub(crate) struct IfExpressionMissingCondition {
     #[primary_span]
-    #[label(condition_label)]
+    #[label(parse_condition_label)]
     pub if_span: Span,
-    #[label(block_label)]
+    #[label(parse_block_label)]
     pub block_span: Span,
 }
 
@@ -392,10 +405,10 @@ pub(crate) struct OuterAttributeNotAllowedOnIfElse {
     #[primary_span]
     pub last: Span,
 
-    #[label(branch_label)]
+    #[label(parse_branch_label)]
     pub branch_span: Span,
 
-    #[label(ctx_label)]
+    #[label(parse_ctx_label)]
     pub ctx_span: Span,
     pub ctx: String,
 
@@ -415,13 +428,41 @@ pub(crate) struct MissingInInForLoop {
 #[derive(Subdiagnostic)]
 pub(crate) enum MissingInInForLoopSub {
     // Has been misleading, at least in the past (closed Issue #48492), thus maybe-incorrect
-    #[suggestion(use_in_not_of, style = "short", applicability = "maybe-incorrect", code = "in")]
+    #[suggestion(
+        parse_use_in_not_of,
+        style = "short",
+        applicability = "maybe-incorrect",
+        code = "in"
+    )]
     InNotOf(#[primary_span] Span),
-    #[suggestion(add_in, style = "short", applicability = "maybe-incorrect", code = " in ")]
+    #[suggestion(parse_add_in, style = "short", applicability = "maybe-incorrect", code = " in ")]
     AddIn(#[primary_span] Span),
 }
 
 #[derive(Diagnostic)]
+#[diag(parse_missing_expression_in_for_loop)]
+pub(crate) struct MissingExpressionInForLoop {
+    #[primary_span]
+    #[suggestion(
+        code = "/* expression */ ",
+        applicability = "has-placeholders",
+        style = "verbose"
+    )]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_loop_else)]
+#[note]
+pub(crate) struct LoopElseNotSupported {
+    #[primary_span]
+    pub span: Span,
+    pub loop_kind: &'static str,
+    #[label(parse_loop_keyword)]
+    pub loop_kw: Span,
+}
+
+#[derive(Diagnostic)]
 #[diag(parse_missing_comma_after_match_arm)]
 pub(crate) struct MissingCommaAfterMatchArm {
     #[primary_span]
@@ -460,8 +501,8 @@ pub(crate) struct EqFieldInit {
 #[diag(parse_dotdotdot)]
 pub(crate) struct DotDotDot {
     #[primary_span]
-    #[suggestion(suggest_exclusive_range, applicability = "maybe-incorrect", code = "..")]
-    #[suggestion(suggest_inclusive_range, applicability = "maybe-incorrect", code = "..=")]
+    #[suggestion(parse_suggest_exclusive_range, applicability = "maybe-incorrect", code = "..")]
+    #[suggestion(parse_suggest_inclusive_range, applicability = "maybe-incorrect", code = "..=")]
     pub span: Span,
 }
 
@@ -501,10 +542,10 @@ pub(crate) struct UseEmptyBlockNotSemi {
 #[diag(parse_comparison_interpreted_as_generic)]
 pub(crate) struct ComparisonInterpretedAsGeneric {
     #[primary_span]
-    #[label(label_comparison)]
+    #[label(parse_label_comparison)]
     pub comparison: Span,
     pub r#type: Path,
-    #[label(label_args)]
+    #[label(parse_label_args)]
     pub args: Span,
     #[subdiagnostic]
     pub suggestion: ComparisonOrShiftInterpretedAsGenericSugg,
@@ -514,17 +555,17 @@ pub(crate) struct ComparisonInterpretedAsGeneric {
 #[diag(parse_shift_interpreted_as_generic)]
 pub(crate) struct ShiftInterpretedAsGeneric {
     #[primary_span]
-    #[label(label_comparison)]
+    #[label(parse_label_comparison)]
     pub shift: Span,
     pub r#type: Path,
-    #[label(label_args)]
+    #[label(parse_label_args)]
     pub args: Span,
     #[subdiagnostic]
     pub suggestion: ComparisonOrShiftInterpretedAsGenericSugg,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
 pub(crate) struct ComparisonOrShiftInterpretedAsGenericSugg {
     #[suggestion_part(code = "(")]
     pub left: Span,
@@ -550,7 +591,7 @@ pub(crate) struct LeadingPlusNotSupported {
     #[label]
     pub span: Span,
     #[suggestion(
-        suggestion_remove_plus,
+        parse_suggestion_remove_plus,
         style = "verbose",
         code = "",
         applicability = "machine-applicable"
@@ -573,7 +614,7 @@ pub(crate) struct ParenthesesWithStructFields {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(suggestion_braces_for_struct, applicability = "maybe-incorrect")]
+#[multipart_suggestion(parse_suggestion_braces_for_struct, applicability = "maybe-incorrect")]
 pub(crate) struct BracesForStructLiteral {
     #[suggestion_part(code = " {{ ")]
     pub first: Span,
@@ -582,7 +623,7 @@ pub(crate) struct BracesForStructLiteral {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(suggestion_no_fields_for_fn, applicability = "maybe-incorrect")]
+#[multipart_suggestion(parse_suggestion_no_fields_for_fn, applicability = "maybe-incorrect")]
 pub(crate) struct NoFieldsForFnCall {
     #[suggestion_part(code = "")]
     pub fields: Vec<Span>,
@@ -619,7 +660,7 @@ pub(crate) struct ArrayBracketsInsteadOfSpaces {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(suggestion, applicability = "maybe-incorrect")]
+#[multipart_suggestion(parse_suggestion, applicability = "maybe-incorrect")]
 pub(crate) struct ArrayBracketsInsteadOfSpacesSugg {
     #[suggestion_part(code = "[")]
     pub left: Span,
@@ -631,18 +672,57 @@ pub(crate) struct ArrayBracketsInsteadOfSpacesSugg {
 #[diag(parse_match_arm_body_without_braces)]
 pub(crate) struct MatchArmBodyWithoutBraces {
     #[primary_span]
-    #[label(label_statements)]
+    #[label(parse_label_statements)]
     pub statements: Span,
-    #[label(label_arrow)]
+    #[label(parse_label_arrow)]
     pub arrow: Span,
     pub num_statements: usize,
     #[subdiagnostic]
     pub sub: MatchArmBodyWithoutBracesSugg,
 }
 
+#[derive(Diagnostic)]
+#[diag(parse_inclusive_range_extra_equals)]
+#[note]
+pub(crate) struct InclusiveRangeExtraEquals {
+    #[primary_span]
+    #[suggestion(
+        parse_suggestion_remove_eq,
+        style = "short",
+        code = "..=",
+        applicability = "maybe-incorrect"
+    )]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_inclusive_range_match_arrow)]
+pub(crate) struct InclusiveRangeMatchArrow {
+    #[primary_span]
+    pub arrow: Span,
+    #[label]
+    pub span: Span,
+    #[suggestion(style = "verbose", code = " ", applicability = "machine-applicable")]
+    pub after_pat: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_inclusive_range_no_end, code = "E0586")]
+#[note]
+pub(crate) struct InclusiveRangeNoEnd {
+    #[primary_span]
+    #[suggestion(
+        parse_suggestion_open_range,
+        code = "..",
+        applicability = "machine-applicable",
+        style = "short"
+    )]
+    pub span: Span,
+}
+
 #[derive(Subdiagnostic)]
 pub(crate) enum MatchArmBodyWithoutBracesSugg {
-    #[multipart_suggestion(suggestion_add_braces, applicability = "machine-applicable")]
+    #[multipart_suggestion(parse_suggestion_add_braces, applicability = "machine-applicable")]
     AddBraces {
         #[suggestion_part(code = "{{ ")]
         left: Span,
@@ -650,7 +730,7 @@ pub(crate) enum MatchArmBodyWithoutBracesSugg {
         right: Span,
     },
     #[suggestion(
-        suggestion_use_comma_not_semicolon,
+        parse_suggestion_use_comma_not_semicolon,
         code = ",",
         applicability = "machine-applicable"
     )]
@@ -670,7 +750,7 @@ pub(crate) struct StructLiteralNotAllowedHere {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
 pub(crate) struct StructLiteralNotAllowedHereSugg {
     #[suggestion_part(code = "(")]
     pub left: Span,
@@ -692,9 +772,9 @@ pub(crate) struct InvalidLiteralSuffixOnTupleIndex {
     #[label]
     pub span: Span,
     pub suffix: Symbol,
-    #[help(tuple_exception_line_1)]
-    #[help(tuple_exception_line_2)]
-    #[help(tuple_exception_line_3)]
+    #[help(parse_tuple_exception_line_1)]
+    #[help(parse_tuple_exception_line_2)]
+    #[help(parse_tuple_exception_line_3)]
     pub exception: Option<()>,
 }
 
@@ -712,11 +792,11 @@ pub(crate) struct MismatchedClosingDelimiter {
     #[primary_span]
     pub spans: Vec<Span>,
     pub delimiter: String,
-    #[label(label_unmatched)]
+    #[label(parse_label_unmatched)]
     pub unmatched: Span,
-    #[label(label_opening_candidate)]
+    #[label(parse_label_opening_candidate)]
     pub opening_candidate: Option<Span>,
-    #[label(label_unclosed)]
+    #[label(parse_label_unclosed)]
     pub unclosed: Option<Span>,
 }
 
@@ -867,7 +947,7 @@ impl<'a, G: EmissionGuarantee> IntoDiagnostic<'a, G> for ExpectedIdentifier {
         self,
         handler: &'a rustc_errors::Handler,
     ) -> rustc_errors::DiagnosticBuilder<'a, G> {
-        let token_descr = super::parser::TokenDescription::from_token(&self.token);
+        let token_descr = TokenDescription::from_token(&self.token);
 
         let mut diag = handler.struct_diagnostic(match token_descr {
             Some(TokenDescription::ReservedIdentifier) => {
@@ -913,7 +993,7 @@ impl<'a, G: EmissionGuarantee> IntoDiagnostic<'a, G> for ExpectedSemi {
         self,
         handler: &'a rustc_errors::Handler,
     ) -> rustc_errors::DiagnosticBuilder<'a, G> {
-        let token_descr = super::parser::TokenDescription::from_token(&self.token);
+        let token_descr = TokenDescription::from_token(&self.token);
 
         let mut diag = handler.struct_diagnostic(match token_descr {
             Some(TokenDescription::ReservedIdentifier) => {
@@ -962,7 +1042,7 @@ pub(crate) struct StructLiteralBodyWithoutPath {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(suggestion, applicability = "has-placeholders")]
+#[multipart_suggestion(parse_suggestion, applicability = "has-placeholders")]
 pub(crate) struct StructLiteralBodyWithoutPathSugg {
     #[suggestion_part(code = "{{ SomeStruct ")]
     pub before: Span,
@@ -980,7 +1060,7 @@ pub(crate) struct StructLiteralNeedingParens {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
 pub(crate) struct StructLiteralNeedingParensSugg {
     #[suggestion_part(code = "(")]
     pub before: Span,
@@ -1007,7 +1087,7 @@ pub(crate) struct GenericParamsWithoutAngleBrackets {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
 pub(crate) struct GenericParamsWithoutAngleBracketsSugg {
     #[suggestion_part(code = "<")]
     pub left: Span,
@@ -1028,7 +1108,7 @@ pub(crate) struct ComparisonOperatorsCannotBeChained {
     )]
     pub suggest_turbofish: Option<Span>,
     #[help(parse_sugg_turbofish_syntax)]
-    #[help(sugg_parentheses_for_function_args)]
+    #[help(parse_sugg_parentheses_for_function_args)]
     pub help_turbofish: Option<()>,
     #[subdiagnostic]
     pub chaining_sugg: Option<ComparisonOperatorsCannotBeChainedSugg>,
@@ -1037,7 +1117,7 @@ pub(crate) struct ComparisonOperatorsCannotBeChained {
 #[derive(Subdiagnostic)]
 pub(crate) enum ComparisonOperatorsCannotBeChainedSugg {
     #[suggestion(
-        sugg_split_comparison,
+        parse_sugg_split_comparison,
         style = "verbose",
         code = " && {middle_term}",
         applicability = "maybe-incorrect"
@@ -1047,7 +1127,7 @@ pub(crate) enum ComparisonOperatorsCannotBeChainedSugg {
         span: Span,
         middle_term: String,
     },
-    #[multipart_suggestion(sugg_parenthesize, applicability = "maybe-incorrect")]
+    #[multipart_suggestion(parse_sugg_parenthesize, applicability = "maybe-incorrect")]
     Parenthesize {
         #[suggestion_part(code = "(")]
         left: Span,
@@ -1067,7 +1147,7 @@ pub(crate) struct QuestionMarkInType {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
 pub(crate) struct QuestionMarkInTypeSugg {
     #[suggestion_part(code = "Option<")]
     pub left: Span,
@@ -1085,7 +1165,7 @@ pub(crate) struct ParenthesesInForHead {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
 pub(crate) struct ParenthesesInForHeadSugg {
     #[suggestion_part(code = "{left_snippet}")]
     pub left: Span,
@@ -1145,7 +1225,7 @@ pub(crate) struct ConstGenericWithoutBraces {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
 pub(crate) struct ConstGenericWithoutBracesSugg {
     #[suggestion_part(code = "{{ ")]
     pub left: Span,
@@ -1165,7 +1245,7 @@ pub(crate) struct UnexpectedConstParamDeclaration {
 
 #[derive(Subdiagnostic)]
 pub(crate) enum UnexpectedConstParamDeclarationSugg {
-    #[multipart_suggestion(suggestion, applicability = "machine-applicable")]
+    #[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
     AddParam {
         #[suggestion_part(code = "<{snippet}>")]
         impl_generics: Span,
@@ -1174,7 +1254,7 @@ pub(crate) enum UnexpectedConstParamDeclarationSugg {
         snippet: String,
         ident: String,
     },
-    #[multipart_suggestion(suggestion, applicability = "machine-applicable")]
+    #[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
     AppendParam {
         #[suggestion_part(code = ", {snippet}")]
         impl_generics_end: Span,
@@ -1221,7 +1301,7 @@ pub(crate) struct FnPtrWithGenerics {
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(suggestion, applicability = "maybe-incorrect")]
+#[multipart_suggestion(parse_suggestion, applicability = "maybe-incorrect")]
 pub(crate) struct FnPtrWithGenericsSugg {
     #[suggestion_part(code = "{snippet}")]
     pub left: Span,
@@ -1262,16 +1342,16 @@ pub(crate) struct WhereClauseBeforeTupleStructBody {
     #[primary_span]
     #[label]
     pub span: Span,
-    #[label(name_label)]
+    #[label(parse_name_label)]
     pub name: Span,
-    #[label(body_label)]
+    #[label(parse_body_label)]
     pub body: Span,
     #[subdiagnostic]
     pub sugg: Option<WhereClauseBeforeTupleStructBodySugg>,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(suggestion, applicability = "machine-applicable")]
+#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
 pub(crate) struct WhereClauseBeforeTupleStructBodySugg {
     #[suggestion_part(code = "{snippet}")]
     pub left: Span,
@@ -1279,3 +1359,943 @@ pub(crate) struct WhereClauseBeforeTupleStructBodySugg {
     #[suggestion_part(code = "")]
     pub right: Span,
 }
+
+#[derive(Diagnostic)]
+#[diag(parse_async_fn_in_2015, code = "E0670")]
+pub(crate) struct AsyncFnIn2015 {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    #[subdiagnostic]
+    pub help: HelpUseLatestEdition,
+}
+
+#[derive(Subdiagnostic)]
+#[label(parse_async_block_in_2015)]
+pub(crate) struct AsyncBlockIn2015 {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_self_argument_pointer)]
+pub(crate) struct SelfArgumentPointer {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_unexpected_token_after_dot)]
+pub struct UnexpectedTokenAfterDot<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub actual: Cow<'a, str>,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_visibility_not_followed_by_item)]
+#[help]
+pub(crate) struct VisibilityNotFollowedByItem {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    pub vis: Visibility,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_default_not_followed_by_item)]
+#[note]
+pub(crate) struct DefaultNotFollowedByItem {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+pub(crate) enum MissingKeywordForItemDefinition {
+    #[diag(parse_missing_struct_for_struct_definition)]
+    Struct {
+        #[primary_span]
+        #[suggestion(style = "short", applicability = "maybe-incorrect", code = " struct ")]
+        span: Span,
+        ident: Ident,
+    },
+    #[diag(parse_missing_fn_for_function_definition)]
+    Function {
+        #[primary_span]
+        #[suggestion(style = "short", applicability = "maybe-incorrect", code = " fn ")]
+        span: Span,
+        ident: Ident,
+    },
+    #[diag(parse_missing_fn_for_method_definition)]
+    Method {
+        #[primary_span]
+        #[suggestion(style = "short", applicability = "maybe-incorrect", code = " fn ")]
+        span: Span,
+        ident: Ident,
+    },
+    #[diag(parse_ambiguous_missing_keyword_for_item_definition)]
+    Ambiguous {
+        #[primary_span]
+        span: Span,
+        #[subdiagnostic]
+        subdiag: Option<AmbiguousMissingKwForItemSub>,
+    },
+}
+
+#[derive(Subdiagnostic)]
+pub(crate) enum AmbiguousMissingKwForItemSub {
+    #[suggestion(parse_suggestion, applicability = "maybe-incorrect", code = "{snippet}!")]
+    SuggestMacro {
+        #[primary_span]
+        span: Span,
+        snippet: String,
+    },
+    #[help(parse_help)]
+    HelpMacro,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_missing_trait_in_trait_impl)]
+pub(crate) struct MissingTraitInTraitImpl {
+    #[primary_span]
+    #[suggestion(parse_suggestion_add_trait, code = " Trait ", applicability = "has-placeholders")]
+    pub span: Span,
+    #[suggestion(parse_suggestion_remove_for, code = "", applicability = "maybe-incorrect")]
+    pub for_span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_missing_for_in_trait_impl)]
+pub(crate) struct MissingForInTraitImpl {
+    #[primary_span]
+    #[suggestion(style = "short", code = " for ", applicability = "machine-applicable")]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_expected_trait_in_trait_impl_found_type)]
+pub(crate) struct ExpectedTraitInTraitImplFoundType {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_bounds_not_allowed_on_trait_aliases)]
+pub(crate) struct BoundsNotAllowedOnTraitAliases {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_trait_alias_cannot_be_auto)]
+pub(crate) struct TraitAliasCannotBeAuto {
+    #[primary_span]
+    #[label(parse_trait_alias_cannot_be_auto)]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_trait_alias_cannot_be_unsafe)]
+pub(crate) struct TraitAliasCannotBeUnsafe {
+    #[primary_span]
+    #[label(parse_trait_alias_cannot_be_unsafe)]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_associated_static_item_not_allowed)]
+pub(crate) struct AssociatedStaticItemNotAllowed {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_extern_crate_name_with_dashes)]
+pub(crate) struct ExternCrateNameWithDashes {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    #[subdiagnostic]
+    pub sugg: ExternCrateNameWithDashesSugg,
+}
+
+#[derive(Subdiagnostic)]
+#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
+pub(crate) struct ExternCrateNameWithDashesSugg {
+    #[suggestion_part(code = "_")]
+    pub dashes: Vec<Span>,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_extern_item_cannot_be_const)]
+#[note]
+pub(crate) struct ExternItemCannotBeConst {
+    #[primary_span]
+    pub ident_span: Span,
+    #[suggestion(code = "static ", applicability = "machine-applicable")]
+    pub const_span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_const_global_cannot_be_mutable)]
+pub(crate) struct ConstGlobalCannotBeMutable {
+    #[primary_span]
+    #[label]
+    pub ident_span: Span,
+    #[suggestion(code = "static", applicability = "maybe-incorrect")]
+    pub const_span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_missing_const_type)]
+pub(crate) struct MissingConstType {
+    #[primary_span]
+    #[suggestion(code = "{colon} <type>", applicability = "has-placeholders")]
+    pub span: Span,
+
+    pub kind: &'static str,
+    pub colon: &'static str,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_enum_struct_mutually_exclusive)]
+pub(crate) struct EnumStructMutuallyExclusive {
+    #[primary_span]
+    #[suggestion(code = "enum", applicability = "machine-applicable")]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+pub(crate) enum UnexpectedTokenAfterStructName {
+    #[diag(parse_unexpected_token_after_struct_name_found_reserved_identifier)]
+    ReservedIdentifier {
+        #[primary_span]
+        #[label(parse_unexpected_token_after_struct_name)]
+        span: Span,
+        token: Token,
+    },
+    #[diag(parse_unexpected_token_after_struct_name_found_keyword)]
+    Keyword {
+        #[primary_span]
+        #[label(parse_unexpected_token_after_struct_name)]
+        span: Span,
+        token: Token,
+    },
+    #[diag(parse_unexpected_token_after_struct_name_found_reserved_keyword)]
+    ReservedKeyword {
+        #[primary_span]
+        #[label(parse_unexpected_token_after_struct_name)]
+        span: Span,
+        token: Token,
+    },
+    #[diag(parse_unexpected_token_after_struct_name_found_doc_comment)]
+    DocComment {
+        #[primary_span]
+        #[label(parse_unexpected_token_after_struct_name)]
+        span: Span,
+        token: Token,
+    },
+    #[diag(parse_unexpected_token_after_struct_name_found_other)]
+    Other {
+        #[primary_span]
+        #[label(parse_unexpected_token_after_struct_name)]
+        span: Span,
+        token: Token,
+    },
+}
+
+impl UnexpectedTokenAfterStructName {
+    pub fn new(span: Span, token: Token) -> Self {
+        match TokenDescription::from_token(&token) {
+            Some(TokenDescription::ReservedIdentifier) => Self::ReservedIdentifier { span, token },
+            Some(TokenDescription::Keyword) => Self::Keyword { span, token },
+            Some(TokenDescription::ReservedKeyword) => Self::ReservedKeyword { span, token },
+            Some(TokenDescription::DocComment) => Self::DocComment { span, token },
+            None => Self::Other { span, token },
+        }
+    }
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_unexpected_self_in_generic_parameters)]
+#[note]
+pub(crate) struct UnexpectedSelfInGenericParameters {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_unexpected_default_value_for_lifetime_in_generic_parameters)]
+pub(crate) struct UnexpectedDefaultValueForLifetimeInGenericParameters {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_multiple_where_clauses)]
+pub(crate) struct MultipleWhereClauses {
+    #[primary_span]
+    pub span: Span,
+    #[label]
+    pub previous: Span,
+    #[suggestion(style = "verbose", code = ",", applicability = "maybe-incorrect")]
+    pub between: Span,
+}
+
+#[derive(Diagnostic)]
+pub(crate) enum UnexpectedNonterminal {
+    #[diag(parse_nonterminal_expected_item_keyword)]
+    Item(#[primary_span] Span),
+    #[diag(parse_nonterminal_expected_statement)]
+    Statement(#[primary_span] Span),
+    #[diag(parse_nonterminal_expected_ident)]
+    Ident {
+        #[primary_span]
+        span: Span,
+        token: Token,
+    },
+    #[diag(parse_nonterminal_expected_lifetime)]
+    Lifetime {
+        #[primary_span]
+        span: Span,
+        token: Token,
+    },
+}
+
+#[derive(Diagnostic)]
+pub(crate) enum TopLevelOrPatternNotAllowed {
+    #[diag(parse_or_pattern_not_allowed_in_let_binding)]
+    LetBinding {
+        #[primary_span]
+        span: Span,
+        #[subdiagnostic]
+        sub: Option<TopLevelOrPatternNotAllowedSugg>,
+    },
+    #[diag(parse_or_pattern_not_allowed_in_fn_parameters)]
+    FunctionParameter {
+        #[primary_span]
+        span: Span,
+        #[subdiagnostic]
+        sub: Option<TopLevelOrPatternNotAllowedSugg>,
+    },
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_cannot_be_raw_ident)]
+pub struct CannotBeRawIdent {
+    #[primary_span]
+    pub span: Span,
+    pub ident: Symbol,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_cr_doc_comment)]
+pub struct CrDocComment {
+    #[primary_span]
+    pub span: Span,
+    pub block: bool,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_no_digits_literal, code = "E0768")]
+pub struct NoDigitsLiteral {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_invalid_digit_literal)]
+pub struct InvalidDigitLiteral {
+    #[primary_span]
+    pub span: Span,
+    pub base: u32,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_empty_exponent_float)]
+pub struct EmptyExponentFloat {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_float_literal_unsupported_base)]
+pub struct FloatLiteralUnsupportedBase {
+    #[primary_span]
+    pub span: Span,
+    pub base: &'static str,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_unknown_prefix)]
+#[note]
+pub struct UnknownPrefix<'a> {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    pub prefix: &'a str,
+    #[subdiagnostic]
+    pub sugg: Option<UnknownPrefixSugg>,
+}
+
+#[derive(Subdiagnostic)]
+pub enum UnknownPrefixSugg {
+    #[suggestion(
+        parse_suggestion_br,
+        code = "br",
+        applicability = "maybe-incorrect",
+        style = "verbose"
+    )]
+    UseBr(#[primary_span] Span),
+    #[suggestion(
+        parse_suggestion_whitespace,
+        code = " ",
+        applicability = "maybe-incorrect",
+        style = "verbose"
+    )]
+    Whitespace(#[primary_span] Span),
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_too_many_hashes)]
+pub struct TooManyHashes {
+    #[primary_span]
+    pub span: Span,
+    pub num: u32,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_unknown_start_of_token)]
+pub struct UnknownTokenStart {
+    #[primary_span]
+    pub span: Span,
+    pub escaped: String,
+    #[subdiagnostic]
+    pub sugg: Option<TokenSubstitution>,
+    #[subdiagnostic]
+    pub null: Option<UnknownTokenNull>,
+    #[subdiagnostic]
+    pub repeat: Option<UnknownTokenRepeat>,
+}
+
+#[derive(Subdiagnostic)]
+pub enum TokenSubstitution {
+    #[suggestion(parse_sugg_quotes, code = "{suggestion}", applicability = "maybe-incorrect")]
+    DirectedQuotes {
+        #[primary_span]
+        span: Span,
+        suggestion: String,
+        ascii_str: &'static str,
+        ascii_name: &'static str,
+    },
+    #[suggestion(parse_sugg_other, code = "{suggestion}", applicability = "maybe-incorrect")]
+    Other {
+        #[primary_span]
+        span: Span,
+        suggestion: String,
+        ch: String,
+        u_name: &'static str,
+        ascii_str: &'static str,
+        ascii_name: &'static str,
+    },
+}
+
+#[derive(Subdiagnostic)]
+#[note(parse_note_repeats)]
+pub struct UnknownTokenRepeat {
+    pub repeats: usize,
+}
+
+#[derive(Subdiagnostic)]
+#[help(parse_help_null)]
+pub struct UnknownTokenNull;
+
+#[derive(Diagnostic)]
+pub enum UnescapeError {
+    #[diag(parse_invalid_unicode_escape)]
+    #[help]
+    InvalidUnicodeEscape {
+        #[primary_span]
+        #[label]
+        span: Span,
+        surrogate: bool,
+    },
+    #[diag(parse_escape_only_char)]
+    EscapeOnlyChar {
+        #[primary_span]
+        span: Span,
+        #[suggestion(parse_escape, applicability = "machine-applicable", code = "{escaped_sugg}")]
+        char_span: Span,
+        escaped_sugg: String,
+        escaped_msg: String,
+        byte: bool,
+    },
+    #[diag(parse_bare_cr)]
+    BareCr {
+        #[primary_span]
+        #[suggestion(parse_escape, applicability = "machine-applicable", code = "\\r")]
+        span: Span,
+        double_quotes: bool,
+    },
+    #[diag(parse_bare_cr_in_raw_string)]
+    BareCrRawString(#[primary_span] Span),
+    #[diag(parse_too_short_hex_escape)]
+    TooShortHexEscape(#[primary_span] Span),
+    #[diag(parse_invalid_char_in_escape)]
+    InvalidCharInEscape {
+        #[primary_span]
+        #[label]
+        span: Span,
+        is_hex: bool,
+        ch: String,
+    },
+    #[diag(parse_out_of_range_hex_escape)]
+    OutOfRangeHexEscape(
+        #[primary_span]
+        #[label]
+        Span,
+    ),
+    #[diag(parse_leading_underscore_unicode_escape)]
+    LeadingUnderscoreUnicodeEscape {
+        #[primary_span]
+        #[label(parse_leading_underscore_unicode_escape_label)]
+        span: Span,
+        ch: String,
+    },
+    #[diag(parse_overlong_unicode_escape)]
+    OverlongUnicodeEscape(
+        #[primary_span]
+        #[label]
+        Span,
+    ),
+    #[diag(parse_unclosed_unicode_escape)]
+    UnclosedUnicodeEscape(
+        #[primary_span]
+        #[label]
+        Span,
+        #[suggestion(
+            parse_terminate,
+            code = "}}",
+            applicability = "maybe-incorrect",
+            style = "verbose"
+        )]
+        Span,
+    ),
+    #[diag(parse_no_brace_unicode_escape)]
+    NoBraceInUnicodeEscape {
+        #[primary_span]
+        span: Span,
+        #[label]
+        label: Option<Span>,
+        #[subdiagnostic]
+        sub: NoBraceUnicodeSub,
+    },
+    #[diag(parse_unicode_escape_in_byte)]
+    #[help]
+    UnicodeEscapeInByte(
+        #[primary_span]
+        #[label]
+        Span,
+    ),
+    #[diag(parse_empty_unicode_escape)]
+    EmptyUnicodeEscape(
+        #[primary_span]
+        #[label]
+        Span,
+    ),
+    #[diag(parse_zero_chars)]
+    ZeroChars(
+        #[primary_span]
+        #[label]
+        Span,
+    ),
+    #[diag(parse_lone_slash)]
+    LoneSlash(
+        #[primary_span]
+        #[label]
+        Span,
+    ),
+    #[diag(parse_unskipped_whitespace)]
+    UnskippedWhitespace {
+        #[primary_span]
+        span: Span,
+        #[label]
+        char_span: Span,
+        ch: String,
+    },
+    #[diag(parse_multiple_skipped_lines)]
+    MultipleSkippedLinesWarning(
+        #[primary_span]
+        #[label]
+        Span,
+    ),
+    #[diag(parse_more_than_one_char)]
+    MoreThanOneChar {
+        #[primary_span]
+        span: Span,
+        #[subdiagnostic]
+        note: Option<MoreThanOneCharNote>,
+        #[subdiagnostic]
+        suggestion: MoreThanOneCharSugg,
+    },
+}
+
+#[derive(Subdiagnostic)]
+pub enum MoreThanOneCharSugg {
+    #[suggestion(
+        parse_consider_normalized,
+        code = "{normalized}",
+        applicability = "machine-applicable"
+    )]
+    NormalizedForm {
+        #[primary_span]
+        span: Span,
+        ch: String,
+        normalized: String,
+    },
+    #[suggestion(parse_remove_non, code = "{ch}", applicability = "maybe-incorrect")]
+    RemoveNonPrinting {
+        #[primary_span]
+        span: Span,
+        ch: String,
+    },
+    #[suggestion(parse_use_double_quotes, code = "{sugg}", applicability = "machine-applicable")]
+    Quotes {
+        #[primary_span]
+        span: Span,
+        is_byte: bool,
+        sugg: String,
+    },
+}
+
+#[derive(Subdiagnostic)]
+pub enum MoreThanOneCharNote {
+    #[note(parse_followed_by)]
+    AllCombining {
+        #[primary_span]
+        span: Span,
+        chr: String,
+        len: usize,
+        escaped_marks: String,
+    },
+    #[note(parse_non_printing)]
+    NonPrinting {
+        #[primary_span]
+        span: Span,
+        escaped: String,
+    },
+}
+
+#[derive(Subdiagnostic)]
+pub enum NoBraceUnicodeSub {
+    #[suggestion(parse_use_braces, code = "{suggestion}", applicability = "maybe-incorrect")]
+    Suggestion {
+        #[primary_span]
+        span: Span,
+        suggestion: String,
+    },
+    #[help(parse_format_of_unicode)]
+    Help,
+}
+
+#[derive(Subdiagnostic)]
+pub(crate) enum TopLevelOrPatternNotAllowedSugg {
+    #[suggestion(
+        parse_sugg_remove_leading_vert_in_pattern,
+        code = "{pat}",
+        applicability = "machine-applicable"
+    )]
+    RemoveLeadingVert {
+        #[primary_span]
+        span: Span,
+        pat: String,
+    },
+    #[suggestion(
+        parse_sugg_wrap_pattern_in_parens,
+        code = "({pat})",
+        applicability = "machine-applicable"
+    )]
+    WrapInParens {
+        #[primary_span]
+        span: Span,
+        pat: String,
+    },
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_unexpected_vert_vert_before_function_parameter)]
+#[note(parse_note_pattern_alternatives_use_single_vert)]
+pub(crate) struct UnexpectedVertVertBeforeFunctionParam {
+    #[primary_span]
+    #[suggestion(code = "", applicability = "machine-applicable")]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_unexpected_vert_vert_in_pattern)]
+pub(crate) struct UnexpectedVertVertInPattern {
+    #[primary_span]
+    #[suggestion(code = "|", applicability = "machine-applicable")]
+    pub span: Span,
+    #[label(parse_label_while_parsing_or_pattern_here)]
+    pub start: Option<Span>,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_trailing_vert_not_allowed)]
+pub(crate) struct TrailingVertNotAllowed {
+    #[primary_span]
+    #[suggestion(code = "", applicability = "machine-applicable")]
+    pub span: Span,
+    #[label(parse_label_while_parsing_or_pattern_here)]
+    pub start: Option<Span>,
+    pub token: Token,
+    #[note(parse_note_pattern_alternatives_use_single_vert)]
+    pub note_double_vert: Option<()>,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_dotdotdot_rest_pattern)]
+pub(crate) struct DotDotDotRestPattern {
+    #[primary_span]
+    #[suggestion(style = "short", code = "..", applicability = "machine-applicable")]
+    #[label]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_pattern_on_wrong_side_of_at)]
+pub(crate) struct PatternOnWrongSideOfAt {
+    #[primary_span]
+    #[suggestion(code = "{whole_pat}", applicability = "machine-applicable")]
+    pub whole_span: Span,
+    pub whole_pat: String,
+    #[label(parse_label_pattern)]
+    pub pattern: Span,
+    #[label(parse_label_binding)]
+    pub binding: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_expected_binding_left_of_at)]
+#[note]
+pub(crate) struct ExpectedBindingLeftOfAt {
+    #[primary_span]
+    pub whole_span: Span,
+    #[label(parse_label_lhs)]
+    pub lhs: Span,
+    #[label(parse_label_rhs)]
+    pub rhs: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_ambiguous_range_pattern)]
+pub(crate) struct AmbiguousRangePattern {
+    #[primary_span]
+    #[suggestion(code = "({pat})", applicability = "maybe-incorrect")]
+    pub span: Span,
+    pub pat: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_unexpected_lifetime_in_pattern)]
+pub(crate) struct UnexpectedLifetimeInPattern {
+    #[primary_span]
+    #[suggestion(code = "", applicability = "machine-applicable")]
+    pub span: Span,
+    pub symbol: Symbol,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_ref_mut_order_incorrect)]
+pub(crate) struct RefMutOrderIncorrect {
+    #[primary_span]
+    #[suggestion(code = "ref mut", applicability = "machine-applicable")]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+pub(crate) enum InvalidMutInPattern {
+    #[diag(parse_mut_on_nested_ident_pattern)]
+    #[note(parse_note_mut_pattern_usage)]
+    NestedIdent {
+        #[primary_span]
+        #[suggestion(code = "{pat}", applicability = "machine-applicable")]
+        span: Span,
+        pat: String,
+    },
+    #[diag(parse_mut_on_non_ident_pattern)]
+    #[note(parse_note_mut_pattern_usage)]
+    NonIdent {
+        #[primary_span]
+        #[suggestion(code = "{pat}", applicability = "machine-applicable")]
+        span: Span,
+        pat: String,
+    },
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_repeated_mut_in_pattern)]
+pub(crate) struct RepeatedMutInPattern {
+    #[primary_span]
+    #[suggestion(code = "", applicability = "machine-applicable")]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_dot_dot_dot_range_to_pattern_not_allowed)]
+pub(crate) struct DotDotDotRangeToPatternNotAllowed {
+    #[primary_span]
+    #[suggestion(style = "short", code = "..=", applicability = "machine-applicable")]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_enum_pattern_instead_of_identifier)]
+pub(crate) struct EnumPatternInsteadOfIdentifier {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_dot_dot_dot_for_remaining_fields)]
+pub(crate) struct DotDotDotForRemainingFields {
+    #[primary_span]
+    #[suggestion(code = "..", style = "verbose", applicability = "machine-applicable")]
+    pub span: Span,
+    pub token_str: Cow<'static, str>,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_expected_comma_after_pattern_field)]
+pub(crate) struct ExpectedCommaAfterPatternField {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_return_types_use_thin_arrow)]
+pub(crate) struct ReturnTypesUseThinArrow {
+    #[primary_span]
+    #[suggestion(style = "short", code = "->", applicability = "machine-applicable")]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_need_plus_after_trait_object_lifetime)]
+pub(crate) struct NeedPlusAfterTraitObjectLifetime {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_expected_mut_or_const_in_raw_pointer_type)]
+pub(crate) struct ExpectedMutOrConstInRawPointerType {
+    #[primary_span]
+    pub span: Span,
+    #[suggestion(code("mut ", "const "), applicability = "has-placeholders")]
+    pub after_asterisk: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_lifetime_after_mut)]
+pub(crate) struct LifetimeAfterMut {
+    #[primary_span]
+    pub span: Span,
+    #[suggestion(code = "&{snippet} mut", applicability = "maybe-incorrect")]
+    pub suggest_lifetime: Option<Span>,
+    pub snippet: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_dyn_after_mut)]
+pub(crate) struct DynAfterMut {
+    #[primary_span]
+    #[suggestion(code = "&mut dyn", applicability = "machine-applicable")]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_fn_pointer_cannot_be_const)]
+pub(crate) struct FnPointerCannotBeConst {
+    #[primary_span]
+    pub span: Span,
+    #[suggestion(code = "", applicability = "maybe-incorrect")]
+    #[label]
+    pub qualifier: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_fn_pointer_cannot_be_async)]
+pub(crate) struct FnPointerCannotBeAsync {
+    #[primary_span]
+    pub span: Span,
+    #[suggestion(code = "", applicability = "maybe-incorrect")]
+    #[label]
+    pub qualifier: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_nested_c_variadic_type, code = "E0743")]
+pub(crate) struct NestedCVariadicType {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_invalid_dyn_keyword)]
+#[help]
+pub(crate) struct InvalidDynKeyword {
+    #[primary_span]
+    #[suggestion(code = "", applicability = "machine-applicable")]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_negative_bounds_not_supported)]
+pub(crate) struct NegativeBoundsNotSupported {
+    #[primary_span]
+    pub negative_bounds: Vec<Span>,
+    #[label]
+    pub last_span: Span,
+    #[subdiagnostic]
+    pub sub: Option<NegativeBoundsNotSupportedSugg>,
+}
+
+#[derive(Subdiagnostic)]
+#[suggestion(
+    parse_suggestion,
+    style = "tool-only",
+    code = "{fixed}",
+    applicability = "machine-applicable"
+)]
+pub(crate) struct NegativeBoundsNotSupportedSugg {
+    #[primary_span]
+    pub bound_list: Span,
+    pub num_bounds: usize,
+    pub fixed: String,
+}
+
+#[derive(Subdiagnostic)]
+pub enum HelpUseLatestEdition {
+    #[help(parse_help_set_edition_cargo)]
+    #[note(parse_note_edition_guide)]
+    Cargo { edition: Edition },
+    #[help(parse_help_set_edition_standalone)]
+    #[note(parse_note_edition_guide)]
+    Standalone { edition: Edition },
+}
+
+impl HelpUseLatestEdition {
+    pub fn new() -> Self {
+        let edition = LATEST_STABLE_EDITION;
+        if std::env::var_os("CARGO").is_some() {
+            Self::Cargo { edition }
+        } else {
+            Self::Standalone { edition }
+        }
+    }
+}
diff --git a/compiler/rustc_parse/src/lexer/diagnostics.rs b/compiler/rustc_parse/src/lexer/diagnostics.rs
new file mode 100644
index 000000000..27f4428d3
--- /dev/null
+++ b/compiler/rustc_parse/src/lexer/diagnostics.rs
@@ -0,0 +1,119 @@
+use super::UnmatchedDelim;
+use rustc_ast::token::Delimiter;
+use rustc_errors::Diagnostic;
+use rustc_span::source_map::SourceMap;
+use rustc_span::Span;
+
+#[derive(Default)]
+pub struct TokenTreeDiagInfo {
+    /// Stack of open delimiters and their spans. Used for error message.
+    pub open_braces: Vec<(Delimiter, Span)>,
+    pub unmatched_delims: Vec<UnmatchedDelim>,
+
+    /// Used only for error recovery when arriving to EOF with mismatched braces.
+    pub last_unclosed_found_span: Option<Span>,
+
+    /// Collect empty block spans that might have been auto-inserted by editors.
+    pub empty_block_spans: Vec<Span>,
+
+    /// Collect the spans of braces (Open, Close). Used only
+    /// for detecting if blocks are empty and only braces.
+    pub matching_block_spans: Vec<(Span, Span)>,
+}
+
+pub fn same_identation_level(sm: &SourceMap, open_sp: Span, close_sp: Span) -> bool {
+    match (sm.span_to_margin(open_sp), sm.span_to_margin(close_sp)) {
+        (Some(open_padding), Some(close_padding)) => open_padding == close_padding,
+        _ => false,
+    }
+}
+
+// When we get a `)` or `]` for `{`, we should emit help message here
+// it's more friendly compared to report `unmatched error` in later phase
+pub fn report_missing_open_delim(
+    err: &mut Diagnostic,
+    unmatched_delims: &[UnmatchedDelim],
+) -> bool {
+    let mut reported_missing_open = false;
+    for unmatch_brace in unmatched_delims.iter() {
+        if let Some(delim) = unmatch_brace.found_delim
+            && matches!(delim, Delimiter::Parenthesis | Delimiter::Bracket)
+        {
+            let missed_open = match delim {
+                Delimiter::Parenthesis => "(",
+                Delimiter::Bracket => "[",
+                _ => unreachable!(),
+            };
+            err.span_label(
+                unmatch_brace.found_span.shrink_to_lo(),
+                format!("missing open `{}` for this delimiter", missed_open),
+            );
+            reported_missing_open = true;
+        }
+    }
+    reported_missing_open
+}
+
+pub fn report_suspicious_mismatch_block(
+    err: &mut Diagnostic,
+    diag_info: &TokenTreeDiagInfo,
+    sm: &SourceMap,
+    delim: Delimiter,
+) {
+    if report_missing_open_delim(err, &diag_info.unmatched_delims) {
+        return;
+    }
+
+    let mut matched_spans: Vec<(Span, bool)> = diag_info
+        .matching_block_spans
+        .iter()
+        .map(|&(open, close)| (open.with_hi(close.lo()), same_identation_level(sm, open, close)))
+        .collect();
+
+    // sort by `lo`, so the large block spans in the front
+    matched_spans.sort_by(|a, b| a.0.lo().cmp(&b.0.lo()));
+
+    // We use larger block whose identation is well to cover those inner mismatched blocks
+    // O(N^2) here, but we are on error reporting path, so it is fine
+    for i in 0..matched_spans.len() {
+        let (block_span, same_ident) = matched_spans[i];
+        if same_ident {
+            for j in i + 1..matched_spans.len() {
+                let (inner_block, inner_same_ident) = matched_spans[j];
+                if block_span.contains(inner_block) && !inner_same_ident {
+                    matched_spans[j] = (inner_block, true);
+                }
+            }
+        }
+    }
+
+    // Find the inner-most span candidate for final report
+    let candidate_span =
+        matched_spans.into_iter().rev().find(|&(_, same_ident)| !same_ident).map(|(span, _)| span);
+
+    if let Some(block_span) = candidate_span {
+        err.span_label(block_span.shrink_to_lo(), "this delimiter might not be properly closed...");
+        err.span_label(
+            block_span.shrink_to_hi(),
+            "...as it matches this but it has different indentation",
+        );
+
+        // If there is a empty block in the mismatched span, note it
+        if delim == Delimiter::Brace {
+            for span in diag_info.empty_block_spans.iter() {
+                if block_span.contains(*span) {
+                    err.span_label(*span, "block is empty, you might have not meant to close it");
+                    break;
+                }
+            }
+        }
+    } else {
+        // If there is no suspicious span, give the last properly closed block may help
+        if let Some(parent) = diag_info.matching_block_spans.last()
+            && diag_info.open_braces.last().is_none()
+            && diag_info.empty_block_spans.iter().all(|&sp| sp != parent.0.to(parent.1)) {
+                err.span_label(parent.0, "this opening brace...");
+                err.span_label(parent.1, "...matches this closing brace");
+        }
+    }
+}
diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs
index 9fe8d9836..59958a309 100644
--- a/compiler/rustc_parse/src/lexer/mod.rs
+++ b/compiler/rustc_parse/src/lexer/mod.rs
@@ -1,11 +1,11 @@
+use crate::errors;
 use crate::lexer::unicode_chars::UNICODE_ARRAY;
+use crate::make_unclosed_delims_error;
 use rustc_ast::ast::{self, AttrStyle};
 use rustc_ast::token::{self, CommentKind, Delimiter, Token, TokenKind};
 use rustc_ast::tokenstream::TokenStream;
 use rustc_ast::util::unicode::contains_text_flow_control_chars;
-use rustc_errors::{
-    error_code, Applicability, DiagnosticBuilder, ErrorGuaranteed, PResult, StashKey,
-};
+use rustc_errors::{error_code, Applicability, Diagnostic, DiagnosticBuilder, StashKey};
 use rustc_lexer::unescape::{self, Mode};
 use rustc_lexer::Cursor;
 use rustc_lexer::{Base, DocStyle, RawStrError};
@@ -17,6 +17,7 @@ use rustc_session::parse::ParseSess;
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::{edition::Edition, BytePos, Pos, Span};
 
+mod diagnostics;
 mod tokentrees;
 mod unescape_error_reporting;
 mod unicode_chars;
@@ -31,7 +32,7 @@ use unescape_error_reporting::{emit_unescape_error, escaped_char};
 rustc_data_structures::static_assert_size!(rustc_lexer::Token, 12);
 
 #[derive(Clone, Debug)]
-pub struct UnmatchedBrace {
+pub struct UnmatchedDelim {
     pub expected_delim: Delimiter,
     pub found_delim: Option<Delimiter>,
     pub found_span: Span,
@@ -44,7 +45,7 @@ pub(crate) fn parse_token_trees<'a>(
     mut src: &'a str,
     mut start_pos: BytePos,
     override_span: Option<Span>,
-) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) {
+) -> Result<TokenStream, Vec<Diagnostic>> {
     // Skip `#!`, if present.
     if let Some(shebang_len) = rustc_lexer::strip_shebang(src) {
         src = &src[shebang_len..];
@@ -61,7 +62,29 @@ pub(crate) fn parse_token_trees<'a>(
         override_span,
         nbsp_is_whitespace: false,
     };
-    tokentrees::TokenTreesReader::parse_all_token_trees(string_reader)
+    let (token_trees, unmatched_delims) =
+        tokentrees::TokenTreesReader::parse_all_token_trees(string_reader);
+    match token_trees {
+        Ok(stream) if unmatched_delims.is_empty() => Ok(stream),
+        _ => {
+            // Return error if there are unmatched delimiters or unclosng delimiters.
+            // We emit delimiter mismatch errors first, then emit the unclosing delimiter mismatch
+            // because the delimiter mismatch is more likely to be the root cause of error
+
+            let mut buffer = Vec::with_capacity(1);
+            // Not using `emit_unclosed_delims` to use `db.buffer`
+            for unmatched in unmatched_delims {
+                if let Some(err) = make_unclosed_delims_error(unmatched, &sess) {
+                    err.buffer(&mut buffer);
+                }
+            }
+            if let Err(err) = token_trees {
+                // Add unclosing delimiter error
+                err.buffer(&mut buffer);
+            }
+            Err(buffer)
+        }
+    }
 }
 
 struct StringReader<'a> {
@@ -150,7 +173,7 @@ impl<'a> StringReader<'a> {
                     let span = self.mk_sp(start, self.pos);
                     self.sess.symbol_gallery.insert(sym, span);
                     if !sym.can_be_raw() {
-                        self.err_span(span, &format!("`{}` cannot be a raw identifier", sym));
+                        self.sess.emit_err(errors::CannotBeRawIdent { span, ident: sym });
                     }
                     self.sess.raw_identifier_spans.borrow_mut().push(span);
                     token::Ident(sym, true)
@@ -261,27 +284,24 @@ impl<'a> StringReader<'a> {
                         self.nbsp_is_whitespace = true;
                     }
                     let repeats = it.take_while(|c1| *c1 == c).count();
-                    let mut err =
-                        self.struct_err_span_char(start, self.pos + Pos::from_usize(repeats * c.len_utf8()), "unknown start of token", c);
                     // FIXME: the lexer could be used to turn the ASCII version of unicode
                     // homoglyphs, instead of keeping a table in `check_for_substitution`into the
                     // token. Ideally, this should be inside `rustc_lexer`. However, we should
                     // first remove compound tokens like `<<` from `rustc_lexer`, and then add
                     // fancier error recovery to it, as there will be less overall work to do this
                     // way.
-                    let token = unicode_chars::check_for_substitution(self, start, c, &mut err, repeats+1);
-                    if c == '\x00' {
-                        err.help("source files must contain UTF-8 encoded text, unexpected null bytes might occur when a different encoding is used");
-                    }
-                    if repeats > 0 {
-                        if repeats == 1 {
-                            err.note(format!("character appears once more"));
-                        } else {
-                            err.note(format!("character appears {repeats} more times"));
-                        }
-                        swallow_next_invalid = repeats;
-                    }
-                    err.emit();
+                    let (token, sugg) = unicode_chars::check_for_substitution(self, start, c, repeats+1);
+                    self.sess.emit_err(errors::UnknownTokenStart {
+                        span: self.mk_sp(start, self.pos + Pos::from_usize(repeats * c.len_utf8())),
+                        escaped: escaped_char(c),
+                        sugg,
+                        null: if c == '\x00' {Some(errors::UnknownTokenNull)} else {None},
+                        repeat: if repeats > 0 {
+                            swallow_next_invalid = repeats;
+                            Some(errors::UnknownTokenRepeat { repeats })
+                        } else {None}
+                    });
+
                     if let Some(token) = token {
                         token
                     } else {
@@ -296,26 +316,6 @@ impl<'a> StringReader<'a> {
         }
     }
 
-    /// Report a fatal lexical error with a given span.
-    fn fatal_span(&self, sp: Span, m: &str) -> ! {
-        self.sess.span_diagnostic.span_fatal(sp, m)
-    }
-
-    /// Report a lexical error with a given span.
-    fn err_span(&self, sp: Span, m: &str) {
-        self.sess.span_diagnostic.struct_span_err(sp, m).emit();
-    }
-
-    /// Report a fatal error spanning [`from_pos`, `to_pos`).
-    fn fatal_span_(&self, from_pos: BytePos, to_pos: BytePos, m: &str) -> ! {
-        self.fatal_span(self.mk_sp(from_pos, to_pos), m)
-    }
-
-    /// Report a lexical error spanning [`from_pos`, `to_pos`).
-    fn err_span_(&self, from_pos: BytePos, to_pos: BytePos, m: &str) {
-        self.err_span(self.mk_sp(from_pos, to_pos), m)
-    }
-
     fn struct_fatal_span_char(
         &self,
         from_pos: BytePos,
@@ -328,18 +328,6 @@ impl<'a> StringReader<'a> {
             .struct_span_fatal(self.mk_sp(from_pos, to_pos), &format!("{}: {}", m, escaped_char(c)))
     }
 
-    fn struct_err_span_char(
-        &self,
-        from_pos: BytePos,
-        to_pos: BytePos,
-        m: &str,
-        c: char,
-    ) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
-        self.sess
-            .span_diagnostic
-            .struct_span_err(self.mk_sp(from_pos, to_pos), &format!("{}: {}", m, escaped_char(c)))
-    }
-
     /// Detect usages of Unicode codepoints changing the direction of the text on screen and loudly
     /// complain about it.
     fn lint_unicode_text_flow(&self, start: BytePos) {
@@ -367,14 +355,12 @@ impl<'a> StringReader<'a> {
     ) -> TokenKind {
         if content.contains('\r') {
             for (idx, _) in content.char_indices().filter(|&(_, c)| c == '\r') {
-                self.err_span_(
+                let span = self.mk_sp(
                     content_start + BytePos(idx as u32),
                     content_start + BytePos(idx as u32 + 1),
-                    match comment_kind {
-                        CommentKind::Line => "bare CR not allowed in doc-comment",
-                        CommentKind::Block => "bare CR not allowed in block doc-comment",
-                    },
                 );
+                let block = matches!(comment_kind, CommentKind::Block);
+                self.sess.emit_err(errors::CrDocComment { span, block });
             }
         }
 
@@ -453,26 +439,20 @@ impl<'a> StringReader<'a> {
             }
             rustc_lexer::LiteralKind::Int { base, empty_int } => {
                 if empty_int {
-                    self.sess
-                        .span_diagnostic
-                        .struct_span_err_with_code(
-                            self.mk_sp(start, end),
-                            "no valid digits found for number",
-                            error_code!(E0768),
-                        )
-                        .emit();
+                    let span = self.mk_sp(start, end);
+                    self.sess.emit_err(errors::NoDigitsLiteral { span });
                     (token::Integer, sym::integer(0))
                 } else {
                     if matches!(base, Base::Binary | Base::Octal) {
                         let base = base as u32;
                         let s = self.str_from_to(start + BytePos(2), end);
                         for (idx, c) in s.char_indices() {
+                            let span = self.mk_sp(
+                                start + BytePos::from_usize(2 + idx),
+                                start + BytePos::from_usize(2 + idx + c.len_utf8()),
+                            );
                             if c != '_' && c.to_digit(base).is_none() {
-                                self.err_span_(
-                                    start + BytePos::from_usize(2 + idx),
-                                    start + BytePos::from_usize(2 + idx + c.len_utf8()),
-                                    &format!("invalid digit for a base {} literal", base),
-                                );
+                                self.sess.emit_err(errors::InvalidDigitLiteral { span, base });
                             }
                         }
                     }
@@ -481,19 +461,18 @@ impl<'a> StringReader<'a> {
             }
             rustc_lexer::LiteralKind::Float { base, empty_exponent } => {
                 if empty_exponent {
-                    self.err_span_(start, self.pos, "expected at least one digit in exponent");
+                    let span = self.mk_sp(start, self.pos);
+                    self.sess.emit_err(errors::EmptyExponentFloat { span });
                 }
-                match base {
-                    Base::Hexadecimal => {
-                        self.err_span_(start, end, "hexadecimal float literal is not supported")
-                    }
-                    Base::Octal => {
-                        self.err_span_(start, end, "octal float literal is not supported")
-                    }
-                    Base::Binary => {
-                        self.err_span_(start, end, "binary float literal is not supported")
-                    }
-                    _ => {}
+                let base = match base {
+                    Base::Hexadecimal => Some("hexadecimal"),
+                    Base::Octal => Some("octal"),
+                    Base::Binary => Some("binary"),
+                    _ => None,
+                };
+                if let Some(base) = base {
+                    let span = self.mk_sp(start, end);
+                    self.sess.emit_err(errors::FloatLiteralUnsupportedBase { span, base });
                 }
                 (token::Float, self.symbol_from_to(start, end))
             }
@@ -643,54 +622,34 @@ impl<'a> StringReader<'a> {
     // identifier tokens.
     fn report_unknown_prefix(&self, start: BytePos) {
         let prefix_span = self.mk_sp(start, self.pos);
-        let prefix_str = self.str_from_to(start, self.pos);
-        let msg = format!("prefix `{}` is unknown", prefix_str);
+        let prefix = self.str_from_to(start, self.pos);
 
         let expn_data = prefix_span.ctxt().outer_expn_data();
 
         if expn_data.edition >= Edition::Edition2021 {
             // In Rust 2021, this is a hard error.
-            let mut err = self.sess.span_diagnostic.struct_span_err(prefix_span, &msg);
-            err.span_label(prefix_span, "unknown prefix");
-            if prefix_str == "rb" {
-                err.span_suggestion_verbose(
-                    prefix_span,
-                    "use `br` for a raw byte string",
-                    "br",
-                    Applicability::MaybeIncorrect,
-                );
+            let sugg = if prefix == "rb" {
+                Some(errors::UnknownPrefixSugg::UseBr(prefix_span))
             } else if expn_data.is_root() {
-                err.span_suggestion_verbose(
-                    prefix_span.shrink_to_hi(),
-                    "consider inserting whitespace here",
-                    " ",
-                    Applicability::MaybeIncorrect,
-                );
-            }
-            err.note("prefixed identifiers and literals are reserved since Rust 2021");
-            err.emit();
+                Some(errors::UnknownPrefixSugg::Whitespace(prefix_span.shrink_to_hi()))
+            } else {
+                None
+            };
+            self.sess.emit_err(errors::UnknownPrefix { span: prefix_span, prefix, sugg });
         } else {
             // Before Rust 2021, only emit a lint for migration.
             self.sess.buffer_lint_with_diagnostic(
                 &RUST_2021_PREFIXES_INCOMPATIBLE_SYNTAX,
                 prefix_span,
                 ast::CRATE_NODE_ID,
-                &msg,
+                &format!("prefix `{prefix}` is unknown"),
                 BuiltinLintDiagnostics::ReservedPrefix(prefix_span),
             );
         }
     }
 
-    fn report_too_many_hashes(&self, start: BytePos, found: u32) -> ! {
-        self.fatal_span_(
-            start,
-            self.pos,
-            &format!(
-                "too many `#` symbols: raw strings may be delimited \
-                by up to 255 `#` symbols, but found {}",
-                found
-            ),
-        )
+    fn report_too_many_hashes(&self, start: BytePos, num: u32) -> ! {
+        self.sess.emit_fatal(errors::TooManyHashes { span: self.mk_sp(start, self.pos), num });
     }
 
     fn cook_quoted(
diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs
index b2701817d..36fd1e37d 100644
--- a/compiler/rustc_parse/src/lexer/tokentrees.rs
+++ b/compiler/rustc_parse/src/lexer/tokentrees.rs
@@ -1,47 +1,31 @@
-use super::{StringReader, UnmatchedBrace};
+use super::diagnostics::report_suspicious_mismatch_block;
+use super::diagnostics::same_identation_level;
+use super::diagnostics::TokenTreeDiagInfo;
+use super::{StringReader, UnmatchedDelim};
 use rustc_ast::token::{self, Delimiter, Token};
 use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree};
 use rustc_ast_pretty::pprust::token_to_string;
-use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{PErr, PResult};
-use rustc_span::Span;
 
 pub(super) struct TokenTreesReader<'a> {
     string_reader: StringReader<'a>,
     /// The "next" token, which has been obtained from the `StringReader` but
     /// not yet handled by the `TokenTreesReader`.
     token: Token,
-    /// Stack of open delimiters and their spans. Used for error message.
-    open_braces: Vec<(Delimiter, Span)>,
-    unmatched_braces: Vec<UnmatchedBrace>,
-    /// The type and spans for all braces
-    ///
-    /// Used only for error recovery when arriving to EOF with mismatched braces.
-    matching_delim_spans: Vec<(Delimiter, Span, Span)>,
-    last_unclosed_found_span: Option<Span>,
-    /// Collect empty block spans that might have been auto-inserted by editors.
-    last_delim_empty_block_spans: FxHashMap<Delimiter, Span>,
-    /// Collect the spans of braces (Open, Close). Used only
-    /// for detecting if blocks are empty and only braces.
-    matching_block_spans: Vec<(Span, Span)>,
+    diag_info: TokenTreeDiagInfo,
 }
 
 impl<'a> TokenTreesReader<'a> {
     pub(super) fn parse_all_token_trees(
         string_reader: StringReader<'a>,
-    ) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) {
+    ) -> (PResult<'a, TokenStream>, Vec<UnmatchedDelim>) {
         let mut tt_reader = TokenTreesReader {
             string_reader,
             token: Token::dummy(),
-            open_braces: Vec::new(),
-            unmatched_braces: Vec::new(),
-            matching_delim_spans: Vec::new(),
-            last_unclosed_found_span: None,
-            last_delim_empty_block_spans: FxHashMap::default(),
-            matching_block_spans: Vec::new(),
+            diag_info: TokenTreeDiagInfo::default(),
         };
         let res = tt_reader.parse_token_trees(/* is_delimited */ false);
-        (res, tt_reader.unmatched_braces)
+        (res, tt_reader.diag_info.unmatched_delims)
     }
 
     // Parse a stream of tokens into a list of `TokenTree`s.
@@ -50,7 +34,7 @@ impl<'a> TokenTreesReader<'a> {
         let mut buf = Vec::new();
         loop {
             match self.token.kind {
-                token::OpenDelim(delim) => buf.push(self.parse_token_tree_open_delim(delim)),
+                token::OpenDelim(delim) => buf.push(self.parse_token_tree_open_delim(delim)?),
                 token::CloseDelim(delim) => {
                     return if is_delimited {
                         Ok(TokenStream::new(buf))
@@ -59,10 +43,11 @@ impl<'a> TokenTreesReader<'a> {
                     };
                 }
                 token::Eof => {
-                    if is_delimited {
-                        self.eof_err().emit();
-                    }
-                    return Ok(TokenStream::new(buf));
+                    return if is_delimited {
+                        Err(self.eof_err())
+                    } else {
+                        Ok(TokenStream::new(buf))
+                    };
                 }
                 _ => {
                     // Get the next normal token. This might require getting multiple adjacent
@@ -92,9 +77,9 @@ impl<'a> TokenTreesReader<'a> {
     fn eof_err(&mut self) -> PErr<'a> {
         let msg = "this file contains an unclosed delimiter";
         let mut err = self.string_reader.sess.span_diagnostic.struct_span_err(self.token.span, msg);
-        for &(_, sp) in &self.open_braces {
+        for &(_, sp) in &self.diag_info.open_braces {
             err.span_label(sp, "unclosed delimiter");
-            self.unmatched_braces.push(UnmatchedBrace {
+            self.diag_info.unmatched_delims.push(UnmatchedDelim {
                 expected_delim: Delimiter::Brace,
                 found_delim: None,
                 found_span: self.token.span,
@@ -103,69 +88,53 @@ impl<'a> TokenTreesReader<'a> {
             });
         }
 
-        if let Some((delim, _)) = self.open_braces.last() {
-            if let Some((_, open_sp, close_sp)) =
-                self.matching_delim_spans.iter().find(|(d, open_sp, close_sp)| {
-                    let sm = self.string_reader.sess.source_map();
-                    if let Some(close_padding) = sm.span_to_margin(*close_sp) {
-                        if let Some(open_padding) = sm.span_to_margin(*open_sp) {
-                            return delim == d && close_padding != open_padding;
-                        }
-                    }
-                    false
-                })
-            // these are in reverse order as they get inserted on close, but
-            {
-                // we want the last open/first close
-                err.span_label(*open_sp, "this delimiter might not be properly closed...");
-                err.span_label(*close_sp, "...as it matches this but it has different indentation");
-            }
+        if let Some((delim, _)) = self.diag_info.open_braces.last() {
+            report_suspicious_mismatch_block(
+                &mut err,
+                &self.diag_info,
+                &self.string_reader.sess.source_map(),
+                *delim,
+            )
         }
         err
     }
 
-    fn parse_token_tree_open_delim(&mut self, open_delim: Delimiter) -> TokenTree {
+    fn parse_token_tree_open_delim(&mut self, open_delim: Delimiter) -> PResult<'a, TokenTree> {
         // The span for beginning of the delimited section
         let pre_span = self.token.span;
 
-        self.open_braces.push((open_delim, self.token.span));
+        self.diag_info.open_braces.push((open_delim, self.token.span));
 
         // Parse the token trees within the delimiters.
         // We stop at any delimiter so we can try to recover if the user
         // uses an incorrect delimiter.
-        let tts = self.parse_token_trees(/* is_delimited */ true).unwrap();
+        let tts = self.parse_token_trees(/* is_delimited */ true)?;
 
         // Expand to cover the entire delimited token tree
         let delim_span = DelimSpan::from_pair(pre_span, self.token.span);
+        let sm = self.string_reader.sess.source_map();
 
         match self.token.kind {
             // Correct delimiter.
             token::CloseDelim(close_delim) if close_delim == open_delim => {
-                let (open_brace, open_brace_span) = self.open_braces.pop().unwrap();
+                let (open_brace, open_brace_span) = self.diag_info.open_braces.pop().unwrap();
                 let close_brace_span = self.token.span;
 
-                if tts.is_empty() {
+                if tts.is_empty() && close_delim == Delimiter::Brace {
                     let empty_block_span = open_brace_span.to(close_brace_span);
-                    let sm = self.string_reader.sess.source_map();
                     if !sm.is_multiline(empty_block_span) {
                         // Only track if the block is in the form of `{}`, otherwise it is
                         // likely that it was written on purpose.
-                        self.last_delim_empty_block_spans.insert(open_delim, empty_block_span);
+                        self.diag_info.empty_block_spans.push(empty_block_span);
                     }
                 }
 
-                //only add braces
+                // only add braces
                 if let (Delimiter::Brace, Delimiter::Brace) = (open_brace, open_delim) {
-                    self.matching_block_spans.push((open_brace_span, close_brace_span));
+                    // Add all the matching spans, we will sort by span later
+                    self.diag_info.matching_block_spans.push((open_brace_span, close_brace_span));
                 }
 
-                if self.open_braces.is_empty() {
-                    // Clear up these spans to avoid suggesting them as we've found
-                    // properly matched delimiters so far for an entire block.
-                    self.matching_delim_spans.clear();
-                } else {
-                    self.matching_delim_spans.push((open_brace, open_brace_span, close_brace_span));
-                }
                 // Move past the closing delimiter.
                 self.token = self.string_reader.next_token().0;
             }
@@ -174,28 +143,25 @@ impl<'a> TokenTreesReader<'a> {
                 let mut unclosed_delimiter = None;
                 let mut candidate = None;
 
-                if self.last_unclosed_found_span != Some(self.token.span) {
+                if self.diag_info.last_unclosed_found_span != Some(self.token.span) {
                     // do not complain about the same unclosed delimiter multiple times
-                    self.last_unclosed_found_span = Some(self.token.span);
+                    self.diag_info.last_unclosed_found_span = Some(self.token.span);
                     // This is a conservative error: only report the last unclosed
                     // delimiter. The previous unclosed delimiters could actually be
                     // closed! The parser just hasn't gotten to them yet.
-                    if let Some(&(_, sp)) = self.open_braces.last() {
+                    if let Some(&(_, sp)) = self.diag_info.open_braces.last() {
                         unclosed_delimiter = Some(sp);
                     };
-                    let sm = self.string_reader.sess.source_map();
-                    if let Some(current_padding) = sm.span_to_margin(self.token.span) {
-                        for (brace, brace_span) in &self.open_braces {
-                            if let Some(padding) = sm.span_to_margin(*brace_span) {
-                                // high likelihood of these two corresponding
-                                if current_padding == padding && brace == &close_delim {
-                                    candidate = Some(*brace_span);
-                                }
-                            }
+                    for (brace, brace_span) in &self.diag_info.open_braces {
+                        if same_identation_level(&sm, self.token.span, *brace_span)
+                            && brace == &close_delim
+                        {
+                            // high likelihood of these two corresponding
+                            candidate = Some(*brace_span);
                         }
                     }
-                    let (tok, _) = self.open_braces.pop().unwrap();
-                    self.unmatched_braces.push(UnmatchedBrace {
+                    let (tok, _) = self.diag_info.open_braces.pop().unwrap();
+                    self.diag_info.unmatched_delims.push(UnmatchedDelim {
                         expected_delim: tok,
                         found_delim: Some(close_delim),
                         found_span: self.token.span,
@@ -203,7 +169,7 @@ impl<'a> TokenTreesReader<'a> {
                         candidate_span: candidate,
                     });
                 } else {
-                    self.open_braces.pop();
+                    self.diag_info.open_braces.pop();
                 }
 
                 // If the incorrect delimiter matches an earlier opening
@@ -213,7 +179,7 @@ impl<'a> TokenTreesReader<'a> {
                 // fn foo() {
                 //     bar(baz(
                 // }  // Incorrect delimiter but matches the earlier `{`
-                if !self.open_braces.iter().any(|&(b, _)| b == close_delim) {
+                if !self.diag_info.open_braces.iter().any(|&(b, _)| b == close_delim) {
                     self.token = self.string_reader.next_token().0;
                 }
             }
@@ -225,7 +191,7 @@ impl<'a> TokenTreesReader<'a> {
             _ => unreachable!(),
         }
 
-        TokenTree::Delimited(delim_span, open_delim, tts)
+        Ok(TokenTree::Delimited(delim_span, open_delim, tts))
     }
 
     fn close_delim_err(&mut self, delim: Delimiter) -> PErr<'a> {
@@ -236,22 +202,12 @@ impl<'a> TokenTreesReader<'a> {
         let mut err =
             self.string_reader.sess.span_diagnostic.struct_span_err(self.token.span, &msg);
 
-        // Braces are added at the end, so the last element is the biggest block
-        if let Some(parent) = self.matching_block_spans.last() {
-            if let Some(span) = self.last_delim_empty_block_spans.remove(&delim) {
-                // Check if the (empty block) is in the last properly closed block
-                if (parent.0.to(parent.1)).contains(span) {
-                    err.span_label(span, "block is empty, you might have not meant to close it");
-                } else {
-                    err.span_label(parent.0, "this opening brace...");
-                    err.span_label(parent.1, "...matches this closing brace");
-                }
-            } else {
-                err.span_label(parent.0, "this opening brace...");
-                err.span_label(parent.1, "...matches this closing brace");
-            }
-        }
-
+        report_suspicious_mismatch_block(
+            &mut err,
+            &self.diag_info,
+            &self.string_reader.sess.source_map(),
+            delim,
+        );
         err.span_label(self.token.span, "unexpected closing delimiter");
         err
     }
diff --git a/compiler/rustc_parse/src/lexer/unescape_error_reporting.rs b/compiler/rustc_parse/src/lexer/unescape_error_reporting.rs
index 6373f5b4f..0d12ec608 100644
--- a/compiler/rustc_parse/src/lexer/unescape_error_reporting.rs
+++ b/compiler/rustc_parse/src/lexer/unescape_error_reporting.rs
@@ -3,10 +3,12 @@
 use std::iter::once;
 use std::ops::Range;
 
-use rustc_errors::{pluralize, Applicability, Handler};
+use rustc_errors::{Applicability, Handler};
 use rustc_lexer::unescape::{EscapeError, Mode};
 use rustc_span::{BytePos, Span};
 
+use crate::errors::{MoreThanOneCharNote, MoreThanOneCharSugg, NoBraceUnicodeSub, UnescapeError};
+
 pub(crate) fn emit_unescape_error(
     handler: &Handler,
     // interior part of the literal, without quotes
@@ -31,53 +33,32 @@ pub(crate) fn emit_unescape_error(
     };
     match error {
         EscapeError::LoneSurrogateUnicodeEscape => {
-            handler
-                .struct_span_err(span, "invalid unicode character escape")
-                .span_label(span, "invalid escape")
-                .help("unicode escape must not be a surrogate")
-                .emit();
+            handler.emit_err(UnescapeError::InvalidUnicodeEscape { span, surrogate: true });
         }
         EscapeError::OutOfRangeUnicodeEscape => {
-            handler
-                .struct_span_err(span, "invalid unicode character escape")
-                .span_label(span, "invalid escape")
-                .help("unicode escape must be at most 10FFFF")
-                .emit();
+            handler.emit_err(UnescapeError::InvalidUnicodeEscape { span, surrogate: false });
         }
         EscapeError::MoreThanOneChar => {
             use unicode_normalization::{char::is_combining_mark, UnicodeNormalization};
+            let mut sugg = None;
+            let mut note = None;
 
-            let mut has_help = false;
-            let mut handler = handler.struct_span_err(
-                span_with_quotes,
-                "character literal may only contain one codepoint",
-            );
-
-            if lit.chars().skip(1).all(|c| is_combining_mark(c)) {
-                let escaped_marks =
-                    lit.chars().skip(1).map(|c| c.escape_default().to_string()).collect::<Vec<_>>();
-                handler.span_note(
-                    span,
-                    &format!(
-                        "this `{}` is followed by the combining mark{} `{}`",
-                        lit.chars().next().unwrap(),
-                        pluralize!(escaped_marks.len()),
-                        escaped_marks.join(""),
-                    ),
-                );
+            let lit_chars = lit.chars().collect::<Vec<_>>();
+            let (first, rest) = lit_chars.split_first().unwrap();
+            if rest.iter().copied().all(is_combining_mark) {
                 let normalized = lit.nfc().to_string();
                 if normalized.chars().count() == 1 {
-                    has_help = true;
-                    handler.span_suggestion(
-                        span,
-                        &format!(
-                            "consider using the normalized form `{}` of this character",
-                            normalized.chars().next().unwrap().escape_default()
-                        ),
-                        normalized,
-                        Applicability::MachineApplicable,
-                    );
+                    let ch = normalized.chars().next().unwrap().escape_default().to_string();
+                    sugg = Some(MoreThanOneCharSugg::NormalizedForm { span, ch, normalized });
                 }
+                let escaped_marks =
+                    rest.iter().map(|c| c.escape_default().to_string()).collect::<Vec<_>>();
+                note = Some(MoreThanOneCharNote::AllCombining {
+                    span,
+                    chr: format!("{first}"),
+                    len: escaped_marks.len(),
+                    escaped_marks: escaped_marks.join(""),
+                });
             } else {
                 let printable: Vec<char> = lit
                     .chars()
@@ -87,32 +68,18 @@ pub(crate) fn emit_unescape_error(
                     })
                     .collect();
 
-                if let [ch] = printable.as_slice() {
-                    has_help = true;
-
-                    handler.span_note(
+                if let &[ch] = printable.as_slice() {
+                    sugg =
+                        Some(MoreThanOneCharSugg::RemoveNonPrinting { span, ch: ch.to_string() });
+                    note = Some(MoreThanOneCharNote::NonPrinting {
                         span,
-                        &format!(
-                            "there are non-printing characters, the full sequence is `{}`",
-                            lit.escape_default(),
-                        ),
-                    );
-
-                    handler.span_suggestion(
-                        span,
-                        "consider removing the non-printing characters",
-                        ch,
-                        Applicability::MaybeIncorrect,
-                    );
+                        escaped: lit.escape_default().to_string(),
+                    });
                 }
-            }
-
-            if !has_help {
-                let (prefix, msg) = if mode.is_byte() {
-                    ("b", "if you meant to write a byte string literal, use double quotes")
-                } else {
-                    ("", "if you meant to write a `str` literal, use double quotes")
-                };
+            };
+            let sugg = sugg.unwrap_or_else(|| {
+                let is_byte = mode.is_byte();
+                let prefix = if is_byte { "b" } else { "" };
                 let mut escaped = String::with_capacity(lit.len());
                 let mut chrs = lit.chars().peekable();
                 while let Some(first) = chrs.next() {
@@ -129,54 +96,32 @@ pub(crate) fn emit_unescape_error(
                         (c, _) => escaped.push(c),
                     };
                 }
-                handler.span_suggestion(
-                    span_with_quotes,
-                    msg,
-                    format!("{prefix}\"{escaped}\""),
-                    Applicability::MachineApplicable,
-                );
-            }
-
-            handler.emit();
+                let sugg = format!("{prefix}\"{escaped}\"");
+                MoreThanOneCharSugg::Quotes { span: span_with_quotes, is_byte, sugg }
+            });
+            handler.emit_err(UnescapeError::MoreThanOneChar {
+                span: span_with_quotes,
+                note,
+                suggestion: sugg,
+            });
         }
         EscapeError::EscapeOnlyChar => {
             let (c, char_span) = last_char();
-
-            let msg = if mode.is_byte() {
-                "byte constant must be escaped"
-            } else {
-                "character constant must be escaped"
-            };
-            handler
-                .struct_span_err(span, &format!("{}: `{}`", msg, escaped_char(c)))
-                .span_suggestion(
-                    char_span,
-                    "escape the character",
-                    c.escape_default(),
-                    Applicability::MachineApplicable,
-                )
-                .emit();
+            handler.emit_err(UnescapeError::EscapeOnlyChar {
+                span,
+                char_span,
+                escaped_sugg: c.escape_default().to_string(),
+                escaped_msg: escaped_char(c),
+                byte: mode.is_byte(),
+            });
         }
         EscapeError::BareCarriageReturn => {
-            let msg = if mode.in_double_quotes() {
-                "bare CR not allowed in string, use `\\r` instead"
-            } else {
-                "character constant must be escaped: `\\r`"
-            };
-            handler
-                .struct_span_err(span, msg)
-                .span_suggestion(
-                    span,
-                    "escape the character",
-                    "\\r",
-                    Applicability::MachineApplicable,
-                )
-                .emit();
+            let double_quotes = mode.in_double_quotes();
+            handler.emit_err(UnescapeError::BareCr { span, double_quotes });
         }
         EscapeError::BareCarriageReturnInRawString => {
             assert!(mode.in_double_quotes());
-            let msg = "bare CR not allowed in raw string";
-            handler.span_err(span, msg);
+            handler.emit_err(UnescapeError::BareCrRawString(span));
         }
         EscapeError::InvalidEscape => {
             let (c, span) = last_char();
@@ -213,22 +158,13 @@ pub(crate) fn emit_unescape_error(
             diag.emit();
         }
         EscapeError::TooShortHexEscape => {
-            handler.span_err(span, "numeric character escape is too short");
+            handler.emit_err(UnescapeError::TooShortHexEscape(span));
         }
         EscapeError::InvalidCharInHexEscape | EscapeError::InvalidCharInUnicodeEscape => {
             let (c, span) = last_char();
-
-            let msg = if error == EscapeError::InvalidCharInHexEscape {
-                "invalid character in numeric character escape"
-            } else {
-                "invalid character in unicode escape"
-            };
-            let c = escaped_char(c);
-
-            handler
-                .struct_span_err(span, &format!("{}: `{}`", msg, c))
-                .span_label(span, msg)
-                .emit();
+            let is_hex = error == EscapeError::InvalidCharInHexEscape;
+            let ch = escaped_char(c);
+            handler.emit_err(UnescapeError::InvalidCharInEscape { span, is_hex, ch });
         }
         EscapeError::NonAsciiCharInByte => {
             let (c, span) = last_char();
@@ -278,41 +214,22 @@ pub(crate) fn emit_unescape_error(
             err.emit();
         }
         EscapeError::OutOfRangeHexEscape => {
-            handler
-                .struct_span_err(span, "out of range hex escape")
-                .span_label(span, "must be a character in the range [\\x00-\\x7f]")
-                .emit();
+            handler.emit_err(UnescapeError::OutOfRangeHexEscape(span));
         }
         EscapeError::LeadingUnderscoreUnicodeEscape => {
             let (c, span) = last_char();
-            let msg = "invalid start of unicode escape";
-            handler
-                .struct_span_err(span, &format!("{}: `{}`", msg, c))
-                .span_label(span, msg)
-                .emit();
+            handler.emit_err(UnescapeError::LeadingUnderscoreUnicodeEscape {
+                span,
+                ch: escaped_char(c),
+            });
         }
         EscapeError::OverlongUnicodeEscape => {
-            handler
-                .struct_span_err(span, "overlong unicode escape")
-                .span_label(span, "must have at most 6 hex digits")
-                .emit();
+            handler.emit_err(UnescapeError::OverlongUnicodeEscape(span));
         }
         EscapeError::UnclosedUnicodeEscape => {
-            handler
-                .struct_span_err(span, "unterminated unicode escape")
-                .span_label(span, "missing a closing `}`")
-                .span_suggestion_verbose(
-                    span.shrink_to_hi(),
-                    "terminate the unicode escape",
-                    "}",
-                    Applicability::MaybeIncorrect,
-                )
-                .emit();
+            handler.emit_err(UnescapeError::UnclosedUnicodeEscape(span, span.shrink_to_hi()));
         }
         EscapeError::NoBraceInUnicodeEscape => {
-            let msg = "incorrect unicode escape sequence";
-            let mut diag = handler.struct_span_err(span, msg);
-
             let mut suggestion = "\\u{".to_owned();
             let mut suggestion_len = 0;
             let (c, char_span) = last_char();
@@ -322,54 +239,37 @@ pub(crate) fn emit_unescape_error(
                 suggestion_len += c.len_utf8();
             }
 
-            if suggestion_len > 0 {
+            let (label, sub) = if suggestion_len > 0 {
                 suggestion.push('}');
                 let hi = char_span.lo() + BytePos(suggestion_len as u32);
-                diag.span_suggestion(
-                    span.with_hi(hi),
-                    "format of unicode escape sequences uses braces",
-                    suggestion,
-                    Applicability::MaybeIncorrect,
-                );
+                (None, NoBraceUnicodeSub::Suggestion { span: span.with_hi(hi), suggestion })
             } else {
-                diag.span_label(span, msg);
-                diag.help("format of unicode escape sequences is `\\u{...}`");
-            }
-
-            diag.emit();
+                (Some(span), NoBraceUnicodeSub::Help)
+            };
+            handler.emit_err(UnescapeError::NoBraceInUnicodeEscape { span, label, sub });
         }
         EscapeError::UnicodeEscapeInByte => {
-            let msg = "unicode escape in byte string";
-            handler
-                .struct_span_err(span, msg)
-                .span_label(span, msg)
-                .help("unicode escape sequences cannot be used as a byte or in a byte string")
-                .emit();
+            handler.emit_err(UnescapeError::UnicodeEscapeInByte(span));
         }
         EscapeError::EmptyUnicodeEscape => {
-            handler
-                .struct_span_err(span, "empty unicode escape")
-                .span_label(span, "this escape must have at least 1 hex digit")
-                .emit();
+            handler.emit_err(UnescapeError::EmptyUnicodeEscape(span));
         }
         EscapeError::ZeroChars => {
-            let msg = "empty character literal";
-            handler.struct_span_err(span, msg).span_label(span, msg).emit();
+            handler.emit_err(UnescapeError::ZeroChars(span));
         }
         EscapeError::LoneSlash => {
-            let msg = "invalid trailing slash in literal";
-            handler.struct_span_err(span, msg).span_label(span, msg).emit();
+            handler.emit_err(UnescapeError::LoneSlash(span));
         }
         EscapeError::UnskippedWhitespaceWarning => {
             let (c, char_span) = last_char();
-            let msg =
-                format!("non-ASCII whitespace symbol '{}' is not skipped", c.escape_unicode());
-            handler.struct_span_warn(span, &msg).span_label(char_span, &msg).emit();
+            handler.emit_warning(UnescapeError::UnskippedWhitespace {
+                span,
+                ch: escaped_char(c),
+                char_span,
+            });
         }
         EscapeError::MultipleSkippedLinesWarning => {
-            let msg = "multiple lines skipped by escaped newline";
-            let bottom_msg = "skipping everything up to and including this point";
-            handler.struct_span_warn(span, msg).span_label(span, bottom_msg).emit();
+            handler.emit_warning(UnescapeError::MultipleSkippedLinesWarning(span));
         }
     }
 }
diff --git a/compiler/rustc_parse/src/lexer/unicode_chars.rs b/compiler/rustc_parse/src/lexer/unicode_chars.rs
index 34d003ccf..d4f971d5b 100644
--- a/compiler/rustc_parse/src/lexer/unicode_chars.rs
+++ b/compiler/rustc_parse/src/lexer/unicode_chars.rs
@@ -2,8 +2,10 @@
 //! <https://www.unicode.org/Public/security/10.0.0/confusables.txt>
 
 use super::StringReader;
-use crate::token::{self, Delimiter};
-use rustc_errors::{Applicability, Diagnostic};
+use crate::{
+    errors::TokenSubstitution,
+    token::{self, Delimiter},
+};
 use rustc_span::{symbol::kw, BytePos, Pos, Span};
 
 #[rustfmt::skip] // for line breaks
@@ -338,48 +340,44 @@ pub(super) fn check_for_substitution<'a>(
     reader: &StringReader<'a>,
     pos: BytePos,
     ch: char,
-    err: &mut Diagnostic,
     count: usize,
-) -> Option<token::TokenKind> {
-    let &(_, u_name, ascii_str) = UNICODE_ARRAY.iter().find(|&&(c, _, _)| c == ch)?;
+) -> (Option<token::TokenKind>, Option<TokenSubstitution>) {
+    let Some(&(_, u_name, ascii_str)) = UNICODE_ARRAY.iter().find(|&&(c, _, _)| c == ch) else {
+        return (None, None);
+    };
 
     let span = Span::with_root_ctxt(pos, pos + Pos::from_usize(ch.len_utf8() * count));
 
     let Some((_, ascii_name, token)) = ASCII_ARRAY.iter().find(|&&(s, _, _)| s == ascii_str) else {
         let msg = format!("substitution character not found for '{}'", ch);
         reader.sess.span_diagnostic.span_bug_no_panic(span, &msg);
-        return None;
+        return (None, None);
     };
 
     // special help suggestion for "directed" double quotes
-    if let Some(s) = peek_delimited(&reader.src[reader.src_index(pos)..], '“', '”') {
-        let msg = format!(
-            "Unicode characters '“' (Left Double Quotation Mark) and \
-             '”' (Right Double Quotation Mark) look like '{}' ({}), but are not",
-            ascii_str, ascii_name
-        );
-        err.span_suggestion(
-            Span::with_root_ctxt(
-                pos,
-                pos + Pos::from_usize('“'.len_utf8() + s.len() + '”'.len_utf8()),
-            ),
-            &msg,
-            format!("\"{}\"", s),
-            Applicability::MaybeIncorrect,
+    let sugg = if let Some(s) = peek_delimited(&reader.src[reader.src_index(pos)..], '“', '”') {
+        let span = Span::with_root_ctxt(
+            pos,
+            pos + Pos::from_usize('“'.len_utf8() + s.len() + '”'.len_utf8()),
         );
+        Some(TokenSubstitution::DirectedQuotes {
+            span,
+            suggestion: format!("\"{s}\""),
+            ascii_str,
+            ascii_name,
+        })
     } else {
-        let msg = format!(
-            "Unicode character '{}' ({}) looks like '{}' ({}), but it is not",
-            ch, u_name, ascii_str, ascii_name
-        );
-        err.span_suggestion(
+        let suggestion = ascii_str.to_string().repeat(count);
+        Some(TokenSubstitution::Other {
             span,
-            &msg,
-            ascii_str.to_string().repeat(count),
-            Applicability::MaybeIncorrect,
-        );
-    }
-    token.clone()
+            suggestion,
+            ch: ch.to_string(),
+            u_name,
+            ascii_str,
+            ascii_name,
+        })
+    };
+    (token.clone(), sugg)
 }
 
 /// Extract string if found at current position with given delimiters
diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs
index b49a01d75..d1c3fd0cd 100644
--- a/compiler/rustc_parse/src/lib.rs
+++ b/compiler/rustc_parse/src/lib.rs
@@ -19,6 +19,8 @@ use rustc_ast::{AttrItem, Attribute, MetaItem};
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::{Applicability, Diagnostic, FatalError, Level, PResult};
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
+use rustc_macros::fluent_messages;
 use rustc_session::parse::ParseSess;
 use rustc_span::{FileName, SourceFile, Span};
 
@@ -28,12 +30,14 @@ pub const MACRO_ARGUMENTS: Option<&str> = Some("macro arguments");
 
 #[macro_use]
 pub mod parser;
-use parser::{emit_unclosed_delims, make_unclosed_delims_error, Parser};
+use parser::{make_unclosed_delims_error, Parser};
 pub mod lexer;
 pub mod validate_attr;
 
 mod errors;
 
+fluent_messages! { "../locales/en-US.ftl" }
+
 // A bunch of utility functions of the form `parse_<thing>_from_<source>`
 // where <thing> includes crate, expr, item, stmt, tts, and one that
 // uses a HOF to parse anything, and <source> includes file and
@@ -92,10 +96,7 @@ pub fn parse_stream_from_source_str(
     sess: &ParseSess,
     override_span: Option<Span>,
 ) -> TokenStream {
-    let (stream, mut errors) =
-        source_file_to_stream(sess, sess.source_map().new_source_file(name, source), override_span);
-    emit_unclosed_delims(&mut errors, &sess);
-    stream
+    source_file_to_stream(sess, sess.source_map().new_source_file(name, source), override_span)
 }
 
 /// Creates a new parser from a source string.
@@ -131,9 +132,8 @@ fn maybe_source_file_to_parser(
     source_file: Lrc<SourceFile>,
 ) -> Result<Parser<'_>, Vec<Diagnostic>> {
     let end_pos = source_file.end_pos;
-    let (stream, unclosed_delims) = maybe_file_to_stream(sess, source_file, None)?;
+    let stream = maybe_file_to_stream(sess, source_file, None)?;
     let mut parser = stream_to_parser(sess, stream, None);
-    parser.unclosed_delims = unclosed_delims;
     if parser.token == token::Eof {
         parser.token.span = Span::new(end_pos, end_pos, parser.token.span.ctxt(), None);
     }
@@ -178,7 +178,7 @@ pub fn source_file_to_stream(
     sess: &ParseSess,
     source_file: Lrc<SourceFile>,
     override_span: Option<Span>,
-) -> (TokenStream, Vec<lexer::UnmatchedBrace>) {
+) -> TokenStream {
     panictry_buffer!(&sess.span_diagnostic, maybe_file_to_stream(sess, source_file, override_span))
 }
 
@@ -188,7 +188,7 @@ pub fn maybe_file_to_stream(
     sess: &ParseSess,
     source_file: Lrc<SourceFile>,
     override_span: Option<Span>,
-) -> Result<(TokenStream, Vec<lexer::UnmatchedBrace>), Vec<Diagnostic>> {
+) -> Result<TokenStream, Vec<Diagnostic>> {
     let src = source_file.src.as_ref().unwrap_or_else(|| {
         sess.span_diagnostic.bug(&format!(
             "cannot lex `source_file` without source: {}",
@@ -196,23 +196,7 @@ pub fn maybe_file_to_stream(
         ));
     });
 
-    let (token_trees, unmatched_braces) =
-        lexer::parse_token_trees(sess, src.as_str(), source_file.start_pos, override_span);
-
-    match token_trees {
-        Ok(stream) => Ok((stream, unmatched_braces)),
-        Err(err) => {
-            let mut buffer = Vec::with_capacity(1);
-            err.buffer(&mut buffer);
-            // Not using `emit_unclosed_delims` to use `db.buffer`
-            for unmatched in unmatched_braces {
-                if let Some(err) = make_unclosed_delims_error(unmatched, &sess) {
-                    err.buffer(&mut buffer);
-                }
-            }
-            Err(buffer)
-        }
-    }
+    lexer::parse_token_trees(sess, src.as_str(), source_file.start_pos, override_span)
 }
 
 /// Given a stream and the `ParseSess`, produces a parser.
diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs
index 686454a8f..e3e7c63e3 100644
--- a/compiler/rustc_parse/src/parser/attr.rs
+++ b/compiler/rustc_parse/src/parser/attr.rs
@@ -1,11 +1,15 @@
 use crate::errors::{InvalidMetaItem, SuffixedLiteralInAttribute};
+use crate::fluent_generated as fluent;
 
 use super::{AttrWrapper, Capturing, FnParseMode, ForceCollect, Parser, PathStyle};
 use rustc_ast as ast;
 use rustc_ast::attr;
 use rustc_ast::token::{self, Delimiter, Nonterminal};
-use rustc_errors::{error_code, fluent, Diagnostic, IntoDiagnostic, PResult};
+use rustc_errors::{error_code, Diagnostic, IntoDiagnostic, PResult};
 use rustc_span::{sym, BytePos, Span};
+use std::convert::TryInto;
+use thin_vec::ThinVec;
+use tracing::debug;
 
 // Public for rustfmt usage
 #[derive(Debug)]
@@ -65,10 +69,10 @@ impl<'a> Parser<'a> {
                             token::CommentKind::Block => OuterAttributeType::DocBlockComment,
                         },
                     ) {
-                        err.note(fluent::note);
+                        err.note(fluent::parse_note);
                         err.span_suggestion_verbose(
                             replacement_span,
-                            fluent::suggestion,
+                            fluent::parse_suggestion,
                             "",
                             rustc_errors::Applicability::MachineApplicable,
                         );
@@ -172,10 +176,10 @@ impl<'a> Parser<'a> {
             Ok(Some(item)) => {
                 // FIXME(#100717)
                 err.set_arg("item", item.kind.descr());
-                err.span_label(item.span, fluent::label_does_not_annotate_this);
+                err.span_label(item.span, fluent::parse_label_does_not_annotate_this);
                 err.span_suggestion_verbose(
                     replacement_span,
-                    fluent::sugg_change_inner_to_outer,
+                    fluent::parse_sugg_change_inner_to_outer,
                     match attr_type {
                         OuterAttributeType::Attribute => "",
                         OuterAttributeType::DocBlockComment => "*",
@@ -201,8 +205,8 @@ impl<'a> Parser<'a> {
                         attr_sp,
                         fluent::parse_inner_attr_not_permitted_after_outer_doc_comment,
                     );
-                    diag.span_label(attr_sp, fluent::label_attr)
-                        .span_label(prev_doc_comment_span, fluent::label_prev_doc_comment);
+                    diag.span_label(attr_sp, fluent::parse_label_attr)
+                        .span_label(prev_doc_comment_span, fluent::parse_label_prev_doc_comment);
                     diag
                 }
                 Some(InnerAttrForbiddenReason::AfterOuterAttribute { prev_outer_attr_sp }) => {
@@ -210,8 +214,8 @@ impl<'a> Parser<'a> {
                         attr_sp,
                         fluent::parse_inner_attr_not_permitted_after_outer_attr,
                     );
-                    diag.span_label(attr_sp, fluent::label_attr)
-                        .span_label(prev_outer_attr_sp, fluent::label_prev_attr);
+                    diag.span_label(attr_sp, fluent::parse_label_attr)
+                        .span_label(prev_outer_attr_sp, fluent::parse_label_prev_attr);
                     diag
                 }
                 Some(InnerAttrForbiddenReason::InCodeBlock) | None => {
@@ -346,9 +350,9 @@ impl<'a> Parser<'a> {
     }
 
     /// Matches `COMMASEP(meta_item_inner)`.
-    pub(crate) fn parse_meta_seq_top(&mut self) -> PResult<'a, Vec<ast::NestedMetaItem>> {
+    pub(crate) fn parse_meta_seq_top(&mut self) -> PResult<'a, ThinVec<ast::NestedMetaItem>> {
         // Presumably, the majority of the time there will only be one attr.
-        let mut nmis = Vec::with_capacity(1);
+        let mut nmis = ThinVec::with_capacity(1);
         while self.token.kind != token::Eof {
             nmis.push(self.parse_meta_item_inner()?);
             if !self.eat(&token::Comma) {
diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs
index b97f22417..b0ab0f106 100644
--- a/compiler/rustc_parse/src/parser/attr_wrapper.rs
+++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs
@@ -134,11 +134,11 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl {
             // Process the replace ranges, starting from the highest start
             // position and working our way back. If have tokens like:
             //
-            // `#[cfg(FALSE)]` struct Foo { #[cfg(FALSE)] field: bool }`
+            // `#[cfg(FALSE)] struct Foo { #[cfg(FALSE)] field: bool }`
             //
             // Then we will generate replace ranges for both
             // the `#[cfg(FALSE)] field: bool` and the entire
-            // `#[cfg(FALSE)]` struct Foo { #[cfg(FALSE)] field: bool }`
+            // `#[cfg(FALSE)] struct Foo { #[cfg(FALSE)] field: bool }`
             //
             // By starting processing from the replace range with the greatest
             // start position, we ensure that any replace range which encloses
@@ -469,6 +469,6 @@ mod size_asserts {
     use rustc_data_structures::static_assert_size;
     // tidy-alphabetical-start
     static_assert_size!(AttrWrapper, 16);
-    static_assert_size!(LazyAttrTokenStreamImpl, 144);
+    static_assert_size!(LazyAttrTokenStreamImpl, 120);
     // tidy-alphabetical-end
 }
diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs
index 4c918c670..a051dbe9f 100644
--- a/compiler/rustc_parse/src/parser/diagnostics.rs
+++ b/compiler/rustc_parse/src/parser/diagnostics.rs
@@ -18,7 +18,8 @@ use crate::errors::{
     UseEqInstead,
 };
 
-use crate::lexer::UnmatchedBrace;
+use crate::fluent_generated as fluent;
+use crate::lexer::UnmatchedDelim;
 use crate::parser;
 use rustc_ast as ast;
 use rustc_ast::ptr::P;
@@ -32,10 +33,9 @@ use rustc_ast::{
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::{
-    fluent, Applicability, DiagnosticBuilder, DiagnosticMessage, FatalError, Handler, MultiSpan,
-    PResult,
+    pluralize, Applicability, Diagnostic, DiagnosticBuilder, DiagnosticMessage, ErrorGuaranteed,
+    FatalError, Handler, IntoDiagnostic, MultiSpan, PResult,
 };
-use rustc_errors::{pluralize, Diagnostic, ErrorGuaranteed, IntoDiagnostic};
 use rustc_session::errors::ExprParenthesesNeeded;
 use rustc_span::source_map::Spanned;
 use rustc_span::symbol::{kw, sym, Ident};
@@ -165,8 +165,6 @@ enum IsStandalone {
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 enum IncOrDec {
     Inc,
-    // FIXME: `i--` recovery isn't implemented yet
-    #[allow(dead_code)]
     Dec,
 }
 
@@ -222,7 +220,7 @@ impl MultiSugg {
 /// is dropped.
 pub struct SnapshotParser<'a> {
     parser: Parser<'a>,
-    unclosed_delims: Vec<UnmatchedBrace>,
+    unclosed_delims: Vec<UnmatchedDelim>,
 }
 
 impl<'a> Deref for SnapshotParser<'a> {
@@ -264,7 +262,7 @@ impl<'a> Parser<'a> {
         self.unclosed_delims.extend(snapshot.unclosed_delims);
     }
 
-    pub fn unclosed_delims(&self) -> &[UnmatchedBrace] {
+    pub fn unclosed_delims(&self) -> &[UnmatchedDelim] {
         &self.unclosed_delims
     }
 
@@ -284,7 +282,7 @@ impl<'a> Parser<'a> {
         self.sess.source_map().span_to_snippet(span)
     }
 
-    pub(super) fn expected_ident_found(&self) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
+    pub(super) fn expected_ident_found(&mut self) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
         let valid_follow = &[
             TokenKind::Eq,
             TokenKind::Colon,
@@ -324,7 +322,61 @@ impl<'a> Parser<'a> {
             suggest_raw,
             suggest_remove_comma,
         };
-        err.into_diagnostic(&self.sess.span_diagnostic)
+        let mut err = err.into_diagnostic(&self.sess.span_diagnostic);
+
+        // if the token we have is a `<`
+        // it *might* be a misplaced generic
+        if self.token == token::Lt {
+            // all keywords that could have generic applied
+            let valid_prev_keywords =
+                [kw::Fn, kw::Type, kw::Struct, kw::Enum, kw::Union, kw::Trait];
+
+            // If we've expected an identifier,
+            // and the current token is a '<'
+            // if the previous token is a valid keyword
+            // that might use a generic, then suggest a correct
+            // generic placement (later on)
+            let maybe_keyword = self.prev_token.clone();
+            if valid_prev_keywords.into_iter().any(|x| maybe_keyword.is_keyword(x)) {
+                // if we have a valid keyword, attempt to parse generics
+                // also obtain the keywords symbol
+                match self.parse_generics() {
+                    Ok(generic) => {
+                        if let TokenKind::Ident(symbol, _) = maybe_keyword.kind {
+                            let ident_name = symbol;
+                            // at this point, we've found something like
+                            // `fn <T>id`
+                            // and current token should be Ident with the item name (i.e. the function name)
+                            // if there is a `<` after the fn name, then don't show a suggestion, show help
+
+                            if !self.look_ahead(1, |t| *t == token::Lt) &&
+                                let Ok(snippet) = self.sess.source_map().span_to_snippet(generic.span) {
+                                    err.multipart_suggestion_verbose(
+                                        format!("place the generic parameter name after the {ident_name} name"),
+                                        vec![
+                                            (self.token.span.shrink_to_hi(), snippet),
+                                            (generic.span, String::new())
+                                        ],
+                                        Applicability::MaybeIncorrect,
+                                    );
+                                } else {
+                                    err.help(format!(
+                                        "place the generic parameter name after the {ident_name} name"
+                                    ));
+                                }
+                        }
+                    }
+                    Err(err) => {
+                        // if there's an error parsing the generics,
+                        // then don't do a misplaced generics suggestion
+                        // and emit the expected ident error instead;
+                        err.cancel();
+                    }
+                }
+            }
+        }
+
+        err
     }
 
     pub(super) fn expected_one_of_not_found(
@@ -639,7 +691,7 @@ impl<'a> Parser<'a> {
                 span: self.prev_token.span.shrink_to_lo(),
                 tokens: None,
             };
-            let struct_expr = snapshot.parse_struct_expr(None, path, false);
+            let struct_expr = snapshot.parse_expr_struct(None, path, false);
             let block_tail = self.parse_block_tail(lo, s, AttemptLocalParseRecovery::No);
             return Some(match (struct_expr, block_tail) {
                 (Ok(expr), Err(mut err)) => {
@@ -654,7 +706,7 @@ impl<'a> Parser<'a> {
                     err.delay_as_bug();
                     self.restore_snapshot(snapshot);
                     let mut tail = self.mk_block(
-                        vec![self.mk_stmt_err(expr.span)],
+                        thin_vec![self.mk_stmt_err(expr.span)],
                         s,
                         lo.to(self.prev_token.span),
                     );
@@ -1303,6 +1355,20 @@ impl<'a> Parser<'a> {
         self.recover_from_inc_dec(operand_expr, kind, op_span)
     }
 
+    pub(super) fn recover_from_postfix_decrement(
+        &mut self,
+        operand_expr: P<Expr>,
+        op_span: Span,
+        start_stmt: bool,
+    ) -> PResult<'a, P<Expr>> {
+        let kind = IncDecRecovery {
+            standalone: if start_stmt { IsStandalone::Standalone } else { IsStandalone::Subexpr },
+            op: IncOrDec::Dec,
+            fixity: UnaryFixity::Post,
+        };
+        self.recover_from_inc_dec(operand_expr, kind, op_span)
+    }
+
     fn recover_from_inc_dec(
         &mut self,
         base: P<Expr>,
@@ -1570,7 +1636,7 @@ impl<'a> Parser<'a> {
             // Handle `await { <expr> }`.
             // This needs to be handled separately from the next arm to avoid
             // interpreting `await { <expr> }?` as `<expr>?.await`.
-            self.parse_block_expr(None, self.token.span, BlockCheckMode::Default)
+            self.parse_expr_block(None, self.token.span, BlockCheckMode::Default)
         } else {
             self.parse_expr()
         }
@@ -2030,7 +2096,7 @@ impl<'a> Parser<'a> {
     }
 
     pub(super) fn recover_arg_parse(&mut self) -> PResult<'a, (P<ast::Pat>, P<ast::Ty>)> {
-        let pat = self.parse_pat_no_top_alt(Some("argument name"))?;
+        let pat = self.parse_pat_no_top_alt(Some(Expected::ArgumentName))?;
         self.expect(&token::Colon)?;
         let ty = self.parse_ty()?;
 
@@ -2121,7 +2187,7 @@ impl<'a> Parser<'a> {
     /// the parameters are *names* (so we don't emit errors about not being able to find `b` in
     /// the local scope), but if we find the same name multiple times, like in `fn foo(i8, i8)`,
     /// we deduplicate them to not complain about duplicated parameter names.
-    pub(super) fn deduplicate_recovered_params_names(&self, fn_inputs: &mut Vec<Param>) {
+    pub(super) fn deduplicate_recovered_params_names(&self, fn_inputs: &mut ThinVec<Param>) {
         let mut seen_inputs = FxHashSet::default();
         for input in fn_inputs.iter_mut() {
             let opt_ident = if let (PatKind::Ident(_, ident, _), TyKind::Err) =
@@ -2145,7 +2211,7 @@ impl<'a> Parser<'a> {
     /// like the user has forgotten them.
     pub fn handle_ambiguous_unbraced_const_arg(
         &mut self,
-        args: &mut Vec<AngleBracketedArg>,
+        args: &mut ThinVec<AngleBracketedArg>,
     ) -> PResult<'a, bool> {
         // If we haven't encountered a closing `>`, then the argument is malformed.
         // It's likely that the user has written a const expression without enclosing it
@@ -2353,6 +2419,28 @@ impl<'a> Parser<'a> {
         Err(err)
     }
 
+    /// Try to recover from an unbraced const argument whose first token [could begin a type][ty].
+    ///
+    /// [ty]: token::Token::can_begin_type
+    pub(crate) fn recover_unbraced_const_arg_that_can_begin_ty(
+        &mut self,
+        mut snapshot: SnapshotParser<'a>,
+    ) -> Option<P<ast::Expr>> {
+        match snapshot.parse_expr_res(Restrictions::CONST_EXPR, None) {
+            // Since we don't know the exact reason why we failed to parse the type or the
+            // expression, employ a simple heuristic to weed out some pathological cases.
+            Ok(expr) if let token::Comma | token::Gt = snapshot.token.kind => {
+                self.restore_snapshot(snapshot);
+                Some(expr)
+            }
+            Ok(_) => None,
+            Err(err) => {
+                err.cancel();
+                None
+            }
+        }
+    }
+
     /// Creates a dummy const argument, and reports that the expression must be enclosed in braces
     pub fn dummy_const_arg_needs_braces(
         &self,
@@ -2375,7 +2463,7 @@ impl<'a> Parser<'a> {
     pub(crate) fn maybe_recover_colon_colon_in_pat_typo(
         &mut self,
         mut first_pat: P<Pat>,
-        expected: Expected,
+        expected: Option<Expected>,
     ) -> P<Pat> {
         if token::Colon != self.token.kind {
             return first_pat;
@@ -2383,26 +2471,42 @@ impl<'a> Parser<'a> {
         if !matches!(first_pat.kind, PatKind::Ident(_, _, None) | PatKind::Path(..))
             || !self.look_ahead(1, |token| token.is_ident() && !token.is_reserved_ident())
         {
+            let mut snapshot_type = self.create_snapshot_for_diagnostic();
+            snapshot_type.bump(); // `:`
+            match snapshot_type.parse_ty() {
+                Err(inner_err) => {
+                    inner_err.cancel();
+                }
+                Ok(ty) => {
+                    let Err(mut err) = self.expected_one_of_not_found(&[], &[]) else {
+                        return first_pat;
+                    };
+                    err.span_label(ty.span, "specifying the type of a pattern isn't supported");
+                    self.restore_snapshot(snapshot_type);
+                    let span = first_pat.span.to(ty.span);
+                    first_pat = self.mk_pat(span, PatKind::Wild);
+                    err.emit();
+                }
+            }
             return first_pat;
         }
         // The pattern looks like it might be a path with a `::` -> `:` typo:
         // `match foo { bar:baz => {} }`
-        let span = self.token.span;
+        let colon_span = self.token.span;
         // We only emit "unexpected `:`" error here if we can successfully parse the
         // whole pattern correctly in that case.
-        let snapshot = self.create_snapshot_for_diagnostic();
+        let mut snapshot_pat = self.create_snapshot_for_diagnostic();
+        let mut snapshot_type = self.create_snapshot_for_diagnostic();
 
         // Create error for "unexpected `:`".
         match self.expected_one_of_not_found(&[], &[]) {
             Err(mut err) => {
-                self.bump(); // Skip the `:`.
-                match self.parse_pat_no_top_alt(expected) {
+                // Skip the `:`.
+                snapshot_pat.bump();
+                snapshot_type.bump();
+                match snapshot_pat.parse_pat_no_top_alt(expected) {
                     Err(inner_err) => {
-                        // Carry on as if we had not done anything, callers will emit a
-                        // reasonable error.
                         inner_err.cancel();
-                        err.cancel();
-                        self.restore_snapshot(snapshot);
                     }
                     Ok(mut pat) => {
                         // We've parsed the rest of the pattern.
@@ -2466,8 +2570,8 @@ impl<'a> Parser<'a> {
                             _ => {}
                         }
                         if show_sugg {
-                            err.span_suggestion(
-                                span,
+                            err.span_suggestion_verbose(
+                                colon_span.until(self.look_ahead(1, |t| t.span)),
                                 "maybe write a path separator here",
                                 "::",
                                 Applicability::MaybeIncorrect,
@@ -2475,13 +2579,24 @@ impl<'a> Parser<'a> {
                         } else {
                             first_pat = self.mk_pat(new_span, PatKind::Wild);
                         }
-                        err.emit();
+                        self.restore_snapshot(snapshot_pat);
+                    }
+                }
+                match snapshot_type.parse_ty() {
+                    Err(inner_err) => {
+                        inner_err.cancel();
+                    }
+                    Ok(ty) => {
+                        err.span_label(ty.span, "specifying the type of a pattern isn't supported");
+                        self.restore_snapshot(snapshot_type);
+                        let new_span = first_pat.span.to(ty.span);
+                        first_pat = self.mk_pat(new_span, PatKind::Wild);
                     }
                 }
+                err.emit();
             }
             _ => {
                 // Carry on as if we had not done anything. This should be unreachable.
-                self.restore_snapshot(snapshot);
             }
         };
         first_pat
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index bf93a89f0..95a7ca80d 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -1,29 +1,12 @@
 use super::diagnostics::SnapshotParser;
-use super::pat::{CommaRecoveryMode, RecoverColon, RecoverComma, PARAM_EXPECTED};
+use super::pat::{CommaRecoveryMode, Expected, RecoverColon, RecoverComma};
 use super::ty::{AllowPlus, RecoverQPath, RecoverReturnSign};
 use super::{
     AttrWrapper, BlockMode, ClosureSpans, ForceCollect, Parser, PathStyle, Restrictions,
     SemiColonMode, SeqSep, TokenExpectType, TokenType, TrailingToken,
 };
-use crate::errors::{
-    ArrayBracketsInsteadOfSpaces, ArrayBracketsInsteadOfSpacesSugg, AsyncMoveOrderIncorrect,
-    BracesForStructLiteral, CatchAfterTry, CommaAfterBaseStruct, ComparisonInterpretedAsGeneric,
-    ComparisonOrShiftInterpretedAsGenericSugg, DoCatchSyntaxRemoved, DotDotDot, EqFieldInit,
-    ExpectedElseBlock, ExpectedEqForLetExpr, ExpectedExpressionFoundLet,
-    FieldExpressionWithGeneric, FloatLiteralRequiresIntegerPart, FoundExprWouldBeStmt,
-    IfExpressionMissingCondition, IfExpressionMissingThenBlock, IfExpressionMissingThenBlockSub,
-    InvalidBlockMacroSegment, InvalidComparisonOperator, InvalidComparisonOperatorSub,
-    InvalidInterpolatedExpression, InvalidLiteralSuffixOnTupleIndex, InvalidLogicalOperator,
-    InvalidLogicalOperatorSub, LabeledLoopInBreak, LeadingPlusNotSupported, LeftArrowOperator,
-    LifetimeInBorrowExpression, MacroInvocationWithQualifiedPath, MalformedLoopLabel,
-    MatchArmBodyWithoutBraces, MatchArmBodyWithoutBracesSugg, MissingCommaAfterMatchArm,
-    MissingDotDot, MissingInInForLoop, MissingInInForLoopSub, MissingSemicolonBeforeArray,
-    NoFieldsForFnCall, NotAsNegationOperator, NotAsNegationOperatorSub,
-    OuterAttributeNotAllowedOnIfElse, ParenthesesWithStructFields,
-    RequireColonAfterLabeledExpression, ShiftInterpretedAsGeneric, StructLiteralNotAllowedHere,
-    StructLiteralNotAllowedHereSugg, TildeAsUnaryOperator, UnexpectedIfWithIf,
-    UnexpectedTokenAfterLabel, UnexpectedTokenAfterLabelSugg, WrapExpressionInParentheses,
-};
+
+use crate::errors;
 use crate::maybe_recover_from_interpolated_ty_qpath;
 use core::mem;
 use rustc_ast::ptr::P;
@@ -39,8 +22,8 @@ use rustc_ast::{Arm, Async, BlockCheckMode, Expr, ExprKind, Label, Movability, R
 use rustc_ast::{ClosureBinder, MetaItemLit, StmtKind};
 use rustc_ast_pretty::pprust;
 use rustc_errors::{
-    Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic, PResult,
-    StashKey,
+    AddToDiagnostic, Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic,
+    PResult, StashKey,
 };
 use rustc_session::errors::{report_lit_error, ExprParenthesesNeeded};
 use rustc_session::lint::builtin::BREAK_WITH_LABEL_AND_LOOP;
@@ -48,6 +31,7 @@ use rustc_session::lint::BuiltinLintDiagnostics;
 use rustc_span::source_map::{self, Span, Spanned};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{BytePos, Pos};
+use thin_vec::{thin_vec, ThinVec};
 
 /// Possibly accepts an `token::Interpolated` expression (a pre-parsed expression
 /// dropped into the token stream, which happens while parsing the result of
@@ -119,7 +103,7 @@ impl<'a> Parser<'a> {
         self.collect_tokens_no_attrs(|this| this.parse_expr())
     }
 
-    pub fn parse_anon_const_expr(&mut self) -> PResult<'a, AnonConst> {
+    pub fn parse_expr_anon_const(&mut self) -> PResult<'a, AnonConst> {
         self.parse_expr().map(|value| AnonConst { id: DUMMY_NODE_ID, value })
     }
 
@@ -141,7 +125,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses a sequence of expressions delimited by parentheses.
-    fn parse_paren_expr_seq(&mut self) -> PResult<'a, Vec<P<Expr>>> {
+    fn parse_expr_paren_seq(&mut self) -> PResult<'a, ThinVec<P<Expr>>> {
         self.parse_paren_comma_seq(|p| p.parse_expr_catch_underscore()).map(|(r, _)| r)
     }
 
@@ -152,7 +136,7 @@ impl<'a> Parser<'a> {
         r: Restrictions,
         already_parsed_attrs: Option<AttrWrapper>,
     ) -> PResult<'a, P<Expr>> {
-        self.with_res(r, |this| this.parse_assoc_expr(already_parsed_attrs))
+        self.with_res(r, |this| this.parse_expr_assoc(already_parsed_attrs))
     }
 
     /// Parses an associative expression.
@@ -160,15 +144,15 @@ impl<'a> Parser<'a> {
     /// This parses an expression accounting for associativity and precedence of the operators in
     /// the expression.
     #[inline]
-    fn parse_assoc_expr(
+    fn parse_expr_assoc(
         &mut self,
         already_parsed_attrs: Option<AttrWrapper>,
     ) -> PResult<'a, P<Expr>> {
-        self.parse_assoc_expr_with(0, already_parsed_attrs.into())
+        self.parse_expr_assoc_with(0, already_parsed_attrs.into())
     }
 
     /// Parses an associative expression with operators of at least `min_prec` precedence.
-    pub(super) fn parse_assoc_expr_with(
+    pub(super) fn parse_expr_assoc_with(
         &mut self,
         min_prec: usize,
         lhs: LhsExpr,
@@ -183,9 +167,9 @@ impl<'a> Parser<'a> {
                 _ => None,
             };
             if self.token.is_range_separator() {
-                return self.parse_prefix_range_expr(attrs);
+                return self.parse_expr_prefix_range(attrs);
             } else {
-                self.parse_prefix_expr(attrs)?
+                self.parse_expr_prefix(attrs)?
             }
         };
         let last_type_ascription_set = self.last_type_ascription.is_some();
@@ -243,10 +227,10 @@ impl<'a> Parser<'a> {
                 }
                 .into();
                 let invalid = format!("{}=", &sugg);
-                self.sess.emit_err(InvalidComparisonOperator {
+                self.sess.emit_err(errors::InvalidComparisonOperator {
                     span: sp,
                     invalid: invalid.clone(),
-                    sub: InvalidComparisonOperatorSub::Correctable {
+                    sub: errors::InvalidComparisonOperatorSub::Correctable {
                         span: sp,
                         invalid,
                         correct: sugg,
@@ -261,10 +245,10 @@ impl<'a> Parser<'a> {
                 && self.prev_token.span.hi() == self.token.span.lo()
             {
                 let sp = op.span.to(self.token.span);
-                self.sess.emit_err(InvalidComparisonOperator {
+                self.sess.emit_err(errors::InvalidComparisonOperator {
                     span: sp,
                     invalid: "<>".into(),
-                    sub: InvalidComparisonOperatorSub::Correctable {
+                    sub: errors::InvalidComparisonOperatorSub::Correctable {
                         span: sp,
                         invalid: "<>".into(),
                         correct: "!=".into(),
@@ -279,10 +263,10 @@ impl<'a> Parser<'a> {
                 && self.prev_token.span.hi() == self.token.span.lo()
             {
                 let sp = op.span.to(self.token.span);
-                self.sess.emit_err(InvalidComparisonOperator {
+                self.sess.emit_err(errors::InvalidComparisonOperator {
                     span: sp,
                     invalid: "<=>".into(),
-                    sub: InvalidComparisonOperatorSub::Spaceship(sp),
+                    sub: errors::InvalidComparisonOperatorSub::Spaceship(sp),
                 });
                 self.bump();
             }
@@ -298,6 +282,18 @@ impl<'a> Parser<'a> {
                 continue;
             }
 
+            if self.prev_token == token::BinOp(token::Minus)
+                && self.token == token::BinOp(token::Minus)
+                && self.prev_token.span.between(self.token.span).is_empty()
+                && !self.look_ahead(1, |tok| tok.can_begin_expr())
+            {
+                let op_span = self.prev_token.span.to(self.token.span);
+                // Eat the second `-`
+                self.bump();
+                lhs = self.recover_from_postfix_decrement(lhs, op_span, starts_stmt)?;
+                continue;
+            }
+
             let op = op.node;
             // Special cases:
             if op == AssocOp::As {
@@ -309,7 +305,7 @@ impl<'a> Parser<'a> {
             } else if op == AssocOp::DotDot || op == AssocOp::DotDotEq {
                 // If we didn't have to handle `x..`/`x..=`, it would be pretty easy to
                 // generalise it to the Fixity::None code.
-                lhs = self.parse_range_expr(prec, lhs, op, cur_op_span)?;
+                lhs = self.parse_expr_range(prec, lhs, op, cur_op_span)?;
                 break;
             }
 
@@ -322,7 +318,7 @@ impl<'a> Parser<'a> {
                 Fixity::None => 1,
             };
             let rhs = self.with_res(restrictions - Restrictions::STMT_EXPR, |this| {
-                this.parse_assoc_expr_with(prec + prec_adjustment, LhsExpr::NotYetParsed)
+                this.parse_expr_assoc_with(prec + prec_adjustment, LhsExpr::NotYetParsed)
             })?;
 
             let span = self.mk_expr_sp(&lhs, lhs_span, rhs.span);
@@ -419,7 +415,7 @@ impl<'a> Parser<'a> {
     /// but the next token implies this should be parsed as an expression.
     /// For example: `if let Some(x) = x { x } else { 0 } / 2`.
     fn error_found_expr_would_be_stmt(&self, lhs: &Expr) {
-        self.sess.emit_err(FoundExprWouldBeStmt {
+        self.sess.emit_err(errors::FoundExprWouldBeStmt {
             span: self.token.span,
             token: self.token.clone(),
             suggestion: ExprParenthesesNeeded::surrounding(lhs.span),
@@ -446,18 +442,18 @@ impl<'a> Parser<'a> {
             }
             (Some(op), _) => (op, self.token.span),
             (None, Some((Ident { name: sym::and, span }, false))) if self.may_recover() => {
-                self.sess.emit_err(InvalidLogicalOperator {
+                self.sess.emit_err(errors::InvalidLogicalOperator {
                     span: self.token.span,
                     incorrect: "and".into(),
-                    sub: InvalidLogicalOperatorSub::Conjunction(self.token.span),
+                    sub: errors::InvalidLogicalOperatorSub::Conjunction(self.token.span),
                 });
                 (AssocOp::LAnd, span)
             }
             (None, Some((Ident { name: sym::or, span }, false))) if self.may_recover() => {
-                self.sess.emit_err(InvalidLogicalOperator {
+                self.sess.emit_err(errors::InvalidLogicalOperator {
                     span: self.token.span,
                     incorrect: "or".into(),
-                    sub: InvalidLogicalOperatorSub::Disjunction(self.token.span),
+                    sub: errors::InvalidLogicalOperatorSub::Disjunction(self.token.span),
                 });
                 (AssocOp::LOr, span)
             }
@@ -474,7 +470,7 @@ impl<'a> Parser<'a> {
 
     /// Parses `x..y`, `x..=y`, and `x..`/`x..=`.
     /// The other two variants are handled in `parse_prefix_range_expr` below.
-    fn parse_range_expr(
+    fn parse_expr_range(
         &mut self,
         prec: usize,
         lhs: P<Expr>,
@@ -482,7 +478,7 @@ impl<'a> Parser<'a> {
         cur_op_span: Span,
     ) -> PResult<'a, P<Expr>> {
         let rhs = if self.is_at_start_of_range_notation_rhs() {
-            Some(self.parse_assoc_expr_with(prec + 1, LhsExpr::NotYetParsed)?)
+            Some(self.parse_expr_assoc_with(prec + 1, LhsExpr::NotYetParsed)?)
         } else {
             None
         };
@@ -507,7 +503,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses prefix-forms of range notation: `..expr`, `..`, `..=expr`.
-    fn parse_prefix_range_expr(&mut self, attrs: Option<AttrWrapper>) -> PResult<'a, P<Expr>> {
+    fn parse_expr_prefix_range(&mut self, attrs: Option<AttrWrapper>) -> PResult<'a, P<Expr>> {
         // Check for deprecated `...` syntax.
         if self.token == token::DotDotDot {
             self.err_dotdotdot_syntax(self.token.span);
@@ -534,7 +530,7 @@ impl<'a> Parser<'a> {
             this.bump();
             let (span, opt_end) = if this.is_at_start_of_range_notation_rhs() {
                 // RHS must be parsed with more associativity than the dots.
-                this.parse_assoc_expr_with(op.unwrap().precedence() + 1, LhsExpr::NotYetParsed)
+                this.parse_expr_assoc_with(op.unwrap().precedence() + 1, LhsExpr::NotYetParsed)
                     .map(|x| (lo.to(x.span), Some(x)))?
             } else {
                 (lo, None)
@@ -545,7 +541,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses a prefix-unary-operator expr.
-    fn parse_prefix_expr(&mut self, attrs: Option<AttrWrapper>) -> PResult<'a, P<Expr>> {
+    fn parse_expr_prefix(&mut self, attrs: Option<AttrWrapper>) -> PResult<'a, P<Expr>> {
         let attrs = self.parse_or_use_outer_attributes(attrs)?;
         let lo = self.token.span;
 
@@ -563,25 +559,28 @@ impl<'a> Parser<'a> {
         // Note: when adding new unary operators, don't forget to adjust TokenKind::can_begin_expr()
         match this.token.uninterpolate().kind {
             // `!expr`
-            token::Not => make_it!(this, attrs, |this, _| this.parse_unary_expr(lo, UnOp::Not)),
+            token::Not => make_it!(this, attrs, |this, _| this.parse_expr_unary(lo, UnOp::Not)),
             // `~expr`
             token::Tilde => make_it!(this, attrs, |this, _| this.recover_tilde_expr(lo)),
             // `-expr`
             token::BinOp(token::Minus) => {
-                make_it!(this, attrs, |this, _| this.parse_unary_expr(lo, UnOp::Neg))
+                make_it!(this, attrs, |this, _| this.parse_expr_unary(lo, UnOp::Neg))
             }
             // `*expr`
             token::BinOp(token::Star) => {
-                make_it!(this, attrs, |this, _| this.parse_unary_expr(lo, UnOp::Deref))
+                make_it!(this, attrs, |this, _| this.parse_expr_unary(lo, UnOp::Deref))
             }
             // `&expr` and `&&expr`
             token::BinOp(token::And) | token::AndAnd => {
-                make_it!(this, attrs, |this, _| this.parse_borrow_expr(lo))
+                make_it!(this, attrs, |this, _| this.parse_expr_borrow(lo))
             }
             // `+lit`
             token::BinOp(token::Plus) if this.look_ahead(1, |tok| tok.is_numeric_lit()) => {
-                let mut err =
-                    LeadingPlusNotSupported { span: lo, remove_plus: None, add_parentheses: None };
+                let mut err = errors::LeadingPlusNotSupported {
+                    span: lo,
+                    remove_plus: None,
+                    add_parentheses: None,
+                };
 
                 // a block on the LHS might have been intended to be an expression instead
                 if let Some(sp) = this.sess.ambiguous_block_expr_parse.borrow().get(&lo) {
@@ -592,7 +591,7 @@ impl<'a> Parser<'a> {
                 this.sess.emit_err(err);
 
                 this.bump();
-                this.parse_prefix_expr(None)
+                this.parse_expr_prefix(None)
             }
             // Recover from `++x`:
             token::BinOp(token::Plus)
@@ -605,41 +604,41 @@ impl<'a> Parser<'a> {
                 this.bump();
                 this.bump();
 
-                let operand_expr = this.parse_dot_or_call_expr(Default::default())?;
+                let operand_expr = this.parse_expr_dot_or_call(Default::default())?;
                 this.recover_from_prefix_increment(operand_expr, pre_span, starts_stmt)
             }
             token::Ident(..) if this.token.is_keyword(kw::Box) => {
-                make_it!(this, attrs, |this, _| this.parse_box_expr(lo))
+                make_it!(this, attrs, |this, _| this.parse_expr_box(lo))
             }
             token::Ident(..) if this.may_recover() && this.is_mistaken_not_ident_negation() => {
                 make_it!(this, attrs, |this, _| this.recover_not_expr(lo))
             }
-            _ => return this.parse_dot_or_call_expr(Some(attrs)),
+            _ => return this.parse_expr_dot_or_call(Some(attrs)),
         }
     }
 
-    fn parse_prefix_expr_common(&mut self, lo: Span) -> PResult<'a, (Span, P<Expr>)> {
+    fn parse_expr_prefix_common(&mut self, lo: Span) -> PResult<'a, (Span, P<Expr>)> {
         self.bump();
-        let expr = self.parse_prefix_expr(None);
+        let expr = self.parse_expr_prefix(None);
         let (span, expr) = self.interpolated_or_expr_span(expr)?;
         Ok((lo.to(span), expr))
     }
 
-    fn parse_unary_expr(&mut self, lo: Span, op: UnOp) -> PResult<'a, (Span, ExprKind)> {
-        let (span, expr) = self.parse_prefix_expr_common(lo)?;
+    fn parse_expr_unary(&mut self, lo: Span, op: UnOp) -> PResult<'a, (Span, ExprKind)> {
+        let (span, expr) = self.parse_expr_prefix_common(lo)?;
         Ok((span, self.mk_unary(op, expr)))
     }
 
     /// Recover on `~expr` in favor of `!expr`.
     fn recover_tilde_expr(&mut self, lo: Span) -> PResult<'a, (Span, ExprKind)> {
-        self.sess.emit_err(TildeAsUnaryOperator(lo));
+        self.sess.emit_err(errors::TildeAsUnaryOperator(lo));
 
-        self.parse_unary_expr(lo, UnOp::Not)
+        self.parse_expr_unary(lo, UnOp::Not)
     }
 
     /// Parse `box expr`.
-    fn parse_box_expr(&mut self, lo: Span) -> PResult<'a, (Span, ExprKind)> {
-        let (span, expr) = self.parse_prefix_expr_common(lo)?;
+    fn parse_expr_box(&mut self, lo: Span) -> PResult<'a, (Span, ExprKind)> {
+        let (span, expr) = self.parse_expr_prefix_common(lo)?;
         self.sess.gated_spans.gate(sym::box_syntax, span);
         Ok((span, ExprKind::Box(expr)))
     }
@@ -660,14 +659,14 @@ impl<'a> Parser<'a> {
         let negated_token = self.look_ahead(1, |t| t.clone());
 
         let sub_diag = if negated_token.is_numeric_lit() {
-            NotAsNegationOperatorSub::SuggestNotBitwise
+            errors::NotAsNegationOperatorSub::SuggestNotBitwise
         } else if negated_token.is_bool_lit() {
-            NotAsNegationOperatorSub::SuggestNotLogical
+            errors::NotAsNegationOperatorSub::SuggestNotLogical
         } else {
-            NotAsNegationOperatorSub::SuggestNotDefault
+            errors::NotAsNegationOperatorSub::SuggestNotDefault
         };
 
-        self.sess.emit_err(NotAsNegationOperator {
+        self.sess.emit_err(errors::NotAsNegationOperator {
             negated: negated_token.span,
             negated_desc: super::token_descr(&negated_token),
             // Span the `not` plus trailing whitespace to avoid
@@ -677,7 +676,7 @@ impl<'a> Parser<'a> {
             ),
         });
 
-        self.parse_unary_expr(lo, UnOp::Not)
+        self.parse_expr_unary(lo, UnOp::Not)
     }
 
     /// Returns the span of expr, if it was not interpolated or the span of the interpolated token.
@@ -735,10 +734,10 @@ impl<'a> Parser<'a> {
                                 segments[0].ident.span,
                             ),
                         };
-                        match self.parse_labeled_expr(label, false) {
+                        match self.parse_expr_labeled(label, false) {
                             Ok(expr) => {
                                 type_err.cancel();
-                                self.sess.emit_err(MalformedLoopLabel {
+                                self.sess.emit_err(errors::MalformedLoopLabel {
                                     span: label.ident.span,
                                     correct_label: label.ident,
                                 });
@@ -763,20 +762,22 @@ impl<'a> Parser<'a> {
                         );
 
                         let args_span = self.look_ahead(1, |t| t.span).to(span_after_type);
-                        let suggestion = ComparisonOrShiftInterpretedAsGenericSugg {
+                        let suggestion = errors::ComparisonOrShiftInterpretedAsGenericSugg {
                             left: expr.span.shrink_to_lo(),
                             right: expr.span.shrink_to_hi(),
                         };
 
                         match self.token.kind {
-                            token::Lt => self.sess.emit_err(ComparisonInterpretedAsGeneric {
-                                comparison: self.token.span,
-                                r#type: path,
-                                args: args_span,
-                                suggestion,
-                            }),
+                            token::Lt => {
+                                self.sess.emit_err(errors::ComparisonInterpretedAsGeneric {
+                                    comparison: self.token.span,
+                                    r#type: path,
+                                    args: args_span,
+                                    suggestion,
+                                })
+                            }
                             token::BinOp(token::Shl) => {
-                                self.sess.emit_err(ShiftInterpretedAsGeneric {
+                                self.sess.emit_err(errors::ShiftInterpretedAsGeneric {
                                     shift: self.token.span,
                                     r#type: path,
                                     args: args_span,
@@ -827,7 +828,7 @@ impl<'a> Parser<'a> {
                 ("cast", None)
             };
 
-        let with_postfix = self.parse_dot_or_call_expr_with_(cast_expr, span)?;
+        let with_postfix = self.parse_expr_dot_or_call_with_(cast_expr, span)?;
 
         // Check if an illegal postfix operator has been added after the cast.
         // If the resulting expression is not a cast, it is an illegal postfix operator.
@@ -898,15 +899,15 @@ impl<'a> Parser<'a> {
     }
 
     /// Parse `& mut? <expr>` or `& raw [ const | mut ] <expr>`.
-    fn parse_borrow_expr(&mut self, lo: Span) -> PResult<'a, (Span, ExprKind)> {
+    fn parse_expr_borrow(&mut self, lo: Span) -> PResult<'a, (Span, ExprKind)> {
         self.expect_and()?;
         let has_lifetime = self.token.is_lifetime() && self.look_ahead(1, |t| t != &token::Colon);
         let lifetime = has_lifetime.then(|| self.expect_lifetime()); // For recovery, see below.
         let (borrow_kind, mutbl) = self.parse_borrow_modifiers(lo);
         let expr = if self.token.is_range_separator() {
-            self.parse_prefix_range_expr(None)
+            self.parse_expr_prefix_range(None)
         } else {
-            self.parse_prefix_expr(None)
+            self.parse_expr_prefix(None)
         };
         let (hi, expr) = self.interpolated_or_expr_span(expr)?;
         let span = lo.to(hi);
@@ -917,7 +918,7 @@ impl<'a> Parser<'a> {
     }
 
     fn error_remove_borrow_lifetime(&self, span: Span, lt_span: Span) {
-        self.sess.emit_err(LifetimeInBorrowExpression { span, lifetime_span: lt_span });
+        self.sess.emit_err(errors::LifetimeInBorrowExpression { span, lifetime_span: lt_span });
     }
 
     /// Parse `mut?` or `raw [ const | mut ]`.
@@ -936,16 +937,16 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses `a.b` or `a(13)` or `a[4]` or just `a`.
-    fn parse_dot_or_call_expr(&mut self, attrs: Option<AttrWrapper>) -> PResult<'a, P<Expr>> {
+    fn parse_expr_dot_or_call(&mut self, attrs: Option<AttrWrapper>) -> PResult<'a, P<Expr>> {
         let attrs = self.parse_or_use_outer_attributes(attrs)?;
         self.collect_tokens_for_expr(attrs, |this, attrs| {
-            let base = this.parse_bottom_expr();
+            let base = this.parse_expr_bottom();
             let (span, base) = this.interpolated_or_expr_span(base)?;
-            this.parse_dot_or_call_expr_with(base, span, attrs)
+            this.parse_expr_dot_or_call_with(base, span, attrs)
         })
     }
 
-    pub(super) fn parse_dot_or_call_expr_with(
+    pub(super) fn parse_expr_dot_or_call_with(
         &mut self,
         e0: P<Expr>,
         lo: Span,
@@ -954,7 +955,7 @@ impl<'a> Parser<'a> {
         // Stitch the list of outer attributes onto the return value.
         // A little bit ugly, but the best way given the current code
         // structure
-        let res = self.parse_dot_or_call_expr_with_(e0, lo);
+        let res = self.parse_expr_dot_or_call_with_(e0, lo);
         if attrs.is_empty() {
             res
         } else {
@@ -968,7 +969,7 @@ impl<'a> Parser<'a> {
         }
     }
 
-    fn parse_dot_or_call_expr_with_(&mut self, mut e: P<Expr>, lo: Span) -> PResult<'a, P<Expr>> {
+    fn parse_expr_dot_or_call_with_(&mut self, mut e: P<Expr>, lo: Span) -> PResult<'a, P<Expr>> {
         loop {
             let has_question = if self.prev_token.kind == TokenKind::Ident(kw::Return, false) {
                 // we are using noexpect here because we don't expect a `?` directly after a `return`
@@ -991,15 +992,15 @@ impl<'a> Parser<'a> {
             };
             if has_dot {
                 // expr.f
-                e = self.parse_dot_suffix_expr(lo, e)?;
+                e = self.parse_expr_dot_suffix(lo, e)?;
                 continue;
             }
             if self.expr_is_complete(&e) {
                 return Ok(e);
             }
             e = match self.token.kind {
-                token::OpenDelim(Delimiter::Parenthesis) => self.parse_fn_call_expr(lo, e),
-                token::OpenDelim(Delimiter::Bracket) => self.parse_index_expr(lo, e)?,
+                token::OpenDelim(Delimiter::Parenthesis) => self.parse_expr_fn_call(lo, e),
+                token::OpenDelim(Delimiter::Bracket) => self.parse_expr_index(lo, e)?,
                 _ => return Ok(e),
             }
         }
@@ -1011,14 +1012,14 @@ impl<'a> Parser<'a> {
             && self.look_ahead(3, |t| t.can_begin_expr())
     }
 
-    fn parse_dot_suffix_expr(&mut self, lo: Span, base: P<Expr>) -> PResult<'a, P<Expr>> {
+    fn parse_expr_dot_suffix(&mut self, lo: Span, base: P<Expr>) -> PResult<'a, P<Expr>> {
         match self.token.uninterpolate().kind {
             token::Ident(..) => self.parse_dot_suffix(base, lo),
             token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) => {
-                Ok(self.parse_tuple_field_access_expr(lo, base, symbol, suffix, None))
+                Ok(self.parse_expr_tuple_field_access(lo, base, symbol, suffix, None))
             }
             token::Literal(token::Lit { kind: token::Float, symbol, suffix }) => {
-                Ok(self.parse_tuple_field_access_expr_float(lo, base, symbol, suffix))
+                Ok(self.parse_expr_tuple_field_access_float(lo, base, symbol, suffix))
             }
             _ => {
                 self.error_unexpected_after_dot();
@@ -1030,7 +1031,7 @@ impl<'a> Parser<'a> {
     fn error_unexpected_after_dot(&self) {
         // FIXME Could factor this out into non_fatal_unexpected or something.
         let actual = pprust::token_to_string(&self.token);
-        self.struct_span_err(self.token.span, &format!("unexpected token: `{actual}`")).emit();
+        self.sess.emit_err(errors::UnexpectedTokenAfterDot { span: self.token.span, actual });
     }
 
     // We need an identifier or integer, but the next token is a float.
@@ -1040,7 +1041,7 @@ impl<'a> Parser<'a> {
     // support pushing "future tokens" (would be also helpful to `break_and_eat`), or
     // we should break everything including floats into more basic proc-macro style
     // tokens in the lexer (probably preferable).
-    fn parse_tuple_field_access_expr_float(
+    fn parse_expr_tuple_field_access_float(
         &mut self,
         lo: Span,
         base: P<Expr>,
@@ -1083,7 +1084,7 @@ impl<'a> Parser<'a> {
         match &*components {
             // 1e2
             [IdentLike(i)] => {
-                self.parse_tuple_field_access_expr(lo, base, Symbol::intern(&i), suffix, None)
+                self.parse_expr_tuple_field_access(lo, base, Symbol::intern(&i), suffix, None)
             }
             // 1.
             [IdentLike(i), Punct('.')] => {
@@ -1099,7 +1100,7 @@ impl<'a> Parser<'a> {
                 let symbol = Symbol::intern(&i);
                 self.token = Token::new(token::Ident(symbol, false), ident_span);
                 let next_token = (Token::new(token::Dot, dot_span), self.token_spacing);
-                self.parse_tuple_field_access_expr(lo, base, symbol, None, Some(next_token))
+                self.parse_expr_tuple_field_access(lo, base, symbol, None, Some(next_token))
             }
             // 1.2 | 1.2e3
             [IdentLike(i1), Punct('.'), IdentLike(i2)] => {
@@ -1120,11 +1121,11 @@ impl<'a> Parser<'a> {
                 // See issue #76399 and PR #76285 for more details
                 let next_token1 = (Token::new(token::Dot, dot_span), Spacing::Alone);
                 let base1 =
-                    self.parse_tuple_field_access_expr(lo, base, symbol1, None, Some(next_token1));
+                    self.parse_expr_tuple_field_access(lo, base, symbol1, None, Some(next_token1));
                 let symbol2 = Symbol::intern(&i2);
                 let next_token2 = Token::new(token::Ident(symbol2, false), ident2_span);
                 self.bump_with((next_token2, self.token_spacing)); // `.`
-                self.parse_tuple_field_access_expr(lo, base1, symbol2, suffix, None)
+                self.parse_expr_tuple_field_access(lo, base1, symbol2, suffix, None)
             }
             // 1e+ | 1e- (recovered)
             [IdentLike(_), Punct('+' | '-')] |
@@ -1142,7 +1143,7 @@ impl<'a> Parser<'a> {
         }
     }
 
-    fn parse_tuple_field_access_expr(
+    fn parse_expr_tuple_field_access(
         &mut self,
         lo: Span,
         base: P<Expr>,
@@ -1163,7 +1164,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parse a function call expression, `expr(...)`.
-    fn parse_fn_call_expr(&mut self, lo: Span, fun: P<Expr>) -> P<Expr> {
+    fn parse_expr_fn_call(&mut self, lo: Span, fun: P<Expr>) -> P<Expr> {
         let snapshot = if self.token.kind == token::OpenDelim(Delimiter::Parenthesis)
             && self.look_ahead_type_ascription_as_field()
         {
@@ -1174,7 +1175,7 @@ impl<'a> Parser<'a> {
         let open_paren = self.token.span;
 
         let mut seq = self
-            .parse_paren_expr_seq()
+            .parse_expr_paren_seq()
             .map(|args| self.mk_expr(lo.to(self.prev_token.span), self.mk_call(fun, args)));
         if let Some(expr) =
             self.maybe_recover_struct_lit_bad_delims(lo, open_paren, &mut seq, snapshot)
@@ -1209,16 +1210,21 @@ impl<'a> Parser<'a> {
                         // `Enum::Foo { a: 3, b: 4 }` or `Enum::Foo(3, 4)`.
                         self.restore_snapshot(snapshot);
                         let close_paren = self.prev_token.span;
-                        let span = lo.to(self.prev_token.span);
-                        if !fields.is_empty() {
-                            let mut replacement_err = ParenthesesWithStructFields {
+                        let span = lo.to(close_paren);
+                        if !fields.is_empty() &&
+                            // `token.kind` should not be compared here.
+                            // This is because the `snapshot.token.kind` is treated as the same as
+                            // that of the open delim in `TokenTreesReader::parse_token_tree`, even if they are different.
+                            self.span_to_snippet(close_paren).map_or(false, |snippet| snippet == ")")
+                        {
+                            let mut replacement_err = errors::ParenthesesWithStructFields {
                                 span,
                                 r#type: path,
-                                braces_for_struct: BracesForStructLiteral {
+                                braces_for_struct: errors::BracesForStructLiteral {
                                     first: open_paren,
                                     second: close_paren,
                                 },
-                                no_fields_for_fn: NoFieldsForFnCall {
+                                no_fields_for_fn: errors::NoFieldsForFnCall {
                                     fields: fields
                                         .into_iter()
                                         .map(|field| field.span.until(field.expr.span))
@@ -1247,7 +1253,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parse an indexing expression `expr[...]`.
-    fn parse_index_expr(&mut self, lo: Span, base: P<Expr>) -> PResult<'a, P<Expr>> {
+    fn parse_expr_index(&mut self, lo: Span, base: P<Expr>) -> PResult<'a, P<Expr>> {
         let prev_span = self.prev_token.span;
         let open_delim_span = self.token.span;
         self.bump(); // `[`
@@ -1270,7 +1276,7 @@ impl<'a> Parser<'a> {
 
         if self.check(&token::OpenDelim(Delimiter::Parenthesis)) {
             // Method call `expr.f()`
-            let args = self.parse_paren_expr_seq()?;
+            let args = self.parse_expr_paren_seq()?;
             let fn_span = fn_span_lo.to(self.prev_token.span);
             let span = lo.to(self.prev_token.span);
             Ok(self.mk_expr(
@@ -1285,7 +1291,7 @@ impl<'a> Parser<'a> {
         } else {
             // Field access `expr.f`
             if let Some(args) = seg.args {
-                self.sess.emit_err(FieldExpressionWithGeneric(args.span()));
+                self.sess.emit_err(errors::FieldExpressionWithGeneric(args.span()));
             }
 
             let span = lo.to(self.prev_token.span);
@@ -1298,7 +1304,7 @@ impl<'a> Parser<'a> {
     ///
     /// N.B., this does not parse outer attributes, and is private because it only works
     /// correctly if called from `parse_dot_or_call_expr()`.
-    fn parse_bottom_expr(&mut self) -> PResult<'a, P<Expr>> {
+    fn parse_expr_bottom(&mut self) -> PResult<'a, P<Expr>> {
         maybe_recover_from_interpolated_ty_qpath!(self, true);
         maybe_whole_expr!(self);
 
@@ -1311,13 +1317,13 @@ impl<'a> Parser<'a> {
             // This match arm is a special-case of the `_` match arm below and
             // could be removed without changing functionality, but it's faster
             // to have it here, especially for programs with large constants.
-            self.parse_lit_expr()
+            self.parse_expr_lit()
         } else if self.check(&token::OpenDelim(Delimiter::Parenthesis)) {
-            self.parse_tuple_parens_expr()
+            self.parse_expr_tuple_parens()
         } else if self.check(&token::OpenDelim(Delimiter::Brace)) {
-            self.parse_block_expr(None, lo, BlockCheckMode::Default)
+            self.parse_expr_block(None, lo, BlockCheckMode::Default)
         } else if self.check(&token::BinOp(token::Or)) || self.check(&token::OrOr) {
-            self.parse_closure_expr().map_err(|mut err| {
+            self.parse_expr_closure().map_err(|mut err| {
                 // If the input is something like `if a { 1 } else { 2 } | if a { 3 } else { 4 }`
                 // then suggest parens around the lhs.
                 if let Some(sp) = self.sess.ambiguous_block_expr_parse.borrow().get(&lo) {
@@ -1326,42 +1332,42 @@ impl<'a> Parser<'a> {
                 err
             })
         } else if self.check(&token::OpenDelim(Delimiter::Bracket)) {
-            self.parse_array_or_repeat_expr(Delimiter::Bracket)
+            self.parse_expr_array_or_repeat(Delimiter::Bracket)
         } else if self.check_path() {
-            self.parse_path_start_expr()
+            self.parse_expr_path_start()
         } else if self.check_keyword(kw::Move)
             || self.check_keyword(kw::Static)
             || self.check_const_closure()
         {
-            self.parse_closure_expr()
+            self.parse_expr_closure()
         } else if self.eat_keyword(kw::If) {
-            self.parse_if_expr()
+            self.parse_expr_if()
         } else if self.check_keyword(kw::For) {
             if self.choose_generics_over_qpath(1) {
-                self.parse_closure_expr()
+                self.parse_expr_closure()
             } else {
                 assert!(self.eat_keyword(kw::For));
-                self.parse_for_expr(None, self.prev_token.span)
+                self.parse_expr_for(None, self.prev_token.span)
             }
         } else if self.eat_keyword(kw::While) {
-            self.parse_while_expr(None, self.prev_token.span)
+            self.parse_expr_while(None, self.prev_token.span)
         } else if let Some(label) = self.eat_label() {
-            self.parse_labeled_expr(label, true)
+            self.parse_expr_labeled(label, true)
         } else if self.eat_keyword(kw::Loop) {
             let sp = self.prev_token.span;
-            self.parse_loop_expr(None, self.prev_token.span).map_err(|mut err| {
+            self.parse_expr_loop(None, self.prev_token.span).map_err(|mut err| {
                 err.span_label(sp, "while parsing this `loop` expression");
                 err
             })
         } else if self.eat_keyword(kw::Match) {
             let match_sp = self.prev_token.span;
-            self.parse_match_expr().map_err(|mut err| {
+            self.parse_expr_match().map_err(|mut err| {
                 err.span_label(match_sp, "while parsing this `match` expression");
                 err
             })
         } else if self.eat_keyword(kw::Unsafe) {
             let sp = self.prev_token.span;
-            self.parse_block_expr(None, lo, BlockCheckMode::Unsafe(ast::UserProvided)).map_err(
+            self.parse_expr_block(None, lo, BlockCheckMode::Unsafe(ast::UserProvided)).map_err(
                 |mut err| {
                     err.span_label(sp, "while parsing this `unsafe` expression");
                     err
@@ -1375,17 +1381,17 @@ impl<'a> Parser<'a> {
             self.expect_keyword(kw::Try)?;
             self.parse_try_block(lo)
         } else if self.eat_keyword(kw::Return) {
-            self.parse_return_expr()
+            self.parse_expr_return()
         } else if self.eat_keyword(kw::Continue) {
-            self.parse_continue_expr(lo)
+            self.parse_expr_continue(lo)
         } else if self.eat_keyword(kw::Break) {
-            self.parse_break_expr()
+            self.parse_expr_break()
         } else if self.eat_keyword(kw::Yield) {
-            self.parse_yield_expr()
+            self.parse_expr_yield()
         } else if self.is_do_yeet() {
-            self.parse_yeet_expr()
+            self.parse_expr_yeet()
         } else if self.check_keyword(kw::Let) {
-            self.parse_let_expr()
+            self.parse_expr_let()
         } else if self.eat_keyword(kw::Underscore) {
             Ok(self.mk_expr(self.prev_token.span, ExprKind::Underscore))
         } else if !self.unclosed_delims.is_empty() && self.check(&token::Semi) {
@@ -1408,19 +1414,19 @@ impl<'a> Parser<'a> {
                     // Check for `async {` and `async move {`.
                     self.parse_async_block()
                 } else {
-                    self.parse_closure_expr()
+                    self.parse_expr_closure()
                 }
             } else if self.eat_keyword(kw::Await) {
                 self.recover_incorrect_await_syntax(lo, self.prev_token.span)
             } else {
-                self.parse_lit_expr()
+                self.parse_expr_lit()
             }
         } else {
-            self.parse_lit_expr()
+            self.parse_expr_lit()
         }
     }
 
-    fn parse_lit_expr(&mut self) -> PResult<'a, P<Expr>> {
+    fn parse_expr_lit(&mut self) -> PResult<'a, P<Expr>> {
         let lo = self.token.span;
         match self.parse_opt_token_lit() {
             Some((token_lit, _)) => {
@@ -1431,7 +1437,7 @@ impl<'a> Parser<'a> {
         }
     }
 
-    fn parse_tuple_parens_expr(&mut self) -> PResult<'a, P<Expr>> {
+    fn parse_expr_tuple_parens(&mut self) -> PResult<'a, P<Expr>> {
         let lo = self.token.span;
         self.expect(&token::OpenDelim(Delimiter::Parenthesis))?;
         let (es, trailing_comma) = match self.parse_seq_to_end(
@@ -1455,20 +1461,20 @@ impl<'a> Parser<'a> {
         self.maybe_recover_from_bad_qpath(expr)
     }
 
-    fn parse_array_or_repeat_expr(&mut self, close_delim: Delimiter) -> PResult<'a, P<Expr>> {
+    fn parse_expr_array_or_repeat(&mut self, close_delim: Delimiter) -> PResult<'a, P<Expr>> {
         let lo = self.token.span;
         self.bump(); // `[` or other open delim
 
         let close = &token::CloseDelim(close_delim);
         let kind = if self.eat(close) {
             // Empty vector
-            ExprKind::Array(Vec::new())
+            ExprKind::Array(ThinVec::new())
         } else {
             // Non-empty vector
             let first_expr = self.parse_expr()?;
             if self.eat(&token::Semi) {
                 // Repeating array syntax: `[ 0; 512 ]`
-                let count = self.parse_anon_const_expr()?;
+                let count = self.parse_expr_anon_const()?;
                 self.expect(close)?;
                 ExprKind::Repeat(first_expr, count)
             } else if self.eat(&token::Comma) {
@@ -1480,14 +1486,14 @@ impl<'a> Parser<'a> {
             } else {
                 // Vector with one element
                 self.expect(close)?;
-                ExprKind::Array(vec![first_expr])
+                ExprKind::Array(thin_vec![first_expr])
             }
         };
         let expr = self.mk_expr(lo.to(self.prev_token.span), kind);
         self.maybe_recover_from_bad_qpath(expr)
     }
 
-    fn parse_path_start_expr(&mut self) -> PResult<'a, P<Expr>> {
+    fn parse_expr_path_start(&mut self) -> PResult<'a, P<Expr>> {
         let (qself, path) = if self.eat_lt() {
             let (qself, path) = self.parse_qpath(PathStyle::Expr)?;
             (Some(qself), path)
@@ -1499,7 +1505,7 @@ impl<'a> Parser<'a> {
         let (span, kind) = if self.eat(&token::Not) {
             // MACRO INVOCATION expression
             if qself.is_some() {
-                self.sess.emit_err(MacroInvocationWithQualifiedPath(path.span));
+                self.sess.emit_err(errors::MacroInvocationWithQualifiedPath(path.span));
             }
             let lo = path.span;
             let mac = P(MacCall {
@@ -1524,7 +1530,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parse `'label: $expr`. The label is already parsed.
-    fn parse_labeled_expr(
+    fn parse_expr_labeled(
         &mut self,
         label_: Label,
         mut consume_colon: bool,
@@ -1533,15 +1539,15 @@ impl<'a> Parser<'a> {
         let label = Some(label_);
         let ate_colon = self.eat(&token::Colon);
         let expr = if self.eat_keyword(kw::While) {
-            self.parse_while_expr(label, lo)
+            self.parse_expr_while(label, lo)
         } else if self.eat_keyword(kw::For) {
-            self.parse_for_expr(label, lo)
+            self.parse_expr_for(label, lo)
         } else if self.eat_keyword(kw::Loop) {
-            self.parse_loop_expr(label, lo)
+            self.parse_expr_loop(label, lo)
         } else if self.check_noexpect(&token::OpenDelim(Delimiter::Brace))
             || self.token.is_whole_block()
         {
-            self.parse_block_expr(label, lo, BlockCheckMode::Default)
+            self.parse_expr_block(label, lo, BlockCheckMode::Default)
         } else if !ate_colon
             && self.may_recover()
             && (matches!(self.token.kind, token::CloseDelim(_) | token::Comma)
@@ -1549,7 +1555,7 @@ impl<'a> Parser<'a> {
         {
             let (lit, _) =
                 self.recover_unclosed_char(label_.ident, Parser::mk_token_lit_char, |self_| {
-                    self_.sess.create_err(UnexpectedTokenAfterLabel {
+                    self_.sess.create_err(errors::UnexpectedTokenAfterLabel {
                         span: self_.token.span,
                         remove_label: None,
                         enclose_in_block: None,
@@ -1561,7 +1567,7 @@ impl<'a> Parser<'a> {
             && (self.check_noexpect(&TokenKind::Comma) || self.check_noexpect(&TokenKind::Gt))
         {
             // We're probably inside of a `Path<'a>` that needs a turbofish
-            self.sess.emit_err(UnexpectedTokenAfterLabel {
+            self.sess.emit_err(errors::UnexpectedTokenAfterLabel {
                 span: self.token.span,
                 remove_label: None,
                 enclose_in_block: None,
@@ -1569,7 +1575,7 @@ impl<'a> Parser<'a> {
             consume_colon = false;
             Ok(self.mk_expr_err(lo))
         } else {
-            let mut err = UnexpectedTokenAfterLabel {
+            let mut err = errors::UnexpectedTokenAfterLabel {
                 span: self.token.span,
                 remove_label: None,
                 enclose_in_block: None,
@@ -1605,14 +1611,14 @@ impl<'a> Parser<'a> {
                     return expr;
                 }
 
-                err.enclose_in_block = Some(UnexpectedTokenAfterLabelSugg {
+                err.enclose_in_block = Some(errors::UnexpectedTokenAfterLabelSugg {
                     left: span.shrink_to_lo(),
                     right: span.shrink_to_hi(),
                 });
 
                 // Replace `'label: non_block_expr` with `'label: {non_block_expr}` in order to suppress future errors about `break 'label`.
                 let stmt = self.mk_stmt(span, StmtKind::Expr(expr));
-                let blk = self.mk_block(vec![stmt], BlockCheckMode::Default, span);
+                let blk = self.mk_block(thin_vec![stmt], BlockCheckMode::Default, span);
                 self.mk_expr(span, ExprKind::Block(blk, label))
             });
 
@@ -1621,7 +1627,7 @@ impl<'a> Parser<'a> {
         }?;
 
         if !ate_colon && consume_colon {
-            self.sess.emit_err(RequireColonAfterLabeledExpression {
+            self.sess.emit_err(errors::RequireColonAfterLabeledExpression {
                 span: expr.span,
                 label: lo,
                 label_end: lo.shrink_to_hi(),
@@ -1670,7 +1676,7 @@ impl<'a> Parser<'a> {
         self.bump(); // `catch`
 
         let span = lo.to(self.prev_token.span);
-        self.sess.emit_err(DoCatchSyntaxRemoved { span });
+        self.sess.emit_err(errors::DoCatchSyntaxRemoved { span });
 
         self.parse_try_block(lo)
     }
@@ -1681,7 +1687,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parse `"return" expr?`.
-    fn parse_return_expr(&mut self) -> PResult<'a, P<Expr>> {
+    fn parse_expr_return(&mut self) -> PResult<'a, P<Expr>> {
         let lo = self.prev_token.span;
         let kind = ExprKind::Ret(self.parse_expr_opt()?);
         let expr = self.mk_expr(lo.to(self.prev_token.span), kind);
@@ -1689,7 +1695,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parse `"do" "yeet" expr?`.
-    fn parse_yeet_expr(&mut self) -> PResult<'a, P<Expr>> {
+    fn parse_expr_yeet(&mut self) -> PResult<'a, P<Expr>> {
         let lo = self.token.span;
 
         self.bump(); // `do`
@@ -1711,16 +1717,16 @@ impl<'a> Parser<'a> {
     /// `break 'lbl: loop {}`); a labeled break with an unlabeled loop as its value
     /// expression only gets a warning for compatibility reasons; and a labeled break
     /// with a labeled loop does not even get a warning because there is no ambiguity.
-    fn parse_break_expr(&mut self) -> PResult<'a, P<Expr>> {
+    fn parse_expr_break(&mut self) -> PResult<'a, P<Expr>> {
         let lo = self.prev_token.span;
         let mut label = self.eat_label();
         let kind = if self.token == token::Colon && let Some(label) = label.take() {
             // The value expression can be a labeled loop, see issue #86948, e.g.:
             // `loop { break 'label: loop { break 'label 42; }; }`
-            let lexpr = self.parse_labeled_expr(label, true)?;
-            self.sess.emit_err(LabeledLoopInBreak {
+            let lexpr = self.parse_expr_labeled(label, true)?;
+            self.sess.emit_err(errors::LabeledLoopInBreak {
                 span: lexpr.span,
-                sub: WrapExpressionInParentheses {
+                sub: errors::WrapExpressionInParentheses {
                     left: lexpr.span.shrink_to_lo(),
                     right: lexpr.span.shrink_to_hi(),
                 },
@@ -1770,7 +1776,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parse `"continue" label?`.
-    fn parse_continue_expr(&mut self, lo: Span) -> PResult<'a, P<Expr>> {
+    fn parse_expr_continue(&mut self, lo: Span) -> PResult<'a, P<Expr>> {
         let mut label = self.eat_label();
 
         // Recover `continue label` -> `continue 'label`
@@ -1787,7 +1793,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parse `"yield" expr?`.
-    fn parse_yield_expr(&mut self) -> PResult<'a, P<Expr>> {
+    fn parse_expr_yield(&mut self) -> PResult<'a, P<Expr>> {
         let lo = self.prev_token.span;
         let kind = ExprKind::Yield(self.parse_expr_opt()?);
         let span = lo.to(self.prev_token.span);
@@ -1840,7 +1846,7 @@ impl<'a> Parser<'a> {
             };
             if let Some(expr) = expr {
                 if matches!(expr.kind, ExprKind::Err) {
-                    let mut err = InvalidInterpolatedExpression { span: self.token.span }
+                    let mut err = errors::InvalidInterpolatedExpression { span: self.token.span }
                         .into_diagnostic(&self.sess.span_diagnostic);
                     err.downgrade_to_delayed_bug();
                     return Err(err);
@@ -1882,7 +1888,16 @@ impl<'a> Parser<'a> {
                 if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) =
                     next_token.kind
                 {
-                    if self.token.span.hi() == next_token.span.lo() {
+                    // If this integer looks like a float, then recover as such.
+                    //
+                    // We will never encounter the exponent part of a floating
+                    // point literal here, since there's no use of the exponent
+                    // syntax that also constitutes a valid integer, so we need
+                    // not check for that.
+                    if suffix.map_or(true, |s| s == sym::f32 || s == sym::f64)
+                        && symbol.as_str().chars().all(|c| c.is_numeric() || c == '_')
+                        && self.token.span.hi() == next_token.span.lo()
+                    {
                         let s = String::from("0.") + symbol.as_str();
                         let kind = TokenKind::lit(token::Float, Symbol::intern(&s), suffix);
                         return Some(Token::new(kind, self.token.span.to(next_token.span)));
@@ -1892,7 +1907,7 @@ impl<'a> Parser<'a> {
             });
             if let Some(token) = &recovered {
                 self.bump();
-                self.sess.emit_err(FloatLiteralRequiresIntegerPart {
+                self.sess.emit_err(errors::FloatLiteralRequiresIntegerPart {
                     span: token.span,
                     correct: pprust::token_to_string(token).into_owned(),
                 });
@@ -1953,13 +1968,17 @@ impl<'a> Parser<'a> {
         if [sym::i32, sym::u32, sym::isize, sym::usize].contains(&suffix) {
             // #59553: warn instead of reject out of hand to allow the fix to percolate
             // through the ecosystem when people fix their macros
-            self.sess.emit_warning(InvalidLiteralSuffixOnTupleIndex {
+            self.sess.emit_warning(errors::InvalidLiteralSuffixOnTupleIndex {
                 span,
                 suffix,
                 exception: Some(()),
             });
         } else {
-            self.sess.emit_err(InvalidLiteralSuffixOnTupleIndex { span, suffix, exception: None });
+            self.sess.emit_err(errors::InvalidLiteralSuffixOnTupleIndex {
+                span,
+                suffix,
+                exception: None,
+            });
         }
     }
 
@@ -1991,11 +2010,11 @@ impl<'a> Parser<'a> {
     /// expression.
     fn maybe_suggest_brackets_instead_of_braces(&mut self, lo: Span) -> Option<P<Expr>> {
         let mut snapshot = self.create_snapshot_for_diagnostic();
-        match snapshot.parse_array_or_repeat_expr(Delimiter::Brace) {
+        match snapshot.parse_expr_array_or_repeat(Delimiter::Brace) {
             Ok(arr) => {
-                self.sess.emit_err(ArrayBracketsInsteadOfSpaces {
+                self.sess.emit_err(errors::ArrayBracketsInsteadOfSpaces {
                     span: arr.span,
-                    sub: ArrayBracketsInsteadOfSpacesSugg {
+                    sub: errors::ArrayBracketsInsteadOfSpacesSugg {
                         left: lo,
                         right: snapshot.prev_token.span,
                     },
@@ -2041,7 +2060,7 @@ impl<'a> Parser<'a> {
                         .span_to_snippet(snapshot.token.span)
                         .map_or(false, |snippet| snippet == "]") =>
                 {
-                    return Err(MissingSemicolonBeforeArray {
+                    return Err(errors::MissingSemicolonBeforeArray {
                         open_delim: open_delim_span,
                         semicolon: prev_span.shrink_to_hi(),
                     }.into_diagnostic(&self.sess.span_diagnostic));
@@ -2054,7 +2073,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses a block or unsafe block.
-    pub(super) fn parse_block_expr(
+    pub(super) fn parse_expr_block(
         &mut self,
         opt_label: Option<Label>,
         lo: Span,
@@ -2067,7 +2086,7 @@ impl<'a> Parser<'a> {
         }
 
         if self.token.is_whole_block() {
-            self.sess.emit_err(InvalidBlockMacroSegment {
+            self.sess.emit_err(errors::InvalidBlockMacroSegment {
                 span: self.token.span,
                 context: lo.to(self.token.span),
             });
@@ -2084,7 +2103,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses a closure expression (e.g., `move |args| expr`).
-    fn parse_closure_expr(&mut self) -> PResult<'a, P<Expr>> {
+    fn parse_expr_closure(&mut self) -> PResult<'a, P<Expr>> {
         let lo = self.token.span;
 
         let binder = if self.check_keyword(kw::For) {
@@ -2094,12 +2113,12 @@ impl<'a> Parser<'a> {
 
             self.sess.gated_spans.gate(sym::closure_lifetime_binder, span);
 
-            ClosureBinder::For { span, generic_params: P::from_vec(lifetime_defs) }
+            ClosureBinder::For { span, generic_params: lifetime_defs }
         } else {
             ClosureBinder::NotPresent
         };
 
-        let constness = self.parse_constness(Case::Sensitive);
+        let constness = self.parse_closure_constness(Case::Sensitive);
 
         let movability =
             if self.eat_keyword(kw::Static) { Movability::Static } else { Movability::Movable };
@@ -2121,7 +2140,7 @@ impl<'a> Parser<'a> {
             _ => {
                 // If an explicit return type is given, require a block to appear (RFC 968).
                 let body_lo = self.token.span;
-                self.parse_block_expr(None, body_lo, BlockCheckMode::Default)?
+                self.parse_expr_block(None, body_lo, BlockCheckMode::Default)?
             }
         };
 
@@ -2131,7 +2150,7 @@ impl<'a> Parser<'a> {
         }
 
         if self.token.kind == TokenKind::Semi
-            && matches!(self.token_cursor.frame.delim_sp, Some((Delimiter::Parenthesis, _)))
+            && matches!(self.token_cursor.stack.last(), Some((_, Delimiter::Parenthesis, _)))
             && self.may_recover()
         {
             // It is likely that the closure body is a block but where the
@@ -2171,7 +2190,7 @@ impl<'a> Parser<'a> {
             // Check for `move async` and recover
             if self.check_keyword(kw::Async) {
                 let move_async_span = self.token.span.with_lo(self.prev_token.span.data().lo);
-                Err(AsyncMoveOrderIncorrect { span: move_async_span }
+                Err(errors::AsyncMoveOrderIncorrect { span: move_async_span }
                     .into_diagnostic(&self.sess.span_diagnostic))
             } else {
                 Ok(CaptureBy::Value)
@@ -2186,7 +2205,7 @@ impl<'a> Parser<'a> {
         let arg_start = self.token.span.lo();
 
         let inputs = if self.eat(&token::OrOr) {
-            Vec::new()
+            ThinVec::new()
         } else {
             self.expect(&token::BinOp(token::Or))?;
             let args = self
@@ -2212,7 +2231,7 @@ impl<'a> Parser<'a> {
         let lo = self.token.span;
         let attrs = self.parse_outer_attributes()?;
         self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| {
-            let pat = this.parse_pat_no_top_alt(PARAM_EXPECTED)?;
+            let pat = this.parse_pat_no_top_alt(Some(Expected::ParameterName))?;
             let ty = if this.eat(&token::Colon) {
                 this.parse_ty()?
             } else {
@@ -2234,9 +2253,9 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses an `if` expression (`if` token already eaten).
-    fn parse_if_expr(&mut self) -> PResult<'a, P<Expr>> {
+    fn parse_expr_if(&mut self) -> PResult<'a, P<Expr>> {
         let lo = self.prev_token.span;
-        let cond = self.parse_cond_expr()?;
+        let cond = self.parse_expr_cond()?;
         self.parse_if_after_cond(lo, cond)
     }
 
@@ -2249,16 +2268,17 @@ impl<'a> Parser<'a> {
             let block = match &mut cond.kind {
                 ExprKind::Binary(Spanned { span: binop_span, .. }, _, right)
                     if let ExprKind::Block(_, None) = right.kind => {
-                        self.sess.emit_err(IfExpressionMissingThenBlock {
+                        self.sess.emit_err(errors::IfExpressionMissingThenBlock {
                             if_span: lo,
-                            sub: IfExpressionMissingThenBlockSub::UnfinishedCondition(
-                                cond_span.shrink_to_lo().to(*binop_span)
-                            ),
+                            missing_then_block_sub:
+                                errors::IfExpressionMissingThenBlockSub::UnfinishedCondition(cond_span.shrink_to_lo().to(*binop_span)),
+                                let_else_sub: None,
+
                         });
                         std::mem::replace(right, this.mk_expr_err(binop_span.shrink_to_hi()))
                     },
                 ExprKind::Block(_, None) => {
-                    self.sess.emit_err(IfExpressionMissingCondition {
+                    self.sess.emit_err(errors::IfExpressionMissingCondition {
                         if_span: lo.shrink_to_hi(),
                         block_span: self.sess.source_map().start_point(cond_span),
                     });
@@ -2279,9 +2299,15 @@ impl<'a> Parser<'a> {
             if let Some(block) = recover_block_from_condition(self) {
                 block
             } else {
-                self.sess.emit_err(IfExpressionMissingThenBlock {
+                let let_else_sub = matches!(cond.kind, ExprKind::Let(..))
+                    .then(|| errors::IfExpressionLetSomeSub { if_span: lo.until(cond_span) });
+
+                self.sess.emit_err(errors::IfExpressionMissingThenBlock {
                     if_span: lo,
-                    sub: IfExpressionMissingThenBlockSub::AddThenBlock(cond_span.shrink_to_hi()),
+                    missing_then_block_sub: errors::IfExpressionMissingThenBlockSub::AddThenBlock(
+                        cond_span.shrink_to_hi(),
+                    ),
+                    let_else_sub,
                 });
                 self.mk_block_err(cond_span.shrink_to_hi())
             }
@@ -2307,12 +2333,12 @@ impl<'a> Parser<'a> {
             self.error_on_if_block_attrs(lo, false, block.span, attrs);
             block
         };
-        let els = if self.eat_keyword(kw::Else) { Some(self.parse_else_expr()?) } else { None };
+        let els = if self.eat_keyword(kw::Else) { Some(self.parse_expr_else()?) } else { None };
         Ok(self.mk_expr(lo.to(self.prev_token.span), ExprKind::If(cond, thn, els)))
     }
 
     /// Parses the condition of a `if` or `while` expression.
-    fn parse_cond_expr(&mut self) -> PResult<'a, P<Expr>> {
+    fn parse_expr_cond(&mut self) -> PResult<'a, P<Expr>> {
         let cond =
             self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL | Restrictions::ALLOW_LET, None)?;
 
@@ -2325,7 +2351,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses a `let $pat = $expr` pseudo-expression.
-    fn parse_let_expr(&mut self) -> PResult<'a, P<Expr>> {
+    fn parse_expr_let(&mut self) -> PResult<'a, P<Expr>> {
         // This is a *approximate* heuristic that detects if `let` chains are
         // being parsed in the right position. It's approximate because it
         // doesn't deny all invalid `let` expressions, just completely wrong usages.
@@ -2334,7 +2360,7 @@ impl<'a> Parser<'a> {
             TokenKind::AndAnd | TokenKind::Ident(kw::If, _) | TokenKind::Ident(kw::While, _)
         );
         if !self.restrictions.contains(Restrictions::ALLOW_LET) || not_in_chain {
-            self.sess.emit_err(ExpectedExpressionFoundLet { span: self.token.span });
+            self.sess.emit_err(errors::ExpectedExpressionFoundLet { span: self.token.span });
         }
 
         self.bump(); // Eat `let` token
@@ -2346,7 +2372,7 @@ impl<'a> Parser<'a> {
             CommaRecoveryMode::LikelyTuple,
         )?;
         if self.token == token::EqEq {
-            self.sess.emit_err(ExpectedEqForLetExpr {
+            self.sess.emit_err(errors::ExpectedEqForLetExpr {
                 span: self.token.span,
                 sugg_span: self.token.span,
             });
@@ -2355,7 +2381,7 @@ impl<'a> Parser<'a> {
             self.expect(&token::Eq)?;
         }
         let expr = self.with_res(self.restrictions | Restrictions::NO_STRUCT_LITERAL, |this| {
-            this.parse_assoc_expr_with(1 + prec_let_scrutinee_needs_par(), None.into())
+            this.parse_expr_assoc_with(1 + prec_let_scrutinee_needs_par(), None.into())
         })?;
         let span = lo.to(expr.span);
         self.sess.gated_spans.gate(sym::let_chains, span);
@@ -2363,11 +2389,11 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses an `else { ... }` expression (`else` token already eaten).
-    fn parse_else_expr(&mut self) -> PResult<'a, P<Expr>> {
+    fn parse_expr_else(&mut self) -> PResult<'a, P<Expr>> {
         let else_span = self.prev_token.span; // `else`
         let attrs = self.parse_outer_attributes()?; // For recovery.
         let expr = if self.eat_keyword(kw::If) {
-            self.parse_if_expr()?
+            self.parse_expr_if()?
         } else if self.check(&TokenKind::OpenDelim(Delimiter::Brace)) {
             self.parse_simple_block()?
         } else {
@@ -2381,7 +2407,7 @@ impl<'a> Parser<'a> {
                     if self.check(&TokenKind::OpenDelim(Delimiter::Brace))
                         && classify::expr_requires_semi_to_be_stmt(&cond) =>
                 {
-                    self.sess.emit_err(ExpectedElseBlock {
+                    self.sess.emit_err(errors::ExpectedElseBlock {
                         first_tok_span,
                         first_tok,
                         else_span,
@@ -2421,7 +2447,7 @@ impl<'a> Parser<'a> {
             [x0 @ xn] | [x0, .., xn] => (x0.span.to(xn.span), xn.span),
         };
         let ctx = if is_ctx_else { "else" } else { "if" };
-        self.sess.emit_err(OuterAttributeNotAllowedOnIfElse {
+        self.sess.emit_err(errors::OuterAttributeNotAllowedOnIfElse {
             last,
             branch_span,
             ctx_span,
@@ -2434,14 +2460,14 @@ impl<'a> Parser<'a> {
         if let ExprKind::Binary(Spanned { span: binop_span, node: binop}, _, right) = &cond.kind &&
             let BinOpKind::And = binop &&
             let ExprKind::If(cond, ..) = &right.kind {
-                    Err(self.sess.create_err(UnexpectedIfWithIf(binop_span.shrink_to_hi().to(cond.span.shrink_to_lo()))))
+                    Err(self.sess.create_err(errors::UnexpectedIfWithIf(binop_span.shrink_to_hi().to(cond.span.shrink_to_lo()))))
             } else {
                 Ok(())
             }
     }
 
     /// Parses `for <src_pat> in <src_expr> <src_loop_block>` (`for` token already eaten).
-    fn parse_for_expr(&mut self, opt_label: Option<Label>, lo: Span) -> PResult<'a, P<Expr>> {
+    fn parse_expr_for(&mut self, opt_label: Option<Label>, lo: Span) -> PResult<'a, P<Expr>> {
         // Record whether we are about to parse `for (`.
         // This is used below for recovery in case of `for ( $stuff ) $block`
         // in which case we will suggest `for $stuff $block`.
@@ -2464,28 +2490,61 @@ impl<'a> Parser<'a> {
 
         let pat = self.recover_parens_around_for_head(pat, begin_paren);
 
+        // Recover from missing expression in `for` loop
+        if matches!(expr.kind, ExprKind::Block(..))
+            && !matches!(self.token.kind, token::OpenDelim(token::Delimiter::Brace))
+            && self.may_recover()
+        {
+            self.sess
+                .emit_err(errors::MissingExpressionInForLoop { span: expr.span.shrink_to_lo() });
+            let err_expr = self.mk_expr(expr.span, ExprKind::Err);
+            let block = self.mk_block(thin_vec![], BlockCheckMode::Default, self.prev_token.span);
+            return Ok(self.mk_expr(
+                lo.to(self.prev_token.span),
+                ExprKind::ForLoop(pat, err_expr, block, opt_label),
+            ));
+        }
+
         let (attrs, loop_block) = self.parse_inner_attrs_and_block()?;
 
         let kind = ExprKind::ForLoop(pat, expr, loop_block, opt_label);
+
+        self.recover_loop_else("for", lo)?;
+
         Ok(self.mk_expr_with_attrs(lo.to(self.prev_token.span), kind, attrs))
     }
 
+    /// Recovers from an `else` clause after a loop (`for...else`, `while...else`)
+    fn recover_loop_else(&mut self, loop_kind: &'static str, loop_kw: Span) -> PResult<'a, ()> {
+        if self.token.is_keyword(kw::Else) && self.may_recover() {
+            let else_span = self.token.span;
+            self.bump();
+            let else_clause = self.parse_expr_else()?;
+            self.sess.emit_err(errors::LoopElseNotSupported {
+                span: else_span.to(else_clause.span),
+                loop_kind,
+                loop_kw,
+            });
+        }
+        Ok(())
+    }
+
     fn error_missing_in_for_loop(&mut self) {
         let (span, sub): (_, fn(_) -> _) = if self.token.is_ident_named(sym::of) {
             // Possibly using JS syntax (#75311).
             let span = self.token.span;
             self.bump();
-            (span, MissingInInForLoopSub::InNotOf)
+            (span, errors::MissingInInForLoopSub::InNotOf)
         } else {
-            (self.prev_token.span.between(self.token.span), MissingInInForLoopSub::AddIn)
+            (self.prev_token.span.between(self.token.span), errors::MissingInInForLoopSub::AddIn)
         };
 
-        self.sess.emit_err(MissingInInForLoop { span, sub: sub(span) });
+        self.sess.emit_err(errors::MissingInInForLoop { span, sub: sub(span) });
     }
 
     /// Parses a `while` or `while let` expression (`while` token already eaten).
-    fn parse_while_expr(&mut self, opt_label: Option<Label>, lo: Span) -> PResult<'a, P<Expr>> {
-        let cond = self.parse_cond_expr().map_err(|mut err| {
+    fn parse_expr_while(&mut self, opt_label: Option<Label>, lo: Span) -> PResult<'a, P<Expr>> {
+        let cond = self.parse_expr_cond().map_err(|mut err| {
             err.span_label(lo, "while parsing the condition of this `while` expression");
             err
         })?;
@@ -2494,6 +2553,9 @@ impl<'a> Parser<'a> {
             err.span_label(cond.span, "this `while` condition successfully parsed");
             err
         })?;
+
+        self.recover_loop_else("while", lo)?;
+
         Ok(self.mk_expr_with_attrs(
             lo.to(self.prev_token.span),
             ExprKind::While(cond, body, opt_label),
@@ -2502,9 +2564,10 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses `loop { ... }` (`loop` token already eaten).
-    fn parse_loop_expr(&mut self, opt_label: Option<Label>, lo: Span) -> PResult<'a, P<Expr>> {
+    fn parse_expr_loop(&mut self, opt_label: Option<Label>, lo: Span) -> PResult<'a, P<Expr>> {
         let loop_span = self.prev_token.span;
         let (attrs, body) = self.parse_inner_attrs_and_block()?;
+        self.recover_loop_else("loop", lo)?;
         Ok(self.mk_expr_with_attrs(
             lo.to(self.prev_token.span),
             ExprKind::Loop(body, opt_label, loop_span),
@@ -2520,7 +2583,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses a `match ... { ... }` expression (`match` token already eaten).
-    fn parse_match_expr(&mut self) -> PResult<'a, P<Expr>> {
+    fn parse_expr_match(&mut self) -> PResult<'a, P<Expr>> {
         let match_span = self.prev_token.span;
         let lo = self.prev_token.span;
         let scrutinee = self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, None)?;
@@ -2542,7 +2605,7 @@ impl<'a> Parser<'a> {
         }
         let attrs = self.parse_inner_attributes()?;
 
-        let mut arms: Vec<Arm> = Vec::new();
+        let mut arms = ThinVec::new();
         while self.token != token::CloseDelim(Delimiter::Brace) {
             match self.parse_arm() {
                 Ok(arm) => arms.push(arm),
@@ -2584,17 +2647,17 @@ impl<'a> Parser<'a> {
         let err = |this: &Parser<'_>, stmts: Vec<ast::Stmt>| {
             let span = stmts[0].span.to(stmts[stmts.len() - 1].span);
 
-            this.sess.emit_err(MatchArmBodyWithoutBraces {
+            this.sess.emit_err(errors::MatchArmBodyWithoutBraces {
                 statements: span,
                 arrow: arrow_span,
                 num_statements: stmts.len(),
                 sub: if stmts.len() > 1 {
-                    MatchArmBodyWithoutBracesSugg::AddBraces {
+                    errors::MatchArmBodyWithoutBracesSugg::AddBraces {
                         left: span.shrink_to_lo(),
                         right: span.shrink_to_hi(),
                     }
                 } else {
-                    MatchArmBodyWithoutBracesSugg::UseComma { semicolon: semi_sp }
+                    errors::MatchArmBodyWithoutBracesSugg::UseComma { semicolon: semi_sp }
                 },
             });
             this.mk_expr_err(span)
@@ -2700,6 +2763,14 @@ impl<'a> Parser<'a> {
                     );
                     err.emit();
                     this.bump();
+                } else if matches!(
+                    (&this.prev_token.kind, &this.token.kind),
+                    (token::DotDotEq, token::Gt)
+                ) {
+                    // `error_inclusive_range_match_arrow` handles cases like `0..=> {}`,
+                    // so we supress the error here
+                    err.delay_as_bug();
+                    this.bump();
                 } else {
                     return Err(err);
                 }
@@ -2777,7 +2848,7 @@ impl<'a> Parser<'a> {
                                 .is_ok();
                             if pattern_follows && snapshot.check(&TokenKind::FatArrow) {
                                 err.cancel();
-                                this.sess.emit_err(MissingCommaAfterMatchArm {
+                                this.sess.emit_err(errors::MissingCommaAfterMatchArm {
                                     span: hi.shrink_to_hi(),
                                 });
                                 return Ok(true);
@@ -2809,7 +2880,7 @@ impl<'a> Parser<'a> {
     fn parse_try_block(&mut self, span_lo: Span) -> PResult<'a, P<Expr>> {
         let (attrs, body) = self.parse_inner_attrs_and_block()?;
         if self.eat_keyword(kw::Catch) {
-            Err(CatchAfterTry { span: self.prev_token.span }
+            Err(errors::CatchAfterTry { span: self.prev_token.span }
                 .into_diagnostic(&self.sess.span_diagnostic))
         } else {
             let span = span_lo.to(body.span);
@@ -2882,12 +2953,12 @@ impl<'a> Parser<'a> {
             if let Err(err) = self.expect(&token::OpenDelim(Delimiter::Brace)) {
                 return Some(Err(err));
             }
-            let expr = self.parse_struct_expr(qself.clone(), path.clone(), true);
+            let expr = self.parse_expr_struct(qself.clone(), path.clone(), true);
             if let (Ok(expr), false) = (&expr, struct_allowed) {
                 // This is a struct literal, but we don't can't accept them here.
-                self.sess.emit_err(StructLiteralNotAllowedHere {
+                self.sess.emit_err(errors::StructLiteralNotAllowedHere {
                     span: expr.span,
-                    sub: StructLiteralNotAllowedHereSugg {
+                    sub: errors::StructLiteralNotAllowedHereSugg {
                         left: path.span.shrink_to_lo(),
                         right: expr.span.shrink_to_hi(),
                     },
@@ -2903,15 +2974,15 @@ impl<'a> Parser<'a> {
         pth: ast::Path,
         recover: bool,
         close_delim: Delimiter,
-    ) -> PResult<'a, (Vec<ExprField>, ast::StructRest, bool)> {
-        let mut fields = Vec::new();
+    ) -> PResult<'a, (ThinVec<ExprField>, ast::StructRest, bool)> {
+        let mut fields = ThinVec::new();
         let mut base = ast::StructRest::None;
         let mut recover_async = false;
 
         let mut async_block_err = |e: &mut Diagnostic, span: Span| {
             recover_async = true;
-            e.span_label(span, "`async` blocks are only allowed in Rust 2018 or later");
-            e.help_use_latest_edition();
+            errors::AsyncBlockIn2015 { span }.add_to_diagnostic(e);
+            errors::HelpUseLatestEdition::new().add_to_diagnostic(e);
         };
 
         while self.token != token::CloseDelim(close_delim) {
@@ -3011,7 +3082,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Precondition: already parsed the '{'.
-    pub(super) fn parse_struct_expr(
+    pub(super) fn parse_expr_struct(
         &mut self,
         qself: Option<P<ast::QSelf>>,
         pth: ast::Path,
@@ -3055,7 +3126,7 @@ impl<'a> Parser<'a> {
         if self.token != token::Comma {
             return;
         }
-        self.sess.emit_err(CommaAfterBaseStruct {
+        self.sess.emit_err(errors::CommaAfterBaseStruct {
             span: span.to(self.prev_token.span),
             comma: self.token.span,
         });
@@ -3068,7 +3139,7 @@ impl<'a> Parser<'a> {
         {
             // recover from typo of `...`, suggest `..`
             let span = self.prev_token.span;
-            self.sess.emit_err(MissingDotDot { token_span: span, sugg_span: span });
+            self.sess.emit_err(errors::MissingDotDot { token_span: span, sugg_span: span });
             return true;
         }
         false
@@ -3136,18 +3207,18 @@ impl<'a> Parser<'a> {
             return;
         }
 
-        self.sess.emit_err(EqFieldInit {
+        self.sess.emit_err(errors::EqFieldInit {
             span: self.token.span,
             eq: field_name.span.shrink_to_hi().to(self.token.span),
         });
     }
 
     fn err_dotdotdot_syntax(&self, span: Span) {
-        self.sess.emit_err(DotDotDot { span });
+        self.sess.emit_err(errors::DotDotDot { span });
     }
 
     fn err_larrow_operator(&self, span: Span) {
-        self.sess.emit_err(LeftArrowOperator { span });
+        self.sess.emit_err(errors::LeftArrowOperator { span });
     }
 
     fn mk_assign_op(&self, binop: BinOp, lhs: P<Expr>, rhs: P<Expr>) -> ExprKind {
@@ -3161,7 +3232,7 @@ impl<'a> Parser<'a> {
         limits: RangeLimits,
     ) -> ExprKind {
         if end.is_none() && limits == RangeLimits::Closed {
-            self.inclusive_range_with_incorrect_end(self.prev_token.span);
+            self.inclusive_range_with_incorrect_end();
             ExprKind::Err
         } else {
             ExprKind::Range(start, end, limits)
@@ -3180,7 +3251,7 @@ impl<'a> Parser<'a> {
         ExprKind::Index(expr, idx)
     }
 
-    fn mk_call(&self, f: P<Expr>, args: Vec<P<Expr>>) -> ExprKind {
+    fn mk_call(&self, f: P<Expr>, args: ThinVec<P<Expr>>) -> ExprKind {
         ExprKind::Call(f, args)
     }
 
diff --git a/compiler/rustc_parse/src/parser/generics.rs b/compiler/rustc_parse/src/parser/generics.rs
index 8ba811715..8d0f168e0 100644
--- a/compiler/rustc_parse/src/parser/generics.rs
+++ b/compiler/rustc_parse/src/parser/generics.rs
@@ -1,4 +1,8 @@
-use crate::errors::{WhereClauseBeforeTupleStructBody, WhereClauseBeforeTupleStructBodySugg};
+use crate::errors::{
+    MultipleWhereClauses, UnexpectedDefaultValueForLifetimeInGenericParameters,
+    UnexpectedSelfInGenericParameters, WhereClauseBeforeTupleStructBody,
+    WhereClauseBeforeTupleStructBodySugg,
+};
 
 use super::{ForceCollect, Parser, TrailingToken};
 
@@ -10,10 +14,11 @@ use rustc_ast::{
 use rustc_errors::{Applicability, PResult};
 use rustc_span::symbol::{kw, Ident};
 use rustc_span::Span;
+use thin_vec::ThinVec;
 
 enum PredicateOrStructBody {
     Predicate(ast::WherePredicate),
-    StructBody(Vec<ast::FieldDef>),
+    StructBody(ThinVec<ast::FieldDef>),
 }
 
 impl<'a> Parser<'a> {
@@ -117,8 +122,8 @@ impl<'a> Parser<'a> {
 
     /// Parses a (possibly empty) list of lifetime and type parameters, possibly including
     /// a trailing comma and erroneous trailing attributes.
-    pub(super) fn parse_generic_params(&mut self) -> PResult<'a, Vec<ast::GenericParam>> {
-        let mut params = Vec::new();
+    pub(super) fn parse_generic_params(&mut self) -> PResult<'a, ThinVec<ast::GenericParam>> {
+        let mut params = ThinVec::new();
         let mut done = false;
         while !done {
             let attrs = self.parse_outer_attributes()?;
@@ -127,12 +132,9 @@ impl<'a> Parser<'a> {
                     if this.eat_keyword_noexpect(kw::SelfUpper) {
                         // `Self` as a generic param is invalid. Here we emit the diagnostic and continue parsing
                         // as if `Self` never existed.
-                        this.struct_span_err(
-                            this.prev_token.span,
-                            "unexpected keyword `Self` in generic parameters",
-                        )
-                        .note("you cannot use `Self` as a generic parameter because it is reserved for associated items")
-                        .emit();
+                        this.sess.emit_err(UnexpectedSelfInGenericParameters {
+                            span: this.prev_token.span,
+                        });
 
                         this.eat(&token::Comma);
                     }
@@ -145,6 +147,20 @@ impl<'a> Parser<'a> {
                         } else {
                             (None, Vec::new())
                         };
+
+                        if this.check_noexpect(&token::Eq)
+                            && this.look_ahead(1, |t| t.is_lifetime())
+                        {
+                            let lo = this.token.span;
+                            // Parse `= 'lifetime`.
+                            this.bump(); // `=`
+                            this.bump(); // `'lifetime`
+                            let span = lo.to(this.prev_token.span);
+                            this.sess.emit_err(
+                                UnexpectedDefaultValueForLifetimeInGenericParameters { span },
+                            );
+                        }
+
                         Some(ast::GenericParam {
                             ident: lifetime.ident,
                             id: lifetime.id,
@@ -236,13 +252,13 @@ impl<'a> Parser<'a> {
             self.expect_gt()?;
             (params, span_lo.to(self.prev_token.span))
         } else {
-            (vec![], self.prev_token.span.shrink_to_hi())
+            (ThinVec::new(), self.prev_token.span.shrink_to_hi())
         };
         Ok(ast::Generics {
             params,
             where_clause: WhereClause {
                 has_where_token: false,
-                predicates: Vec::new(),
+                predicates: ThinVec::new(),
                 span: self.prev_token.span.shrink_to_hi(),
             },
             span,
@@ -262,17 +278,17 @@ impl<'a> Parser<'a> {
         &mut self,
         struct_name: Ident,
         body_insertion_point: Span,
-    ) -> PResult<'a, (WhereClause, Option<Vec<ast::FieldDef>>)> {
+    ) -> PResult<'a, (WhereClause, Option<ThinVec<ast::FieldDef>>)> {
         self.parse_where_clause_common(Some((struct_name, body_insertion_point)))
     }
 
     fn parse_where_clause_common(
         &mut self,
         struct_: Option<(Ident, Span)>,
-    ) -> PResult<'a, (WhereClause, Option<Vec<ast::FieldDef>>)> {
+    ) -> PResult<'a, (WhereClause, Option<ThinVec<ast::FieldDef>>)> {
         let mut where_clause = WhereClause {
             has_where_token: false,
-            predicates: Vec::new(),
+            predicates: ThinVec::new(),
             span: self.prev_token.span.shrink_to_hi(),
         };
         let mut tuple_struct_body = None;
@@ -329,16 +345,11 @@ impl<'a> Parser<'a> {
             let ate_comma = self.eat(&token::Comma);
 
             if self.eat_keyword_noexpect(kw::Where) {
-                let msg = "cannot define duplicate `where` clauses on an item";
-                let mut err = self.struct_span_err(self.token.span, msg);
-                err.span_label(pred_lo, "previous `where` clause starts here");
-                err.span_suggestion_verbose(
-                    prev_token.shrink_to_hi().to(self.prev_token.span),
-                    "consider joining the two `where` clauses into one",
-                    ",",
-                    Applicability::MaybeIncorrect,
-                );
-                err.emit();
+                self.sess.emit_err(MultipleWhereClauses {
+                    span: self.token.span,
+                    previous: pred_lo,
+                    between: prev_token.shrink_to_hi().to(self.prev_token.span),
+                });
             } else if !ate_comma {
                 break;
             }
diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
index 53680a82b..9d9ae154a 100644
--- a/compiler/rustc_parse/src/parser/item.rs
+++ b/compiler/rustc_parse/src/parser/item.rs
@@ -1,9 +1,8 @@
-use crate::errors::{DocCommentDoesNotDocumentAnything, UseEmptyBlockNotSemi};
+use crate::errors;
 
 use super::diagnostics::{dummy_arg, ConsumeClosingDelim};
 use super::ty::{AllowPlus, RecoverQPath, RecoverReturnSign};
 use super::{AttrWrapper, FollowedByType, ForceCollect, Parser, PathStyle, TrailingToken};
-use crate::errors::FnTypoWithImpl;
 use rustc_ast::ast::*;
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Delimiter, TokenKind};
@@ -16,14 +15,18 @@ use rustc_ast::{EnumDef, FieldDef, Generics, TraitRef, Ty, TyKind, Variant, Vari
 use rustc_ast::{FnHeader, ForeignItem, Path, PathSegment, Visibility, VisibilityKind};
 use rustc_ast::{MacCall, MacDelimiter};
 use rustc_ast_pretty::pprust;
-use rustc_errors::{struct_span_err, Applicability, IntoDiagnostic, PResult, StashKey};
+use rustc_errors::{
+    struct_span_err, Applicability, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic, PResult,
+    StashKey,
+};
+use rustc_span::edit_distance::edit_distance;
 use rustc_span::edition::Edition;
-use rustc_span::lev_distance::lev_distance;
 use rustc_span::source_map::{self, Span};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::DUMMY_SP;
+use std::fmt::Write;
 use std::mem;
-use thin_vec::ThinVec;
+use thin_vec::{thin_vec, ThinVec};
 
 impl<'a> Parser<'a> {
     /// Parses a source module as a crate. This is the main entry point for the parser.
@@ -53,12 +56,12 @@ impl<'a> Parser<'a> {
     pub fn parse_mod(
         &mut self,
         term: &TokenKind,
-    ) -> PResult<'a, (AttrVec, Vec<P<Item>>, ModSpans)> {
+    ) -> PResult<'a, (AttrVec, ThinVec<P<Item>>, ModSpans)> {
         let lo = self.token.span;
         let attrs = self.parse_inner_attributes()?;
 
         let post_attr_lo = self.token.span;
-        let mut items = vec![];
+        let mut items = ThinVec::new();
         while let Some(item) = self.parse_item(ForceCollect::No)? {
             items.push(item);
             self.maybe_consume_incorrect_semicolon(&items);
@@ -163,35 +166,18 @@ impl<'a> Parser<'a> {
         }
 
         // At this point, we have failed to parse an item.
-        self.error_on_unmatched_vis(&vis);
-        self.error_on_unmatched_defaultness(def);
-        if !attrs_allowed {
-            self.recover_attrs_no_item(&attrs)?;
+        if !matches!(vis.kind, VisibilityKind::Inherited) {
+            self.sess.emit_err(errors::VisibilityNotFollowedByItem { span: vis.span, vis });
         }
-        Ok(None)
-    }
 
-    /// Error in-case a non-inherited visibility was parsed but no item followed.
-    fn error_on_unmatched_vis(&self, vis: &Visibility) {
-        if let VisibilityKind::Inherited = vis.kind {
-            return;
+        if let Defaultness::Default(span) = def {
+            self.sess.emit_err(errors::DefaultNotFollowedByItem { span });
         }
-        let vs = pprust::vis_to_string(&vis);
-        let vs = vs.trim_end();
-        self.struct_span_err(vis.span, &format!("visibility `{vs}` is not followed by an item"))
-            .span_label(vis.span, "the visibility")
-            .help(&format!("you likely meant to define an item, e.g., `{vs} fn foo() {{}}`"))
-            .emit();
-    }
 
-    /// Error in-case a `default` was parsed but no item followed.
-    fn error_on_unmatched_defaultness(&self, def: Defaultness) {
-        if let Defaultness::Default(sp) = def {
-            self.struct_span_err(sp, "`default` is not followed by an item")
-                .span_label(sp, "the `default` qualifier")
-                .note("only `fn`, `const`, `type`, or `impl` items may be prefixed by `default`")
-                .emit();
+        if !attrs_allowed {
+            self.recover_attrs_no_item(&attrs)?;
         }
+        Ok(None)
     }
 
     /// Error in-case `default` was parsed in an in-appropriate context.
@@ -384,86 +370,74 @@ impl<'a> Parser<'a> {
         let sp = self.prev_token.span.between(self.token.span);
         let full_sp = self.prev_token.span.to(self.token.span);
         let ident_sp = self.token.span;
-        if self.look_ahead(1, |t| *t == token::OpenDelim(Delimiter::Brace)) {
+
+        let ident = if self.look_ahead(1, |t| {
+            [
+                token::Lt,
+                token::OpenDelim(Delimiter::Brace),
+                token::OpenDelim(Delimiter::Parenthesis),
+            ]
+            .contains(&t.kind)
+        }) {
+            self.parse_ident().unwrap()
+        } else {
+            return Ok(());
+        };
+
+        let mut found_generics = false;
+        if self.check(&token::Lt) {
+            found_generics = true;
+            self.eat_to_tokens(&[&token::Gt]);
+            self.bump(); // `>`
+        }
+
+        let err = if self.check(&token::OpenDelim(Delimiter::Brace)) {
             // possible public struct definition where `struct` was forgotten
-            let ident = self.parse_ident().unwrap();
-            let msg = format!("add `struct` here to parse `{ident}` as a public struct");
-            let mut err = self.struct_span_err(sp, "missing `struct` for struct definition");
-            err.span_suggestion_short(
-                sp,
-                &msg,
-                " struct ",
-                Applicability::MaybeIncorrect, // speculative
-            );
-            Err(err)
-        } else if self.look_ahead(1, |t| *t == token::OpenDelim(Delimiter::Parenthesis)) {
-            let ident = self.parse_ident().unwrap();
+            Some(errors::MissingKeywordForItemDefinition::Struct { span: sp, ident })
+        } else if self.check(&token::OpenDelim(Delimiter::Parenthesis)) {
+            // possible public function or tuple struct definition where `fn`/`struct` was
+            // forgotten
             self.bump(); // `(`
-            let kw_name = self.recover_first_param();
+            let is_method = self.recover_self_param();
+
             self.consume_block(Delimiter::Parenthesis, ConsumeClosingDelim::Yes);
-            let (kw, kw_name, ambiguous) = if self.check(&token::RArrow) {
-                self.eat_to_tokens(&[&token::OpenDelim(Delimiter::Brace)]);
-                self.bump(); // `{`
-                ("fn", kw_name, false)
-            } else if self.check(&token::OpenDelim(Delimiter::Brace)) {
-                self.bump(); // `{`
-                ("fn", kw_name, false)
-            } else if self.check(&token::Colon) {
-                let kw = "struct";
-                (kw, kw, false)
-            } else {
-                ("fn` or `struct", "function or struct", true)
-            };
 
-            let msg = format!("missing `{kw}` for {kw_name} definition");
-            let mut err = self.struct_span_err(sp, &msg);
-            if !ambiguous {
-                self.consume_block(Delimiter::Brace, ConsumeClosingDelim::Yes);
-                let suggestion =
-                    format!("add `{kw}` here to parse `{ident}` as a public {kw_name}");
-                err.span_suggestion_short(
-                    sp,
-                    &suggestion,
-                    format!(" {kw} "),
-                    Applicability::MachineApplicable,
-                );
-            } else if let Ok(snippet) = self.span_to_snippet(ident_sp) {
-                err.span_suggestion(
-                    full_sp,
-                    "if you meant to call a macro, try",
-                    format!("{}!", snippet),
-                    // this is the `ambiguous` conditional branch
-                    Applicability::MaybeIncorrect,
-                );
-            } else {
-                err.help(
-                    "if you meant to call a macro, remove the `pub` \
-                              and add a trailing `!` after the identifier",
-                );
-            }
-            Err(err)
-        } else if self.look_ahead(1, |t| *t == token::Lt) {
-            let ident = self.parse_ident().unwrap();
-            self.eat_to_tokens(&[&token::Gt]);
-            self.bump(); // `>`
-            let (kw, kw_name, ambiguous) = if self.eat(&token::OpenDelim(Delimiter::Parenthesis)) {
-                ("fn", self.recover_first_param(), false)
-            } else if self.check(&token::OpenDelim(Delimiter::Brace)) {
-                ("struct", "struct", false)
-            } else {
-                ("fn` or `struct", "function or struct", true)
-            };
-            let msg = format!("missing `{kw}` for {kw_name} definition");
-            let mut err = self.struct_span_err(sp, &msg);
-            if !ambiguous {
-                err.span_suggestion_short(
-                    sp,
-                    &format!("add `{kw}` here to parse `{ident}` as a public {kw_name}"),
-                    format!(" {} ", kw),
-                    Applicability::MachineApplicable,
-                );
-            }
-            Err(err)
+            let err =
+                if self.check(&token::RArrow) || self.check(&token::OpenDelim(Delimiter::Brace)) {
+                    self.eat_to_tokens(&[&token::OpenDelim(Delimiter::Brace)]);
+                    self.bump(); // `{`
+                    self.consume_block(Delimiter::Brace, ConsumeClosingDelim::Yes);
+                    if is_method {
+                        errors::MissingKeywordForItemDefinition::Method { span: sp, ident }
+                    } else {
+                        errors::MissingKeywordForItemDefinition::Function { span: sp, ident }
+                    }
+                } else if self.check(&token::Semi) {
+                    errors::MissingKeywordForItemDefinition::Struct { span: sp, ident }
+                } else {
+                    errors::MissingKeywordForItemDefinition::Ambiguous {
+                        span: sp,
+                        subdiag: if found_generics {
+                            None
+                        } else if let Ok(snippet) = self.span_to_snippet(ident_sp) {
+                            Some(errors::AmbiguousMissingKwForItemSub::SuggestMacro {
+                                span: full_sp,
+                                snippet,
+                            })
+                        } else {
+                            Some(errors::AmbiguousMissingKwForItemSub::HelpMacro)
+                        },
+                    }
+                };
+            Some(err)
+        } else if found_generics {
+            Some(errors::MissingKeywordForItemDefinition::Ambiguous { span: sp, subdiag: None })
+        } else {
+            None
+        };
+
+        if let Some(err) = err {
+            Err(err.into_diagnostic(&self.sess.span_diagnostic))
         } else {
             Ok(())
         }
@@ -485,7 +459,8 @@ impl<'a> Parser<'a> {
                 // Maybe the user misspelled `macro_rules` (issue #91227)
                 if self.token.is_ident()
                     && path.segments.len() == 1
-                    && lev_distance("macro_rules", &path.segments[0].ident.to_string(), 3).is_some()
+                    && edit_distance("macro_rules", &path.segments[0].ident.to_string(), 2)
+                        .is_some()
                 {
                     err.span_suggestion(
                         path.span,
@@ -512,16 +487,13 @@ impl<'a> Parser<'a> {
         let mut err = self.struct_span_err(end.span, msg);
         if end.is_doc_comment() {
             err.span_label(end.span, "this doc comment doesn't document anything");
-        }
-        if end.meta_kind().is_some() {
-            if self.token.kind == TokenKind::Semi {
-                err.span_suggestion_verbose(
-                    self.token.span,
-                    "consider removing this semicolon",
-                    "",
-                    Applicability::MaybeIncorrect,
-                );
-            }
+        } else if self.token.kind == TokenKind::Semi {
+            err.span_suggestion_verbose(
+                self.token.span,
+                "consider removing this semicolon",
+                "",
+                Applicability::MaybeIncorrect,
+            );
         }
         if let [.., penultimate, _] = attrs {
             err.span_label(start.span.to(penultimate.span), "other attributes here");
@@ -588,20 +560,11 @@ impl<'a> Parser<'a> {
         let ty_first = if self.token.is_keyword(kw::For) && self.look_ahead(1, |t| t != &token::Lt)
         {
             let span = self.prev_token.span.between(self.token.span);
-            self.struct_span_err(span, "missing trait in a trait impl")
-                .span_suggestion(
-                    span,
-                    "add a trait here",
-                    " Trait ",
-                    Applicability::HasPlaceholders,
-                )
-                .span_suggestion(
-                    span.to(self.token.span),
-                    "for an inherent impl, drop this `for`",
-                    "",
-                    Applicability::MaybeIncorrect,
-                )
-                .emit();
+            self.sess.emit_err(errors::MissingTraitInTraitImpl {
+                span,
+                for_span: span.to(self.token.span),
+            });
+
             P(Ty {
                 kind: TyKind::Path(None, err_path(span)),
                 span,
@@ -634,14 +597,7 @@ impl<'a> Parser<'a> {
             Some(ty_second) => {
                 // impl Trait for Type
                 if !has_for {
-                    self.struct_span_err(missing_for_span, "missing `for` in a trait impl")
-                        .span_suggestion_short(
-                            missing_for_span,
-                            "add `for` here",
-                            " for ",
-                            Applicability::MachineApplicable,
-                        )
-                        .emit();
+                    self.sess.emit_err(errors::MissingForInTraitImpl { span: missing_for_span });
                 }
 
                 let ty_first = ty_first.into_inner();
@@ -649,7 +605,9 @@ impl<'a> Parser<'a> {
                     // This notably includes paths passed through `ty` macro fragments (#46438).
                     TyKind::Path(None, path) => path,
                     _ => {
-                        self.struct_span_err(ty_first.span, "expected a trait, found type").emit();
+                        self.sess.emit_err(errors::ExpectedTraitInTraitImplFoundType {
+                            span: ty_first.span,
+                        });
                         err_path(ty_first.span)
                     }
                 };
@@ -688,20 +646,20 @@ impl<'a> Parser<'a> {
         &mut self,
         attrs: &mut AttrVec,
         mut parse_item: impl FnMut(&mut Parser<'a>) -> PResult<'a, Option<Option<T>>>,
-    ) -> PResult<'a, Vec<T>> {
+    ) -> PResult<'a, ThinVec<T>> {
         let open_brace_span = self.token.span;
 
         // Recover `impl Ty;` instead of `impl Ty {}`
         if self.token == TokenKind::Semi {
-            self.sess.emit_err(UseEmptyBlockNotSemi { span: self.token.span });
+            self.sess.emit_err(errors::UseEmptyBlockNotSemi { span: self.token.span });
             self.bump();
-            return Ok(vec![]);
+            return Ok(ThinVec::new());
         }
 
         self.expect(&token::OpenDelim(Delimiter::Brace))?;
         attrs.extend(self.parse_inner_attributes()?);
 
-        let mut items = Vec::new();
+        let mut items = ThinVec::new();
         while !self.eat(&token::CloseDelim(Delimiter::Brace)) {
             if self.recover_doc_comment_before_brace() {
                 continue;
@@ -783,6 +741,7 @@ impl<'a> Parser<'a> {
     fn recover_doc_comment_before_brace(&mut self) -> bool {
         if let token::DocComment(..) = self.token.kind {
             if self.look_ahead(1, |tok| tok == &token::CloseDelim(Delimiter::Brace)) {
+                // FIXME: merge with `DocCommentDoesNotDocumentAnything` (E0585)
                 struct_span_err!(
                     self.diagnostic(),
                     self.token.span,
@@ -849,7 +808,7 @@ impl<'a> Parser<'a> {
             // It's a trait alias.
             if had_colon {
                 let span = span_at_colon.to(span_before_eq);
-                self.struct_span_err(span, "bounds are not allowed on trait aliases").emit();
+                self.sess.emit_err(errors::BoundsNotAllowedOnTraitAliases { span });
             }
 
             let bounds = self.parse_generic_bounds(None)?;
@@ -858,12 +817,10 @@ impl<'a> Parser<'a> {
 
             let whole_span = lo.to(self.prev_token.span);
             if is_auto == IsAuto::Yes {
-                let msg = "trait aliases cannot be `auto`";
-                self.struct_span_err(whole_span, msg).span_label(whole_span, msg).emit();
+                self.sess.emit_err(errors::TraitAliasCannotBeAuto { span: whole_span });
             }
             if let Unsafe::Yes(_) = unsafety {
-                let msg = "trait aliases cannot be `unsafe`";
-                self.struct_span_err(whole_span, msg).span_label(whole_span, msg).emit();
+                self.sess.emit_err(errors::TraitAliasCannotBeUnsafe { span: whole_span });
             }
 
             self.sess.gated_spans.gate(sym::trait_alias, whole_span);
@@ -909,8 +866,7 @@ impl<'a> Parser<'a> {
                     Ok(kind) => kind,
                     Err(kind) => match kind {
                         ItemKind::Static(a, _, b) => {
-                            self.struct_span_err(span, "associated `static` items are not allowed")
-                                .emit();
+                            self.sess.emit_err(errors::AssociatedStaticItemNotAllowed { span });
                             AssocItemKind::Const(Defaultness::Final, a, b)
                         }
                         _ => return self.error_bad_item_kind(span, &kind, "`trait`s or `impl`s"),
@@ -1041,7 +997,7 @@ impl<'a> Parser<'a> {
     /// ```text
     /// USE_TREE_LIST = Ø | (USE_TREE `,`)* USE_TREE [`,`]
     /// ```
-    fn parse_use_tree_list(&mut self) -> PResult<'a, Vec<(UseTree, ast::NodeId)>> {
+    fn parse_use_tree_list(&mut self) -> PResult<'a, ThinVec<(UseTree, ast::NodeId)>> {
         self.parse_delim_comma_seq(Delimiter::Brace, |p| {
             p.recover_diff_marker();
             Ok((p.parse_use_tree()?, DUMMY_NODE_ID))
@@ -1084,41 +1040,37 @@ impl<'a> Parser<'a> {
     }
 
     fn parse_crate_name_with_dashes(&mut self) -> PResult<'a, Ident> {
-        let error_msg = "crate name using dashes are not valid in `extern crate` statements";
-        let suggestion_msg = "if the original crate name uses dashes you need to use underscores \
-                              in the code";
-        let mut ident = if self.token.is_keyword(kw::SelfLower) {
+        let ident = if self.token.is_keyword(kw::SelfLower) {
             self.parse_path_segment_ident()
         } else {
             self.parse_ident()
         }?;
-        let mut idents = vec![];
-        let mut replacement = vec![];
-        let mut fixed_crate_name = false;
-        // Accept `extern crate name-like-this` for better diagnostics.
+
         let dash = token::BinOp(token::BinOpToken::Minus);
-        if self.token == dash {
-            // Do not include `-` as part of the expected tokens list.
-            while self.eat(&dash) {
-                fixed_crate_name = true;
-                replacement.push((self.prev_token.span, "_".to_string()));
-                idents.push(self.parse_ident()?);
-            }
+        if self.token != dash {
+            return Ok(ident);
         }
-        if fixed_crate_name {
-            let fixed_name_sp = ident.span.to(idents.last().unwrap().span);
-            let mut fixed_name = ident.name.to_string();
-            for part in idents {
-                fixed_name.push_str(&format!("_{}", part.name));
-            }
-            ident = Ident::from_str_and_span(&fixed_name, fixed_name_sp);
 
-            self.struct_span_err(fixed_name_sp, error_msg)
-                .span_label(fixed_name_sp, "dash-separated idents are not valid")
-                .multipart_suggestion(suggestion_msg, replacement, Applicability::MachineApplicable)
-                .emit();
+        // Accept `extern crate name-like-this` for better diagnostics.
+        let mut dashes = vec![];
+        let mut idents = vec![];
+        while self.eat(&dash) {
+            dashes.push(self.prev_token.span);
+            idents.push(self.parse_ident()?);
         }
-        Ok(ident)
+
+        let fixed_name_sp = ident.span.to(idents.last().unwrap().span);
+        let mut fixed_name = ident.name.to_string();
+        for part in idents {
+            write!(fixed_name, "_{}", part.name).unwrap();
+        }
+
+        self.sess.emit_err(errors::ExternCrateNameWithDashes {
+            span: fixed_name_sp,
+            sugg: errors::ExternCrateNameWithDashesSugg { dashes },
+        });
+
+        Ok(Ident::from_str_and_span(&fixed_name, fixed_name_sp))
     }
 
     /// Parses `extern` for foreign ABIs modules.
@@ -1166,7 +1118,10 @@ impl<'a> Parser<'a> {
                     Ok(kind) => kind,
                     Err(kind) => match kind {
                         ItemKind::Const(_, a, b) => {
-                            self.error_on_foreign_const(span, ident);
+                            self.sess.emit_err(errors::ExternItemCannotBeConst {
+                                ident_span: ident.span,
+                                const_span: span.with_hi(ident.span.lo()),
+                            });
                             ForeignItemKind::Static(a, Mutability::Not, b)
                         }
                         _ => return self.error_bad_item_kind(span, &kind, "`extern` blocks"),
@@ -1178,6 +1133,7 @@ impl<'a> Parser<'a> {
     }
 
     fn error_bad_item_kind<T>(&self, span: Span, kind: &ItemKind, ctx: &str) -> Option<T> {
+        // FIXME(#100717): needs variant for each `ItemKind` (instead of using `ItemKind::descr()`)
         let span = self.sess.source_map().guess_head_span(span);
         let descr = kind.descr();
         self.struct_span_err(span, &format!("{descr} is not supported in {ctx}"))
@@ -1186,18 +1142,6 @@ impl<'a> Parser<'a> {
         None
     }
 
-    fn error_on_foreign_const(&self, span: Span, ident: Ident) {
-        self.struct_span_err(ident.span, "extern items cannot be `const`")
-            .span_suggestion(
-                span.with_hi(ident.span.lo()),
-                "try using a static value",
-                "static ",
-                Applicability::MachineApplicable,
-            )
-            .note("for more information, visit https://doc.rust-lang.org/std/keyword.extern.html")
-            .emit();
-    }
-
     fn is_unsafe_foreign_mod(&self) -> bool {
         self.token.is_keyword(kw::Unsafe)
             && self.is_keyword_ahead(1, &[kw::Extern])
@@ -1225,25 +1169,10 @@ impl<'a> Parser<'a> {
     fn recover_const_mut(&mut self, const_span: Span) {
         if self.eat_keyword(kw::Mut) {
             let span = self.prev_token.span;
-            self.struct_span_err(span, "const globals cannot be mutable")
-                .span_label(span, "cannot be mutable")
-                .span_suggestion(
-                    const_span,
-                    "you might want to declare a static instead",
-                    "static",
-                    Applicability::MaybeIncorrect,
-                )
-                .emit();
+            self.sess.emit_err(errors::ConstGlobalCannotBeMutable { ident_span: span, const_span });
         } else if self.eat_keyword(kw::Let) {
             let span = self.prev_token.span;
-            self.struct_span_err(const_span.to(span), "`const` and `let` are mutually exclusive")
-                .span_suggestion(
-                    const_span.to(span),
-                    "remove `let`",
-                    "const",
-                    Applicability::MaybeIncorrect,
-                )
-                .emit();
+            self.sess.emit_err(errors::ConstLetMutuallyExclusive { span: const_span.to(span) });
         }
     }
 
@@ -1328,13 +1257,9 @@ impl<'a> Parser<'a> {
         };
 
         let span = self.prev_token.span.shrink_to_hi();
-        let mut err = self.struct_span_err(span, &format!("missing type for `{kind}` item"));
-        err.span_suggestion(
-            span,
-            "provide a type for the item",
-            format!("{colon} <type>"),
-            Applicability::HasPlaceholders,
-        );
+        let err: DiagnosticBuilder<'_, ErrorGuaranteed> =
+            errors::MissingConstType { span, colon, kind }
+                .into_diagnostic(&self.sess.span_diagnostic);
         err.stash(span, StashKey::ItemNoType);
 
         // The user intended that the type be inferred,
@@ -1346,18 +1271,12 @@ impl<'a> Parser<'a> {
     fn parse_item_enum(&mut self) -> PResult<'a, ItemInfo> {
         if self.token.is_keyword(kw::Struct) {
             let span = self.prev_token.span.to(self.token.span);
-            let mut err = self.struct_span_err(span, "`enum` and `struct` are mutually exclusive");
-            err.span_suggestion(
-                span,
-                "replace `enum struct` with",
-                "enum",
-                Applicability::MachineApplicable,
-            );
+            let err = errors::EnumStructMutuallyExclusive { span };
             if self.look_ahead(1, |t| t.is_ident()) {
                 self.bump();
-                err.emit();
+                self.sess.emit_err(err);
             } else {
-                return Err(err);
+                return Err(err.into_diagnostic(&self.sess.span_diagnostic));
             }
         }
 
@@ -1367,9 +1286,9 @@ impl<'a> Parser<'a> {
 
         // Possibly recover `enum Foo;` instead of `enum Foo {}`
         let (variants, _) = if self.token == TokenKind::Semi {
-            self.sess.emit_err(UseEmptyBlockNotSemi { span: self.token.span });
+            self.sess.emit_err(errors::UseEmptyBlockNotSemi { span: self.token.span });
             self.bump();
-            (vec![], false)
+            (thin_vec![], false)
         } else {
             self.parse_delim_comma_seq(Delimiter::Brace, |p| p.parse_enum_variant()).map_err(
                 |mut e| {
@@ -1412,7 +1331,7 @@ impl<'a> Parser<'a> {
                 };
 
                 let disr_expr =
-                    if this.eat(&token::Eq) { Some(this.parse_anon_const_expr()?) } else { None };
+                    if this.eat(&token::Eq) { Some(this.parse_expr_anon_const()?) } else { None };
 
                 let vr = ast::Variant {
                     ident,
@@ -1493,13 +1412,9 @@ impl<'a> Parser<'a> {
             self.expect_semi()?;
             body
         } else {
-            let token_str = super::token_descr(&self.token);
-            let msg = &format!(
-                "expected `where`, `{{`, `(`, or `;` after struct name, found {token_str}"
-            );
-            let mut err = self.struct_span_err(self.token.span, msg);
-            err.span_label(self.token.span, "expected `where`, `{`, `(`, or `;` after struct name");
-            return Err(err);
+            let err =
+                errors::UnexpectedTokenAfterStructName::new(self.token.span, self.token.clone());
+            return Err(err.into_diagnostic(&self.sess.span_diagnostic));
         };
 
         Ok((class_name, ItemKind::Struct(vdata, generics)))
@@ -1542,8 +1457,8 @@ impl<'a> Parser<'a> {
         adt_ty: &str,
         ident_span: Span,
         parsed_where: bool,
-    ) -> PResult<'a, (Vec<FieldDef>, /* recovered */ bool)> {
-        let mut fields = Vec::new();
+    ) -> PResult<'a, (ThinVec<FieldDef>, /* recovered */ bool)> {
+        let mut fields = ThinVec::new();
         let mut recovered = false;
         if self.eat(&token::OpenDelim(Delimiter::Brace)) {
             while self.token != token::CloseDelim(Delimiter::Brace) {
@@ -1583,7 +1498,7 @@ impl<'a> Parser<'a> {
         Ok((fields, recovered))
     }
 
-    pub(super) fn parse_tuple_struct_body(&mut self) -> PResult<'a, Vec<FieldDef>> {
+    pub(super) fn parse_tuple_struct_body(&mut self) -> PResult<'a, ThinVec<FieldDef>> {
         // This is the case where we find `struct Foo<T>(T) where T: Copy;`
         // Unit like structs are handled in parse_item_struct function
         self.parse_paren_comma_seq(|p| {
@@ -1676,7 +1591,7 @@ impl<'a> Parser<'a> {
             token::CloseDelim(Delimiter::Brace) => {}
             token::DocComment(..) => {
                 let previous_span = self.prev_token.span;
-                let mut err = DocCommentDoesNotDocumentAnything {
+                let mut err = errors::DocCommentDoesNotDocumentAnything {
                     span: self.token.span,
                     missing_comma: None,
                 };
@@ -1807,7 +1722,7 @@ impl<'a> Parser<'a> {
         }
         if self.token.kind == token::Eq {
             self.bump();
-            let const_expr = self.parse_anon_const_expr()?;
+            let const_expr = self.parse_expr_anon_const()?;
             let sp = ty.span.shrink_to_hi().to(const_expr.value.span);
             self.struct_span_err(sp, "default values on `struct` fields aren't supported")
                 .span_suggestion(
@@ -2186,7 +2101,7 @@ impl<'a> Parser<'a> {
                 // If we see `for Ty ...` then user probably meant `impl` item.
                 if self.token.is_keyword(kw::For) {
                     old_err.cancel();
-                    return Err(self.sess.create_err(FnTypoWithImpl { fn_span }));
+                    return Err(self.sess.create_err(errors::FnTypoWithImpl { fn_span }));
                 } else {
                     return Err(old_err);
                 }
@@ -2330,7 +2245,12 @@ impl<'a> Parser<'a> {
         let ext = self.parse_extern(case);
 
         if let Async::Yes { span, .. } = asyncness {
-            self.ban_async_in_2015(span);
+            if span.is_rust_2015() {
+                self.sess.emit_err(errors::AsyncFnIn2015 {
+                    span,
+                    help: errors::HelpUseLatestEdition::new(),
+                });
+            }
         }
 
         if !self.eat_keyword_case(kw::Fn, case) {
@@ -2440,17 +2360,6 @@ impl<'a> Parser<'a> {
         Ok(FnHeader { constness, unsafety, asyncness, ext })
     }
 
-    /// We are parsing `async fn`. If we are on Rust 2015, emit an error.
-    fn ban_async_in_2015(&self, span: Span) {
-        if span.rust_2015() {
-            let diag = self.diagnostic();
-            struct_span_err!(diag, span, E0670, "`async fn` is not permitted in Rust 2015")
-                .span_label(span, "to use `async fn`, switch to Rust 2018 or later")
-                .help_use_latest_edition()
-                .emit();
-        }
-    }
-
     /// Parses the parameter list and result type of a function declaration.
     pub(super) fn parse_fn_decl(
         &mut self,
@@ -2465,7 +2374,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses the parameter list of a function, including the `(` and `)` delimiters.
-    pub(super) fn parse_fn_params(&mut self, req_name: ReqName) -> PResult<'a, Vec<Param>> {
+    pub(super) fn parse_fn_params(&mut self, req_name: ReqName) -> PResult<'a, ThinVec<Param>> {
         let mut first_param = true;
         // Parse the arguments, starting out with `self` being allowed...
         let (mut params, _) = self.parse_paren_comma_seq(|p| {
@@ -2593,9 +2502,7 @@ impl<'a> Parser<'a> {
         };
         // Recover for the grammar `*self`, `*const self`, and `*mut self`.
         let recover_self_ptr = |this: &mut Self| {
-            let msg = "cannot pass `self` by raw pointer";
-            let span = this.token.span;
-            this.struct_span_err(span, msg).span_label(span, msg).emit();
+            self.sess.emit_err(errors::SelfArgumentPointer { span: this.token.span });
 
             Ok((SelfKind::Value(Mutability::Not), expect_self_ident(this), this.prev_token.span))
         };
@@ -2676,14 +2583,14 @@ impl<'a> Parser<'a> {
             && self.look_ahead(offset + 1, |t| t == &token::Colon)
     }
 
-    fn recover_first_param(&mut self) -> &'static str {
+    fn recover_self_param(&mut self) -> bool {
         match self
             .parse_outer_attributes()
             .and_then(|_| self.parse_self_param())
             .map_err(|e| e.cancel())
         {
-            Ok(Some(_)) => "method",
-            _ => "function",
+            Ok(Some(_)) => true,
+            _ => false,
         }
     }
 }
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
index ffb23b50a..da82e4724 100644
--- a/compiler/rustc_parse/src/parser/mod.rs
+++ b/compiler/rustc_parse/src/parser/mod.rs
@@ -10,7 +10,7 @@ mod path;
 mod stmt;
 mod ty;
 
-use crate::lexer::UnmatchedBrace;
+use crate::lexer::UnmatchedDelim;
 pub use attr_wrapper::AttrWrapper;
 pub use diagnostics::AttemptLocalParseRecovery;
 pub(crate) use item::FnParseMode;
@@ -19,9 +19,8 @@ pub use path::PathStyle;
 
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Delimiter, Nonterminal, Token, TokenKind};
-use rustc_ast::tokenstream::AttributesData;
-use rustc_ast::tokenstream::{self, DelimSpan, Spacing};
-use rustc_ast::tokenstream::{TokenStream, TokenTree};
+use rustc_ast::tokenstream::{AttributesData, DelimSpan, Spacing};
+use rustc_ast::tokenstream::{TokenStream, TokenTree, TokenTreeCursor};
 use rustc_ast::util::case::Case;
 use rustc_ast::AttrId;
 use rustc_ast::DUMMY_NODE_ID;
@@ -37,9 +36,10 @@ use rustc_errors::{
 use rustc_session::parse::ParseSess;
 use rustc_span::source_map::{Span, DUMMY_SP};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
-
 use std::ops::Range;
 use std::{cmp, mem, slice};
+use thin_vec::ThinVec;
+use tracing::debug;
 
 use crate::errors::{
     DocCommentDoesNotDocumentAnything, IncorrectVisibilityRestriction, MismatchedClosingDelimiter,
@@ -149,7 +149,7 @@ pub struct Parser<'a> {
     /// A list of all unclosed delimiters found by the lexer. If an entry is used for error recovery
     /// it gets removed from here. Every entry left at the end gets emitted as an independent
     /// error.
-    pub(super) unclosed_delims: Vec<UnmatchedBrace>,
+    pub(super) unclosed_delims: Vec<UnmatchedDelim>,
     last_unexpected_token_span: Option<Span>,
     /// Span pointing at the `:` for the last type ascription the parser has seen, and whether it
     /// looked like it could have been a mistyped path or literal `Option:Some(42)`).
@@ -168,7 +168,7 @@ pub struct Parser<'a> {
 // This type is used a lot, e.g. it's cloned when matching many declarative macro rules with nonterminals. Make sure
 // it doesn't unintentionally get bigger.
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-rustc_data_structures::static_assert_size!(Parser<'_>, 336);
+rustc_data_structures::static_assert_size!(Parser<'_>, 312);
 
 /// Stores span information about a closure.
 #[derive(Clone)]
@@ -221,18 +221,27 @@ impl<'a> Drop for Parser<'a> {
     }
 }
 
+/// Iterator over a `TokenStream` that produces `Token`s. It's a bit odd that
+/// we (a) lex tokens into a nice tree structure (`TokenStream`), and then (b)
+/// use this type to emit them as a linear sequence. But a linear sequence is
+/// what the parser expects, for the most part.
 #[derive(Clone)]
 struct TokenCursor {
-    // The current (innermost) frame. `frame` and `stack` could be combined,
-    // but it's faster to have them separately to access `frame` directly
-    // rather than via something like `stack.last().unwrap()` or
-    // `stack[stack.len() - 1]`.
-    frame: TokenCursorFrame,
-    // Additional frames that enclose `frame`.
-    stack: Vec<TokenCursorFrame>,
+    // Cursor for the current (innermost) token stream. The delimiters for this
+    // token stream are found in `self.stack.last()`; when that is `None` then
+    // we are in the outermost token stream which never has delimiters.
+    tree_cursor: TokenTreeCursor,
+
+    // Token streams surrounding the current one. The delimiters for stack[n]'s
+    // tokens are in `stack[n-1]`. `stack[0]` (when present) has no delimiters
+    // because it's the outermost token stream which never has delimiters.
+    stack: Vec<(TokenTreeCursor, Delimiter, DelimSpan)>,
+
     desugar_doc_comments: bool,
+
     // Counts the number of calls to `{,inlined_}next`.
     num_next_calls: usize,
+
     // During parsing, we may sometimes need to 'unglue' a
     // glued token into two component tokens
     // (e.g. '>>' into '>' and '>), so that the parser
@@ -257,18 +266,6 @@ struct TokenCursor {
     break_last_token: bool,
 }
 
-#[derive(Clone)]
-struct TokenCursorFrame {
-    delim_sp: Option<(Delimiter, DelimSpan)>,
-    tree_cursor: tokenstream::Cursor,
-}
-
-impl TokenCursorFrame {
-    fn new(delim_sp: Option<(Delimiter, DelimSpan)>, tts: TokenStream) -> Self {
-        TokenCursorFrame { delim_sp, tree_cursor: tts.into_trees() }
-    }
-}
-
 impl TokenCursor {
     fn next(&mut self, desugar_doc_comments: bool) -> (Token, Spacing) {
         self.inlined_next(desugar_doc_comments)
@@ -281,38 +278,47 @@ impl TokenCursor {
             // FIXME: we currently don't return `Delimiter` open/close delims. To fix #67062 we will
             // need to, whereupon the `delim != Delimiter::Invisible` conditions below can be
             // removed.
-            if let Some(tree) = self.frame.tree_cursor.next_ref() {
+            if let Some(tree) = self.tree_cursor.next_ref() {
                 match tree {
                     &TokenTree::Token(ref token, spacing) => match (desugar_doc_comments, token) {
                         (true, &Token { kind: token::DocComment(_, attr_style, data), span }) => {
-                            return self.desugar(attr_style, data, span);
+                            let desugared = self.desugar(attr_style, data, span);
+                            self.tree_cursor.replace_prev_and_rewind(desugared);
+                            // Continue to get the first token of the desugared doc comment.
+                        }
+                        _ => {
+                            debug_assert!(!matches!(
+                                token.kind,
+                                token::OpenDelim(_) | token::CloseDelim(_)
+                            ));
+                            return (token.clone(), spacing);
                         }
-                        _ => return (token.clone(), spacing),
                     },
                     &TokenTree::Delimited(sp, delim, ref tts) => {
-                        // Set `open_delim` to true here because we deal with it immediately.
-                        let frame = TokenCursorFrame::new(Some((delim, sp)), tts.clone());
-                        self.stack.push(mem::replace(&mut self.frame, frame));
+                        let trees = tts.clone().into_trees();
+                        self.stack.push((mem::replace(&mut self.tree_cursor, trees), delim, sp));
                         if delim != Delimiter::Invisible {
                             return (Token::new(token::OpenDelim(delim), sp.open), Spacing::Alone);
                         }
                         // No open delimiter to return; continue on to the next iteration.
                     }
                 };
-            } else if let Some(frame) = self.stack.pop() {
-                if let Some((delim, span)) = self.frame.delim_sp && delim != Delimiter::Invisible {
-                    self.frame = frame;
+            } else if let Some((tree_cursor, delim, span)) = self.stack.pop() {
+                // We have exhausted this token stream. Move back to its parent token stream.
+                self.tree_cursor = tree_cursor;
+                if delim != Delimiter::Invisible {
                     return (Token::new(token::CloseDelim(delim), span.close), Spacing::Alone);
                 }
-                self.frame = frame;
                 // No close delimiter to return; continue on to the next iteration.
             } else {
+                // We have exhausted the outermost token stream.
                 return (Token::new(token::Eof, DUMMY_SP), Spacing::Alone);
             }
         }
     }
 
-    fn desugar(&mut self, attr_style: AttrStyle, data: Symbol, span: Span) -> (Token, Spacing) {
+    // Desugar a doc comment into something like `#[doc = r"foo"]`.
+    fn desugar(&mut self, attr_style: AttrStyle, data: Symbol, span: Span) -> Vec<TokenTree> {
         // Searches for the occurrences of `"#*` and returns the minimum number of `#`s
         // required to wrap the text. E.g.
         // - `abc d` is wrapped as `r"abc d"` (num_of_hashes = 0)
@@ -329,7 +335,7 @@ impl TokenCursor {
             num_of_hashes = cmp::max(num_of_hashes, count);
         }
 
-        // `/// foo` becomes `doc = r"foo".
+        // `/// foo` becomes `doc = r"foo"`.
         let delim_span = DelimSpan::from_single(span);
         let body = TokenTree::Delimited(
             delim_span,
@@ -346,27 +352,15 @@ impl TokenCursor {
             .collect::<TokenStream>(),
         );
 
-        self.stack.push(mem::replace(
-            &mut self.frame,
-            TokenCursorFrame::new(
-                None,
-                if attr_style == AttrStyle::Inner {
-                    [
-                        TokenTree::token_alone(token::Pound, span),
-                        TokenTree::token_alone(token::Not, span),
-                        body,
-                    ]
-                    .into_iter()
-                    .collect::<TokenStream>()
-                } else {
-                    [TokenTree::token_alone(token::Pound, span), body]
-                        .into_iter()
-                        .collect::<TokenStream>()
-                },
-            ),
-        ));
-
-        self.next(/* desugar_doc_comments */ false)
+        if attr_style == AttrStyle::Inner {
+            vec![
+                TokenTree::token_alone(token::Pound, span),
+                TokenTree::token_alone(token::Not, span),
+                body,
+            ]
+        } else {
+            vec![TokenTree::token_alone(token::Pound, span), body]
+        }
     }
 }
 
@@ -475,7 +469,7 @@ impl<'a> Parser<'a> {
             restrictions: Restrictions::empty(),
             expected_tokens: Vec::new(),
             token_cursor: TokenCursor {
-                frame: TokenCursorFrame::new(None, tokens),
+                tree_cursor: tokens.into_trees(),
                 stack: Vec::new(),
                 num_next_calls: 0,
                 desugar_doc_comments,
@@ -739,9 +733,10 @@ impl<'a> Parser<'a> {
     fn check_const_closure(&self) -> bool {
         self.is_keyword_ahead(0, &[kw::Const])
             && self.look_ahead(1, |t| match &t.kind {
-                token::Ident(kw::Move | kw::Static | kw::Async, _)
-                | token::OrOr
-                | token::BinOp(token::Or) => true,
+                // async closures do not work with const closures, so we do not parse that here.
+                token::Ident(kw::Move | kw::Static, _) | token::OrOr | token::BinOp(token::Or) => {
+                    true
+                }
                 _ => false,
             })
     }
@@ -859,11 +854,11 @@ impl<'a> Parser<'a> {
         sep: SeqSep,
         expect: TokenExpectType,
         mut f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
-    ) -> PResult<'a, (Vec<T>, bool /* trailing */, bool /* recovered */)> {
+    ) -> PResult<'a, (ThinVec<T>, bool /* trailing */, bool /* recovered */)> {
         let mut first = true;
         let mut recovered = false;
         let mut trailing = false;
-        let mut v = vec![];
+        let mut v = ThinVec::new();
         let unclosed_delims = !self.unclosed_delims.is_empty();
 
         while !self.expect_any_with_type(kets, expect) {
@@ -987,7 +982,11 @@ impl<'a> Parser<'a> {
         let initial_semicolon = self.token.span;
 
         while self.eat(&TokenKind::Semi) {
-            let _ = self.parse_stmt(ForceCollect::Yes)?;
+            let _ =
+                self.parse_stmt_without_recovery(false, ForceCollect::Yes).unwrap_or_else(|e| {
+                    e.cancel();
+                    None
+                });
         }
 
         expect_err.set_primary_message(
@@ -1043,7 +1042,7 @@ impl<'a> Parser<'a> {
         ket: &TokenKind,
         sep: SeqSep,
         f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
-    ) -> PResult<'a, (Vec<T>, bool, bool)> {
+    ) -> PResult<'a, (ThinVec<T>, bool, bool)> {
         self.parse_seq_to_before_tokens(&[ket], sep, TokenExpectType::Expect, f)
     }
 
@@ -1055,7 +1054,7 @@ impl<'a> Parser<'a> {
         ket: &TokenKind,
         sep: SeqSep,
         f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
-    ) -> PResult<'a, (Vec<T>, bool /* trailing */)> {
+    ) -> PResult<'a, (ThinVec<T>, bool /* trailing */)> {
         let (val, trailing, recovered) = self.parse_seq_to_before_end(ket, sep, f)?;
         if !recovered {
             self.eat(ket);
@@ -1072,7 +1071,7 @@ impl<'a> Parser<'a> {
         ket: &TokenKind,
         sep: SeqSep,
         f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
-    ) -> PResult<'a, (Vec<T>, bool)> {
+    ) -> PResult<'a, (ThinVec<T>, bool)> {
         self.expect(bra)?;
         self.parse_seq_to_end(ket, sep, f)
     }
@@ -1081,7 +1080,7 @@ impl<'a> Parser<'a> {
         &mut self,
         delim: Delimiter,
         f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
-    ) -> PResult<'a, (Vec<T>, bool)> {
+    ) -> PResult<'a, (ThinVec<T>, bool)> {
         self.parse_unspanned_seq(
             &token::OpenDelim(delim),
             &token::CloseDelim(delim),
@@ -1093,7 +1092,7 @@ impl<'a> Parser<'a> {
     fn parse_paren_comma_seq<T>(
         &mut self,
         f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
-    ) -> PResult<'a, (Vec<T>, bool)> {
+    ) -> PResult<'a, (ThinVec<T>, bool)> {
         self.parse_delim_comma_seq(Delimiter::Parenthesis, f)
     }
 
@@ -1142,14 +1141,16 @@ impl<'a> Parser<'a> {
             return looker(&self.token);
         }
 
-        let frame = &self.token_cursor.frame;
-        if let Some((delim, span)) = frame.delim_sp && delim != Delimiter::Invisible {
+        let tree_cursor = &self.token_cursor.tree_cursor;
+        if let Some(&(_, delim, span)) = self.token_cursor.stack.last()
+            && delim != Delimiter::Invisible
+        {
             let all_normal = (0..dist).all(|i| {
-                let token = frame.tree_cursor.look_ahead(i);
+                let token = tree_cursor.look_ahead(i);
                 !matches!(token, Some(TokenTree::Delimited(_, Delimiter::Invisible, _)))
             });
             if all_normal {
-                return match frame.tree_cursor.look_ahead(dist - 1) {
+                return match tree_cursor.look_ahead(dist - 1) {
                     Some(tree) => match tree {
                         TokenTree::Token(token, _) => looker(token),
                         TokenTree::Delimited(dspan, delim, _) => {
@@ -1203,8 +1204,18 @@ impl<'a> Parser<'a> {
 
     /// Parses constness: `const` or nothing.
     fn parse_constness(&mut self, case: Case) -> Const {
-        // Avoid const blocks to be parsed as const items
-        if self.look_ahead(1, |t| t != &token::OpenDelim(Delimiter::Brace))
+        self.parse_constness_(case, false)
+    }
+
+    /// Parses constness for closures
+    fn parse_closure_constness(&mut self, case: Case) -> Const {
+        self.parse_constness_(case, true)
+    }
+
+    fn parse_constness_(&mut self, case: Case, is_closure: bool) -> Const {
+        // Avoid const blocks and const closures to be parsed as const items
+        if (self.check_const_closure() == is_closure)
+            && self.look_ahead(1, |t| t != &token::OpenDelim(Delimiter::Brace))
             && self.eat_keyword_case(kw::Const, case)
         {
             Const::Yes(self.prev_token.uninterpolated_span())
@@ -1277,22 +1288,16 @@ impl<'a> Parser<'a> {
     }
 
     fn parse_delim_args_inner(&mut self) -> Option<DelimArgs> {
-        if self.check(&token::OpenDelim(Delimiter::Parenthesis))
+        let delimited = self.check(&token::OpenDelim(Delimiter::Parenthesis))
             || self.check(&token::OpenDelim(Delimiter::Bracket))
-            || self.check(&token::OpenDelim(Delimiter::Brace))
-        {
-            match self.parse_token_tree() {
-                // We've confirmed above that there is a delimiter so unwrapping is OK.
-                TokenTree::Delimited(dspan, delim, tokens) => Some(DelimArgs {
-                    dspan,
-                    delim: MacDelimiter::from_token(delim).unwrap(),
-                    tokens,
-                }),
-                _ => unreachable!(),
-            }
-        } else {
-            None
-        }
+            || self.check(&token::OpenDelim(Delimiter::Brace));
+
+        delimited.then(|| {
+            // We've confirmed above that there is a delimiter so unwrapping is OK.
+            let TokenTree::Delimited(dspan, delim, tokens) = self.parse_token_tree() else { unreachable!() };
+
+            DelimArgs { dspan, delim: MacDelimiter::from_token(delim).unwrap(), tokens }
+        })
     }
 
     fn parse_or_use_outer_attributes(
@@ -1310,10 +1315,10 @@ impl<'a> Parser<'a> {
     pub(crate) fn parse_token_tree(&mut self) -> TokenTree {
         match self.token.kind {
             token::OpenDelim(..) => {
-                // Grab the tokens from this frame.
-                let frame = &self.token_cursor.frame;
-                let stream = frame.tree_cursor.stream.clone();
-                let (delim, span) = frame.delim_sp.unwrap();
+                // Grab the tokens within the delimiters.
+                let tree_cursor = &self.token_cursor.tree_cursor;
+                let stream = tree_cursor.stream.clone();
+                let (_, delim, span) = *self.token_cursor.stack.last().unwrap();
 
                 // Advance the token cursor through the entire delimited
                 // sequence. After getting the `OpenDelim` we are *within* the
@@ -1516,11 +1521,11 @@ impl<'a> Parser<'a> {
 }
 
 pub(crate) fn make_unclosed_delims_error(
-    unmatched: UnmatchedBrace,
+    unmatched: UnmatchedDelim,
     sess: &ParseSess,
 ) -> Option<DiagnosticBuilder<'_, ErrorGuaranteed>> {
     // `None` here means an `Eof` was found. We already emit those errors elsewhere, we add them to
-    // `unmatched_braces` only for error recovery in the `Parser`.
+    // `unmatched_delims` only for error recovery in the `Parser`.
     let found_delim = unmatched.found_delim?;
     let mut spans = vec![unmatched.found_span];
     if let Some(sp) = unmatched.unclosed_span {
@@ -1537,7 +1542,7 @@ pub(crate) fn make_unclosed_delims_error(
     Some(err)
 }
 
-pub fn emit_unclosed_delims(unclosed_delims: &mut Vec<UnmatchedBrace>, sess: &ParseSess) {
+pub fn emit_unclosed_delims(unclosed_delims: &mut Vec<UnmatchedDelim>, sess: &ParseSess) {
     *sess.reached_eof.borrow_mut() |=
         unclosed_delims.iter().any(|unmatched_delim| unmatched_delim.found_delim.is_none());
     for unmatched in unclosed_delims.drain(..) {
diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs
index 239ed79ce..7a4d53ed8 100644
--- a/compiler/rustc_parse/src/parser/nonterminal.rs
+++ b/compiler/rustc_parse/src/parser/nonterminal.rs
@@ -2,9 +2,11 @@ use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Delimiter, NonterminalKind, Token};
 use rustc_ast::HasTokens;
 use rustc_ast_pretty::pprust;
+use rustc_errors::IntoDiagnostic;
 use rustc_errors::PResult;
 use rustc_span::symbol::{kw, Ident};
 
+use crate::errors::UnexpectedNonterminal;
 use crate::parser::pat::{CommaRecoveryMode, RecoverColon, RecoverComma};
 use crate::parser::{FollowedByType, ForceCollect, NtOrTt, Parser, PathStyle};
 
@@ -113,7 +115,8 @@ impl<'a> Parser<'a> {
             NonterminalKind::Item => match self.parse_item(ForceCollect::Yes)? {
                 Some(item) => token::NtItem(item),
                 None => {
-                    return Err(self.struct_span_err(self.token.span, "expected an item keyword"));
+                    return Err(UnexpectedNonterminal::Item(self.token.span)
+                               .into_diagnostic(&self.sess.span_diagnostic));
                 }
             },
             NonterminalKind::Block => {
@@ -124,7 +127,8 @@ impl<'a> Parser<'a> {
             NonterminalKind::Stmt => match self.parse_stmt(ForceCollect::Yes)? {
                 Some(s) => token::NtStmt(P(s)),
                 None => {
-                    return Err(self.struct_span_err(self.token.span, "expected a statement"));
+                    return Err(UnexpectedNonterminal::Statement(self.token.span)
+                               .into_diagnostic(&self.sess.span_diagnostic));
                 }
             },
             NonterminalKind::PatParam { .. } | NonterminalKind::PatWithOr { .. } => {
@@ -160,9 +164,10 @@ impl<'a> Parser<'a> {
                 token::NtIdent(ident, is_raw)
             }
             NonterminalKind::Ident => {
-                let token_str = pprust::token_to_string(&self.token);
-                let msg = &format!("expected ident, found {}", &token_str);
-                return Err(self.struct_span_err(self.token.span, msg));
+                return Err(UnexpectedNonterminal::Ident {
+                    span: self.token.span,
+                    token: self.token.clone(),
+                }.into_diagnostic(&self.sess.span_diagnostic));
             }
             NonterminalKind::Path => token::NtPath(
                 P(self.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))?),
@@ -175,9 +180,10 @@ impl<'a> Parser<'a> {
                 if self.check_lifetime() {
                     token::NtLifetime(self.expect_lifetime().ident)
                 } else {
-                    let token_str = pprust::token_to_string(&self.token);
-                    let msg = &format!("expected a lifetime, found `{}`", &token_str);
-                    return Err(self.struct_span_err(self.token.span, msg));
+                    return Err(UnexpectedNonterminal::Lifetime {
+                        span: self.token.span,
+                        token: self.token.clone(),
+                    }.into_diagnostic(&self.sess.span_diagnostic));
                 }
             }
         };
diff --git a/compiler/rustc_parse/src/parser/pat.rs b/compiler/rustc_parse/src/parser/pat.rs
index e73a17ced..8e920f1c4 100644
--- a/compiler/rustc_parse/src/parser/pat.rs
+++ b/compiler/rustc_parse/src/parser/pat.rs
@@ -1,5 +1,14 @@
 use super::{ForceCollect, Parser, PathStyle, TrailingToken};
-use crate::errors::RemoveLet;
+use crate::errors::{
+    AmbiguousRangePattern, DotDotDotForRemainingFields, DotDotDotRangeToPatternNotAllowed,
+    DotDotDotRestPattern, EnumPatternInsteadOfIdentifier, ExpectedBindingLeftOfAt,
+    ExpectedCommaAfterPatternField, InclusiveRangeExtraEquals, InclusiveRangeMatchArrow,
+    InclusiveRangeNoEnd, InvalidMutInPattern, PatternOnWrongSideOfAt, RefMutOrderIncorrect,
+    RemoveLet, RepeatedMutInPattern, TopLevelOrPatternNotAllowed, TopLevelOrPatternNotAllowedSugg,
+    TrailingVertNotAllowed, UnexpectedLifetimeInPattern, UnexpectedVertVertBeforeFunctionParam,
+    UnexpectedVertVertInPattern,
+};
+use crate::fluent_generated as fluent;
 use crate::{maybe_recover_from_interpolated_ty_qpath, maybe_whole};
 use rustc_ast::mut_visit::{noop_visit_pat, MutVisitor};
 use rustc_ast::ptr::P;
@@ -9,15 +18,32 @@ use rustc_ast::{
     PatField, PatKind, Path, QSelf, RangeEnd, RangeSyntax,
 };
 use rustc_ast_pretty::pprust;
-use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder, ErrorGuaranteed, PResult};
+use rustc_errors::{Applicability, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic, PResult};
 use rustc_session::errors::ExprParenthesesNeeded;
 use rustc_span::source_map::{respan, Span, Spanned};
 use rustc_span::symbol::{kw, sym, Ident};
+use thin_vec::{thin_vec, ThinVec};
 
-pub(super) type Expected = Option<&'static str>;
+#[derive(PartialEq, Copy, Clone)]
+pub enum Expected {
+    ParameterName,
+    ArgumentName,
+    Identifier,
+    BindingPattern,
+}
 
-/// `Expected` for function and lambda parameter patterns.
-pub(super) const PARAM_EXPECTED: Expected = Some("parameter name");
+impl Expected {
+    // FIXME(#100717): migrate users of this to proper localization
+    fn to_string_or_fallback(expected: Option<Expected>) -> &'static str {
+        match expected {
+            Some(Expected::ParameterName) => "parameter name",
+            Some(Expected::ArgumentName) => "argument name",
+            Some(Expected::Identifier) => "identifier",
+            Some(Expected::BindingPattern) => "binding pattern",
+            None => "pattern",
+        }
+    }
+}
 
 const WHILE_PARSING_OR_MSG: &str = "while parsing this or-pattern starting here";
 
@@ -54,13 +80,19 @@ enum EatOrResult {
     None,
 }
 
+/// The syntax location of a given pattern. Used for diagnostics.
+pub(super) enum PatternLocation {
+    LetBinding,
+    FunctionParameter,
+}
+
 impl<'a> Parser<'a> {
     /// Parses a pattern.
     ///
     /// Corresponds to `pat<no_top_alt>` in RFC 2535 and does not admit or-patterns
     /// at the top level. Used when parsing the parameters of lambda expressions,
     /// functions, function pointers, and `pat` macro fragments.
-    pub fn parse_pat_no_top_alt(&mut self, expected: Expected) -> PResult<'a, P<Pat>> {
+    pub fn parse_pat_no_top_alt(&mut self, expected: Option<Expected>) -> PResult<'a, P<Pat>> {
         self.parse_pat_with_range_pat(true, expected)
     }
 
@@ -74,7 +106,7 @@ impl<'a> Parser<'a> {
     /// simplify the grammar somewhat.
     pub fn parse_pat_allow_top_alt(
         &mut self,
-        expected: Expected,
+        expected: Option<Expected>,
         rc: RecoverComma,
         ra: RecoverColon,
         rt: CommaRecoveryMode,
@@ -86,7 +118,7 @@ impl<'a> Parser<'a> {
     /// recovered).
     fn parse_pat_allow_top_alt_inner(
         &mut self,
-        expected: Expected,
+        expected: Option<Expected>,
         rc: RecoverComma,
         ra: RecoverColon,
         rt: CommaRecoveryMode,
@@ -123,7 +155,7 @@ impl<'a> Parser<'a> {
                 // If there was a leading vert, treat this as an or-pattern. This improves
                 // diagnostics.
                 let span = leading_vert_span.to(self.prev_token.span);
-                return Ok((self.mk_pat(span, PatKind::Or(vec![first_pat])), trailing_vert));
+                return Ok((self.mk_pat(span, PatKind::Or(thin_vec![first_pat])), trailing_vert));
             }
 
             return Ok((first_pat, trailing_vert));
@@ -131,7 +163,7 @@ impl<'a> Parser<'a> {
 
         // Parse the patterns `p_1 | ... | p_n` where `n > 0`.
         let lo = leading_vert_span.unwrap_or(first_pat.span);
-        let mut pats = vec![first_pat];
+        let mut pats = thin_vec![first_pat];
         loop {
             match self.eat_or_separator(Some(lo)) {
                 EatOrResult::AteOr => {}
@@ -165,9 +197,9 @@ impl<'a> Parser<'a> {
     /// otherwise).
     pub(super) fn parse_pat_before_ty(
         &mut self,
-        expected: Expected,
+        expected: Option<Expected>,
         rc: RecoverComma,
-        syntax_loc: &str,
+        syntax_loc: PatternLocation,
     ) -> PResult<'a, (P<Pat>, bool)> {
         // We use `parse_pat_allow_top_alt` regardless of whether we actually want top-level
         // or-patterns so that we can detect when a user tries to use it. This allows us to print a
@@ -181,27 +213,41 @@ impl<'a> Parser<'a> {
         let colon = self.eat(&token::Colon);
 
         if let PatKind::Or(pats) = &pat.kind {
-            let msg = format!("top-level or-patterns are not allowed in {}", syntax_loc);
-            let (help, fix) = if pats.len() == 1 {
-                // If all we have is a leading vert, then print a special message. This is the case
-                // if `parse_pat_allow_top_alt` returns an or-pattern with one variant.
-                let msg = "remove the `|`";
-                let fix = pprust::pat_to_string(&pat);
-                (msg, fix)
-            } else {
-                let msg = "wrap the pattern in parentheses";
-                let fix = format!("({})", pprust::pat_to_string(&pat));
-                (msg, fix)
-            };
+            let span = pat.span;
 
             if trailing_vert {
                 // We already emitted an error and suggestion to remove the trailing vert. Don't
                 // emit again.
-                self.sess.span_diagnostic.delay_span_bug(pat.span, &msg);
+
+                // FIXME(#100717): pass `TopLevelOrPatternNotAllowed::* { sub: None }` to
+                // `delay_span_bug()` instead of fluent message
+                self.sess.span_diagnostic.delay_span_bug(
+                    span,
+                    match syntax_loc {
+                        PatternLocation::LetBinding => {
+                            fluent::parse_or_pattern_not_allowed_in_let_binding
+                        }
+                        PatternLocation::FunctionParameter => {
+                            fluent::parse_or_pattern_not_allowed_in_fn_parameters
+                        }
+                    },
+                );
             } else {
-                self.struct_span_err(pat.span, &msg)
-                    .span_suggestion(pat.span, help, fix, Applicability::MachineApplicable)
-                    .emit();
+                let pat = pprust::pat_to_string(&pat);
+                let sub = if pats.len() == 1 {
+                    Some(TopLevelOrPatternNotAllowedSugg::RemoveLeadingVert { span, pat })
+                } else {
+                    Some(TopLevelOrPatternNotAllowedSugg::WrapInParens { span, pat })
+                };
+
+                self.sess.emit_err(match syntax_loc {
+                    PatternLocation::LetBinding => {
+                        TopLevelOrPatternNotAllowed::LetBinding { span, sub }
+                    }
+                    PatternLocation::FunctionParameter => {
+                        TopLevelOrPatternNotAllowed::FunctionParameter { span, sub }
+                    }
+                });
             }
         }
 
@@ -218,15 +264,15 @@ impl<'a> Parser<'a> {
         // a leading `||` probably doesn't indicate an or-pattern attempt, so we handle that
         // separately.
         if let token::OrOr = self.token.kind {
-            let span = self.token.span;
-            let mut err = self.struct_span_err(span, "unexpected `||` before function parameter");
-            err.span_suggestion(span, "remove the `||`", "", Applicability::MachineApplicable);
-            err.note("alternatives in or-patterns are separated with `|`, not `||`");
-            err.emit();
+            self.sess.emit_err(UnexpectedVertVertBeforeFunctionParam { span: self.token.span });
             self.bump();
         }
 
-        self.parse_pat_before_ty(PARAM_EXPECTED, RecoverComma::No, "function parameters")
+        self.parse_pat_before_ty(
+            Some(Expected::ParameterName),
+            RecoverComma::No,
+            PatternLocation::FunctionParameter,
+        )
     }
 
     /// Eat the or-pattern `|` separator.
@@ -236,7 +282,7 @@ impl<'a> Parser<'a> {
             EatOrResult::TrailingVert
         } else if matches!(self.token.kind, token::OrOr) {
             // Found `||`; Recover and pretend we parsed `|`.
-            self.ban_unexpected_or_or(lo);
+            self.sess.emit_err(UnexpectedVertVertInPattern { span: self.token.span, start: lo });
             self.bump();
             EatOrResult::AteOr
         } else if self.eat(&token::BinOp(token::Or)) {
@@ -270,7 +316,13 @@ impl<'a> Parser<'a> {
         });
         match (is_end_ahead, &self.token.kind) {
             (true, token::BinOp(token::Or) | token::OrOr) => {
-                self.ban_illegal_vert(lo, "trailing", "not allowed in an or-pattern");
+                // A `|` or possibly `||` token shouldn't be here. Ban it.
+                self.sess.emit_err(TrailingVertNotAllowed {
+                    span: self.token.span,
+                    start: lo,
+                    token: self.token.clone(),
+                    note_double_vert: matches!(self.token.kind, token::OrOr).then_some(()),
+                });
                 self.bump();
                 true
             }
@@ -278,46 +330,12 @@ impl<'a> Parser<'a> {
         }
     }
 
-    /// We have parsed `||` instead of `|`. Error and suggest `|` instead.
-    fn ban_unexpected_or_or(&mut self, lo: Option<Span>) {
-        let mut err = self.struct_span_err(self.token.span, "unexpected token `||` in pattern");
-        err.span_suggestion(
-            self.token.span,
-            "use a single `|` to separate multiple alternative patterns",
-            "|",
-            Applicability::MachineApplicable,
-        );
-        if let Some(lo) = lo {
-            err.span_label(lo, WHILE_PARSING_OR_MSG);
-        }
-        err.emit();
-    }
-
-    /// A `|` or possibly `||` token shouldn't be here. Ban it.
-    fn ban_illegal_vert(&mut self, lo: Option<Span>, pos: &str, ctx: &str) {
-        let span = self.token.span;
-        let mut err = self.struct_span_err(span, &format!("a {} `|` is {}", pos, ctx));
-        err.span_suggestion(
-            span,
-            &format!("remove the `{}`", pprust::token_to_string(&self.token)),
-            "",
-            Applicability::MachineApplicable,
-        );
-        if let Some(lo) = lo {
-            err.span_label(lo, WHILE_PARSING_OR_MSG);
-        }
-        if let token::OrOr = self.token.kind {
-            err.note("alternatives in or-patterns are separated with `|`, not `||`");
-        }
-        err.emit();
-    }
-
     /// Parses a pattern, with a setting whether modern range patterns (e.g., `a..=b`, `a..b` are
     /// allowed).
     fn parse_pat_with_range_pat(
         &mut self,
         allow_range_pat: bool,
-        expected: Expected,
+        expected: Option<Expected>,
     ) -> PResult<'a, P<Pat>> {
         maybe_recover_from_interpolated_ty_qpath!(self, true);
         maybe_whole!(self, NtPat, |x| x);
@@ -413,7 +431,7 @@ impl<'a> Parser<'a> {
             let lt = self.expect_lifetime();
             let (lit, _) =
                 self.recover_unclosed_char(lt.ident, Parser::mk_token_lit_char, |self_| {
-                    let expected = expected.unwrap_or("pattern");
+                    let expected = Expected::to_string_or_fallback(expected);
                     let msg = format!(
                         "expected {}, found {}",
                         expected,
@@ -454,15 +472,7 @@ impl<'a> Parser<'a> {
         self.bump(); // `...`
 
         // The user probably mistook `...` for a rest pattern `..`.
-        self.struct_span_err(lo, "unexpected `...`")
-            .span_label(lo, "not a valid pattern")
-            .span_suggestion_short(
-                lo,
-                "for a rest pattern, use `..` instead of `...`",
-                "..",
-                Applicability::MachineApplicable,
-            )
-            .emit();
+        self.sess.emit_err(DotDotDotRestPattern { span: lo });
         PatKind::Rest
     }
 
@@ -487,7 +497,7 @@ impl<'a> Parser<'a> {
         // At this point we attempt to parse `@ $pat_rhs` and emit an error.
         self.bump(); // `@`
         let mut rhs = self.parse_pat_no_top_alt(None)?;
-        let sp = lhs.span.to(rhs.span);
+        let whole_span = lhs.span.to(rhs.span);
 
         if let PatKind::Ident(_, _, sub @ None) = &mut rhs.kind {
             // The user inverted the order, so help them fix that.
@@ -496,27 +506,23 @@ impl<'a> Parser<'a> {
             // The RHS is now the full pattern.
             *sub = Some(lhs);
 
-            self.struct_span_err(sp, "pattern on wrong side of `@`")
-                .span_label(lhs_span, "pattern on the left, should be on the right")
-                .span_label(rhs.span, "binding on the right, should be on the left")
-                .span_suggestion(
-                    sp,
-                    "switch the order",
-                    pprust::pat_to_string(&rhs),
-                    Applicability::MachineApplicable,
-                )
-                .emit();
+            self.sess.emit_err(PatternOnWrongSideOfAt {
+                whole_span,
+                whole_pat: pprust::pat_to_string(&rhs),
+                pattern: lhs_span,
+                binding: rhs.span,
+            });
         } else {
             // The special case above doesn't apply so we may have e.g. `A(x) @ B(y)`.
             rhs.kind = PatKind::Wild;
-            self.struct_span_err(sp, "left-hand side of `@` must be a binding")
-                .span_label(lhs.span, "interpreted as a pattern, not a binding")
-                .span_label(rhs.span, "also a pattern")
-                .note("bindings are `x`, `mut x`, `ref x`, and `ref mut x`")
-                .emit();
+            self.sess.emit_err(ExpectedBindingLeftOfAt {
+                whole_span,
+                lhs: lhs.span,
+                rhs: rhs.span,
+            });
         }
 
-        rhs.span = sp;
+        rhs.span = whole_span;
         Ok(rhs)
     }
 
@@ -531,35 +537,23 @@ impl<'a> Parser<'a> {
             _ => return,
         }
 
-        self.struct_span_err(pat.span, "the range pattern here has ambiguous interpretation")
-            .span_suggestion(
-                pat.span,
-                "add parentheses to clarify the precedence",
-                format!("({})", pprust::pat_to_string(&pat)),
-                // "ambiguous interpretation" implies that we have to be guessing
-                Applicability::MaybeIncorrect,
-            )
-            .emit();
+        self.sess
+            .emit_err(AmbiguousRangePattern { span: pat.span, pat: pprust::pat_to_string(&pat) });
     }
 
     /// Parse `&pat` / `&mut pat`.
-    fn parse_pat_deref(&mut self, expected: Expected) -> PResult<'a, PatKind> {
+    fn parse_pat_deref(&mut self, expected: Option<Expected>) -> PResult<'a, PatKind> {
         self.expect_and()?;
-        self.recover_lifetime_in_deref_pat();
-        let mutbl = self.parse_mutability();
-        let subpat = self.parse_pat_with_range_pat(false, expected)?;
-        Ok(PatKind::Ref(subpat, mutbl))
-    }
-
-    fn recover_lifetime_in_deref_pat(&mut self) {
         if let token::Lifetime(name) = self.token.kind {
             self.bump(); // `'a`
 
-            let span = self.prev_token.span;
-            self.struct_span_err(span, &format!("unexpected lifetime `{}` in pattern", name))
-                .span_suggestion(span, "remove the lifetime", "", Applicability::MachineApplicable)
-                .emit();
+            self.sess
+                .emit_err(UnexpectedLifetimeInPattern { span: self.prev_token.span, symbol: name });
         }
+
+        let mutbl = self.parse_mutability();
+        let subpat = self.parse_pat_with_range_pat(false, expected)?;
+        Ok(PatKind::Ref(subpat, mutbl))
     }
 
     /// Parse a tuple or parenthesis pattern.
@@ -587,7 +581,8 @@ impl<'a> Parser<'a> {
         let mut_span = self.prev_token.span;
 
         if self.eat_keyword(kw::Ref) {
-            return self.recover_mut_ref_ident(mut_span);
+            self.sess.emit_err(RefMutOrderIncorrect { span: mut_span.to(self.prev_token.span) });
+            return self.parse_pat_ident(BindingAnnotation::REF_MUT);
         }
 
         self.recover_additional_muts();
@@ -600,7 +595,7 @@ impl<'a> Parser<'a> {
         }
 
         // Parse the pattern we hope to be an identifier.
-        let mut pat = self.parse_pat_no_top_alt(Some("identifier"))?;
+        let mut pat = self.parse_pat_no_top_alt(Some(Expected::Identifier))?;
 
         // If we don't have `mut $ident (@ pat)?`, error.
         if let PatKind::Ident(BindingAnnotation(ByRef::No, m @ Mutability::Not), ..) = &mut pat.kind
@@ -617,22 +612,6 @@ impl<'a> Parser<'a> {
         Ok(pat.into_inner().kind)
     }
 
-    /// Recover on `mut ref? ident @ pat` and suggest
-    /// that the order of `mut` and `ref` is incorrect.
-    fn recover_mut_ref_ident(&mut self, lo: Span) -> PResult<'a, PatKind> {
-        let mutref_span = lo.to(self.prev_token.span);
-        self.struct_span_err(mutref_span, "the order of `mut` and `ref` is incorrect")
-            .span_suggestion(
-                mutref_span,
-                "try switching the order",
-                "ref mut",
-                Applicability::MachineApplicable,
-            )
-            .emit();
-
-        self.parse_pat_ident(BindingAnnotation::REF_MUT)
-    }
-
     /// Turn all by-value immutable bindings in a pattern into mutable bindings.
     /// Returns `true` if any change was made.
     fn make_all_value_bindings_mutable(pat: &mut P<Pat>) -> bool {
@@ -657,16 +636,13 @@ impl<'a> Parser<'a> {
     /// Error on `mut $pat` where `$pat` is not an ident.
     fn ban_mut_general_pat(&self, lo: Span, pat: &Pat, changed_any_binding: bool) {
         let span = lo.to(pat.span);
-        let fix = pprust::pat_to_string(&pat);
-        let (problem, suggestion) = if changed_any_binding {
-            ("`mut` must be attached to each individual binding", "add `mut` to each binding")
+        let pat = pprust::pat_to_string(&pat);
+
+        self.sess.emit_err(if changed_any_binding {
+            InvalidMutInPattern::NestedIdent { span, pat }
         } else {
-            ("`mut` must be followed by a named binding", "remove the `mut` prefix")
-        };
-        self.struct_span_err(span, problem)
-            .span_suggestion(span, suggestion, fix, Applicability::MachineApplicable)
-            .note("`mut` may be followed by `variable` and `variable @ pattern`")
-            .emit();
+            InvalidMutInPattern::NonIdent { span, pat }
+        });
     }
 
     /// Eat any extraneous `mut`s and error + recover if we ate any.
@@ -677,15 +653,7 @@ impl<'a> Parser<'a> {
             return;
         }
 
-        let span = lo.to(self.prev_token.span);
-        self.struct_span_err(span, "`mut` on a binding may not be repeated")
-            .span_suggestion(
-                span,
-                "remove the additional `mut`s",
-                "",
-                Applicability::MachineApplicable,
-            )
-            .emit();
+        self.sess.emit_err(RepeatedMutInPattern { span: lo.to(self.prev_token.span) });
     }
 
     /// Parse macro invocation
@@ -699,11 +667,11 @@ impl<'a> Parser<'a> {
     fn fatal_unexpected_non_pat(
         &mut self,
         err: DiagnosticBuilder<'a, ErrorGuaranteed>,
-        expected: Expected,
+        expected: Option<Expected>,
     ) -> PResult<'a, P<Pat>> {
         err.cancel();
 
-        let expected = expected.unwrap_or("pattern");
+        let expected = Expected::to_string_or_fallback(expected);
         let msg = format!("expected {}, found {}", expected, super::token_descr(&self.token));
 
         let mut err = self.struct_span_err(self.token.span, &msg);
@@ -745,49 +713,44 @@ impl<'a> Parser<'a> {
             // Parsing e.g. `X..`.
             if let RangeEnd::Included(_) = re.node {
                 // FIXME(Centril): Consider semantic errors instead in `ast_validation`.
-                self.inclusive_range_with_incorrect_end(re.span);
+                self.inclusive_range_with_incorrect_end();
             }
             None
         };
         Ok(PatKind::Range(Some(begin), end, re))
     }
 
-    pub(super) fn inclusive_range_with_incorrect_end(&mut self, span: Span) {
+    pub(super) fn inclusive_range_with_incorrect_end(&mut self) {
         let tok = &self.token;
-
+        let span = self.prev_token.span;
         // If the user typed "..==" instead of "..=", we want to give them
         // a specific error message telling them to use "..=".
+        // If they typed "..=>", suggest they use ".. =>".
         // Otherwise, we assume that they meant to type a half open exclusive
         // range and give them an error telling them to do that instead.
-        if matches!(tok.kind, token::Eq) && tok.span.lo() == span.hi() {
-            let span_with_eq = span.to(tok.span);
+        let no_space = tok.span.lo() == span.hi();
+        match tok.kind {
+            token::Eq if no_space => {
+                let span_with_eq = span.to(tok.span);
 
-            // Ensure the user doesn't receive unhelpful unexpected token errors
-            self.bump();
-            if self.is_pat_range_end_start(0) {
-                let _ = self.parse_pat_range_end().map_err(|e| e.cancel());
-            }
+                // Ensure the user doesn't receive unhelpful unexpected token errors
+                self.bump();
+                if self.is_pat_range_end_start(0) {
+                    let _ = self.parse_pat_range_end().map_err(|e| e.cancel());
+                }
 
-            self.error_inclusive_range_with_extra_equals(span_with_eq);
-        } else {
-            self.error_inclusive_range_with_no_end(span);
+                self.sess.emit_err(InclusiveRangeExtraEquals { span: span_with_eq });
+            }
+            token::Gt if no_space => {
+                let after_pat = span.with_hi(span.hi() - rustc_span::BytePos(1)).shrink_to_hi();
+                self.sess.emit_err(InclusiveRangeMatchArrow { span, arrow: tok.span, after_pat });
+            }
+            _ => {
+                self.sess.emit_err(InclusiveRangeNoEnd { span });
+            }
         }
     }
 
-    fn error_inclusive_range_with_extra_equals(&self, span: Span) {
-        self.struct_span_err(span, "unexpected `=` after inclusive range")
-            .span_suggestion_short(span, "use `..=` instead", "..=", Applicability::MaybeIncorrect)
-            .note("inclusive ranges end with a single equals sign (`..=`)")
-            .emit();
-    }
-
-    fn error_inclusive_range_with_no_end(&self, span: Span) {
-        struct_span_err!(self.sess.span_diagnostic, span, E0586, "inclusive range with no end")
-            .span_suggestion_short(span, "use `..` instead", "..", Applicability::MachineApplicable)
-            .note("inclusive ranges must be bounded at the end (`..=b` or `a..=b`)")
-            .emit();
-    }
-
     /// Parse a range-to pattern, `..X` or `..=X` where `X` remains to be parsed.
     ///
     /// The form `...X` is prohibited to reduce confusion with the potential
@@ -796,14 +759,7 @@ impl<'a> Parser<'a> {
         let end = self.parse_pat_range_end()?;
         if let RangeEnd::Included(syn @ RangeSyntax::DotDotDot) = &mut re.node {
             *syn = RangeSyntax::DotDotEq;
-            self.struct_span_err(re.span, "range-to patterns with `...` are not allowed")
-                .span_suggestion_short(
-                    re.span,
-                    "use `..=` instead",
-                    "..=",
-                    Applicability::MachineApplicable,
-                )
-                .emit();
+            self.sess.emit_err(DotDotDotRangeToPatternNotAllowed { span: re.span });
         }
         Ok(PatKind::Range(None, Some(end), re))
     }
@@ -868,7 +824,7 @@ impl<'a> Parser<'a> {
     fn parse_pat_ident(&mut self, binding_annotation: BindingAnnotation) -> PResult<'a, PatKind> {
         let ident = self.parse_ident()?;
         let sub = if self.eat(&token::At) {
-            Some(self.parse_pat_no_top_alt(Some("binding pattern"))?)
+            Some(self.parse_pat_no_top_alt(Some(Expected::BindingPattern))?)
         } else {
             None
         };
@@ -879,8 +835,8 @@ impl<'a> Parser<'a> {
         // binding mode then we do not end up here, because the lookahead
         // will direct us over to `parse_enum_variant()`.
         if self.token == token::OpenDelim(Delimiter::Parenthesis) {
-            return Err(self
-                .struct_span_err(self.prev_token.span, "expected identifier, found enum pattern"));
+            return Err(EnumPatternInsteadOfIdentifier { span: self.prev_token.span }
+                .into_diagnostic(&self.sess.span_diagnostic));
         }
 
         Ok(PatKind::Ident(binding_annotation, ident, sub))
@@ -897,7 +853,7 @@ impl<'a> Parser<'a> {
             e.span_label(path.span, "while parsing the fields for this pattern");
             e.emit();
             self.recover_stmt();
-            (vec![], true)
+            (ThinVec::new(), true)
         });
         self.bump();
         Ok(PatKind::Struct(qself, path, fields, etc))
@@ -962,7 +918,7 @@ impl<'a> Parser<'a> {
             // We cannot use `parse_pat_ident()` since it will complain `box`
             // is not an identifier.
             let sub = if self.eat(&token::At) {
-                Some(self.parse_pat_no_top_alt(Some("binding pattern"))?)
+                Some(self.parse_pat_no_top_alt(Some(Expected::BindingPattern))?)
             } else {
                 None
             };
@@ -976,8 +932,8 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses the fields of a struct-like pattern.
-    fn parse_pat_fields(&mut self) -> PResult<'a, (Vec<PatField>, bool)> {
-        let mut fields = Vec::new();
+    fn parse_pat_fields(&mut self) -> PResult<'a, (ThinVec<PatField>, bool)> {
+        let mut fields = ThinVec::new();
         let mut etc = false;
         let mut ate_comma = true;
         let mut delayed_err: Option<DiagnosticBuilder<'a, ErrorGuaranteed>> = None;
@@ -997,7 +953,8 @@ impl<'a> Parser<'a> {
 
             // check that a comma comes after every field
             if !ate_comma {
-                let err = self.struct_span_err(self.token.span, "expected `,`");
+                let err = ExpectedCommaAfterPatternField { span: self.token.span }
+                    .into_diagnostic(&self.sess.span_diagnostic);
                 if let Some(mut delayed) = delayed_err {
                     delayed.emit();
                 }
@@ -1005,12 +962,15 @@ impl<'a> Parser<'a> {
             }
             ate_comma = false;
 
-            if self.check(&token::DotDot) || self.token == token::DotDotDot {
+            if self.check(&token::DotDot)
+                || self.check_noexpect(&token::DotDotDot)
+                || self.check_keyword(kw::Underscore)
+            {
                 etc = true;
                 let mut etc_sp = self.token.span;
 
-                self.recover_one_fewer_dotdot();
-                self.bump(); // `..` || `...`
+                self.recover_bad_dot_dot();
+                self.bump(); // `..` || `...` || `_`
 
                 if self.token == token::CloseDelim(Delimiter::Brace) {
                     etc_span = Some(etc_sp);
@@ -1103,21 +1063,15 @@ impl<'a> Parser<'a> {
         Ok((fields, etc))
     }
 
-    /// Recover on `...` as if it were `..` to avoid further errors.
+    /// Recover on `...` or `_` as if it were `..` to avoid further errors.
     /// See issue #46718.
-    fn recover_one_fewer_dotdot(&self) {
-        if self.token != token::DotDotDot {
+    fn recover_bad_dot_dot(&self) {
+        if self.token == token::DotDot {
             return;
         }
 
-        self.struct_span_err(self.token.span, "expected field pattern, found `...`")
-            .span_suggestion(
-                self.token.span,
-                "to omit remaining fields, use one fewer `.`",
-                "..",
-                Applicability::MachineApplicable,
-            )
-            .emit();
+        let token_str = pprust::token_to_string(&self.token);
+        self.sess.emit_err(DotDotDotForRemainingFields { span: self.token.span, token_str });
     }
 
     fn parse_pat_field(&mut self, lo: Span, attrs: AttrVec) -> PResult<'a, PatField> {
diff --git a/compiler/rustc_parse/src/parser/path.rs b/compiler/rustc_parse/src/parser/path.rs
index 5333d3b85..b50d2984a 100644
--- a/compiler/rustc_parse/src/parser/path.rs
+++ b/compiler/rustc_parse/src/parser/path.rs
@@ -332,7 +332,7 @@ impl<'a> Parser<'a> {
         style: PathStyle,
         lo: Span,
         ty_generics: Option<&Generics>,
-    ) -> PResult<'a, Vec<AngleBracketedArg>> {
+    ) -> PResult<'a, ThinVec<AngleBracketedArg>> {
         // We need to detect whether there are extra leading left angle brackets and produce an
         // appropriate error and suggestion. This cannot be implemented by looking ahead at
         // upcoming tokens for a matching `>` character - if there are unmatched `<` tokens
@@ -404,7 +404,7 @@ impl<'a> Parser<'a> {
 
         let is_first_invocation = style == PathStyle::Expr;
         // Take a snapshot before attempting to parse - we can restore this later.
-        let snapshot = if is_first_invocation { Some(self.clone()) } else { None };
+        let snapshot = is_first_invocation.then(|| self.clone());
 
         debug!("parse_generic_args_with_leading_angle_bracket_recovery: (snapshotting)");
         match self.parse_angle_args(ty_generics) {
@@ -472,8 +472,8 @@ impl<'a> Parser<'a> {
     pub(super) fn parse_angle_args(
         &mut self,
         ty_generics: Option<&Generics>,
-    ) -> PResult<'a, Vec<AngleBracketedArg>> {
-        let mut args = Vec::new();
+    ) -> PResult<'a, ThinVec<AngleBracketedArg>> {
+        let mut args = ThinVec::new();
         while let Some(arg) = self.parse_angle_arg(ty_generics)? {
             args.push(arg);
             if !self.eat(&token::Comma) {
@@ -653,7 +653,7 @@ impl<'a> Parser<'a> {
     pub(super) fn parse_const_arg(&mut self) -> PResult<'a, AnonConst> {
         // Parse const argument.
         let value = if let token::OpenDelim(Delimiter::Brace) = self.token.kind {
-            self.parse_block_expr(None, self.token.span, BlockCheckMode::Default)?
+            self.parse_expr_block(None, self.token.span, BlockCheckMode::Default)?
         } else {
             self.handle_unambiguous_unbraced_const_arg()?
         };
@@ -675,22 +675,42 @@ impl<'a> Parser<'a> {
             GenericArg::Const(self.parse_const_arg()?)
         } else if self.check_type() {
             // Parse type argument.
-            let is_const_fn =
-                self.look_ahead(1, |t| t.kind == token::OpenDelim(Delimiter::Parenthesis));
-            let mut snapshot = self.create_snapshot_for_diagnostic();
+
+            // Proactively create a parser snapshot enabling us to rewind and try to reparse the
+            // input as a const expression in case we fail to parse a type. If we successfully
+            // do so, we will report an error that it needs to be wrapped in braces.
+            let mut snapshot = None;
+            if self.may_recover() && self.token.can_begin_expr() {
+                snapshot = Some(self.create_snapshot_for_diagnostic());
+            }
+
             match self.parse_ty() {
-                Ok(ty) => GenericArg::Type(ty),
+                Ok(ty) => {
+                    // Since the type parser recovers from some malformed slice and array types and
+                    // successfully returns a type, we need to look for `TyKind::Err`s in the
+                    // type to determine if error recovery has occurred and if the input is not a
+                    // syntactically valid type after all.
+                    if let ast::TyKind::Slice(inner_ty) | ast::TyKind::Array(inner_ty, _) = &ty.kind
+                        && let ast::TyKind::Err = inner_ty.kind
+                        && let Some(snapshot) = snapshot
+                        && let Some(expr) = self.recover_unbraced_const_arg_that_can_begin_ty(snapshot)
+                    {
+                        return Ok(Some(self.dummy_const_arg_needs_braces(
+                            self.struct_span_err(expr.span, "invalid const generic expression"),
+                            expr.span,
+                        )));
+                    }
+
+                    GenericArg::Type(ty)
+                }
                 Err(err) => {
-                    if is_const_fn {
-                        match (*snapshot).parse_expr_res(Restrictions::CONST_EXPR, None) {
-                            Ok(expr) => {
-                                self.restore_snapshot(snapshot);
-                                return Ok(Some(self.dummy_const_arg_needs_braces(err, expr.span)));
-                            }
-                            Err(err) => {
-                                err.cancel();
-                            }
-                        }
+                    if let Some(snapshot) = snapshot
+                        && let Some(expr) = self.recover_unbraced_const_arg_that_can_begin_ty(snapshot)
+                    {
+                        return Ok(Some(self.dummy_const_arg_needs_braces(
+                            err,
+                            expr.span,
+                        )));
                     }
                     // Try to recover from possible `const` arg without braces.
                     return self.recover_const_arg(start, err).map(Some);
diff --git a/compiler/rustc_parse/src/parser/stmt.rs b/compiler/rustc_parse/src/parser/stmt.rs
index 4ff9927aa..92a22ffc2 100644
--- a/compiler/rustc_parse/src/parser/stmt.rs
+++ b/compiler/rustc_parse/src/parser/stmt.rs
@@ -1,18 +1,13 @@
 use super::attr::InnerAttrForbiddenReason;
 use super::diagnostics::AttemptLocalParseRecovery;
 use super::expr::LhsExpr;
-use super::pat::RecoverComma;
+use super::pat::{PatternLocation, RecoverComma};
 use super::path::PathStyle;
 use super::TrailingToken;
 use super::{
     AttrWrapper, BlockMode, FnParseMode, ForceCollect, Parser, Restrictions, SemiColonMode,
 };
-use crate::errors::{
-    AssignmentElseNotAllowed, CompoundAssignmentExpressionInLet, ConstLetMutuallyExclusive,
-    DocCommentDoesNotDocumentAnything, ExpectedStatementAfterOuterAttr, InvalidCurlyInLetElse,
-    InvalidExpressionInLetElse, InvalidIdentiferStartsWithNumber, InvalidVariableDeclaration,
-    InvalidVariableDeclarationSub, WrapExpressionInParentheses,
-};
+use crate::errors;
 use crate::maybe_whole;
 
 use rustc_ast as ast;
@@ -25,8 +20,8 @@ use rustc_ast::{StmtKind, DUMMY_NODE_ID};
 use rustc_errors::{Applicability, DiagnosticBuilder, ErrorGuaranteed, PResult};
 use rustc_span::source_map::{BytePos, Span};
 use rustc_span::symbol::{kw, sym};
-
 use std::mem;
+use thin_vec::{thin_vec, ThinVec};
 
 impl<'a> Parser<'a> {
     /// Parses a statement. This stops just before trailing semicolons on everything but items.
@@ -64,40 +59,45 @@ impl<'a> Parser<'a> {
         if self.token.is_keyword(kw::Mut) && self.is_keyword_ahead(1, &[kw::Let]) {
             self.bump();
             let mut_let_span = lo.to(self.token.span);
-            self.sess.emit_err(InvalidVariableDeclaration {
+            self.sess.emit_err(errors::InvalidVariableDeclaration {
                 span: mut_let_span,
-                sub: InvalidVariableDeclarationSub::SwitchMutLetOrder(mut_let_span),
+                sub: errors::InvalidVariableDeclarationSub::SwitchMutLetOrder(mut_let_span),
             });
         }
 
         Ok(Some(if self.token.is_keyword(kw::Let) {
             self.parse_local_mk(lo, attrs, capture_semi, force_collect)?
         } else if self.is_kw_followed_by_ident(kw::Mut) && self.may_recover() {
-            self.recover_stmt_local_after_let(lo, attrs, InvalidVariableDeclarationSub::MissingLet)?
+            self.recover_stmt_local_after_let(
+                lo,
+                attrs,
+                errors::InvalidVariableDeclarationSub::MissingLet,
+            )?
         } else if self.is_kw_followed_by_ident(kw::Auto) && self.may_recover() {
             self.bump(); // `auto`
             self.recover_stmt_local_after_let(
                 lo,
                 attrs,
-                InvalidVariableDeclarationSub::UseLetNotAuto,
+                errors::InvalidVariableDeclarationSub::UseLetNotAuto,
             )?
         } else if self.is_kw_followed_by_ident(sym::var) && self.may_recover() {
             self.bump(); // `var`
             self.recover_stmt_local_after_let(
                 lo,
                 attrs,
-                InvalidVariableDeclarationSub::UseLetNotVar,
+                errors::InvalidVariableDeclarationSub::UseLetNotVar,
             )?
         } else if self.check_path() && !self.token.is_qpath_start() && !self.is_path_start_item() {
             // We have avoided contextual keywords like `union`, items with `crate` visibility,
             // or `auto trait` items. We aim to parse an arbitrary path `a::b` but not something
             // that starts like a path (1 token), but it fact not a path.
             // Also, we avoid stealing syntax from `parse_item_`.
-            if force_collect == ForceCollect::Yes {
-                self.collect_tokens_no_attrs(|this| this.parse_stmt_path_start(lo, attrs))
-            } else {
-                self.parse_stmt_path_start(lo, attrs)
-            }?
+            match force_collect {
+                ForceCollect::Yes => {
+                    self.collect_tokens_no_attrs(|this| this.parse_stmt_path_start(lo, attrs))?
+                }
+                ForceCollect::No => self.parse_stmt_path_start(lo, attrs)?,
+            }
         } else if let Some(item) = self.parse_item_common(
             attrs.clone(),
             false,
@@ -113,18 +113,17 @@ impl<'a> Parser<'a> {
             self.mk_stmt(lo, StmtKind::Empty)
         } else if self.token != token::CloseDelim(Delimiter::Brace) {
             // Remainder are line-expr stmts.
-            let e = if force_collect == ForceCollect::Yes {
-                self.collect_tokens_no_attrs(|this| {
+            let e = match force_collect {
+                ForceCollect::Yes => self.collect_tokens_no_attrs(|this| {
                     this.parse_expr_res(Restrictions::STMT_EXPR, Some(attrs))
-                })
-            } else {
-                self.parse_expr_res(Restrictions::STMT_EXPR, Some(attrs))
-            }?;
+                })?,
+                ForceCollect::No => self.parse_expr_res(Restrictions::STMT_EXPR, Some(attrs))?,
+            };
             if matches!(e.kind, ExprKind::Assign(..)) && self.eat_keyword(kw::Else) {
                 let bl = self.parse_block()?;
                 // Destructuring assignment ... else.
                 // This is not allowed, but point it out in a nice way.
-                self.sess.emit_err(AssignmentElseNotAllowed { span: e.span.to(bl.span) });
+                self.sess.emit_err(errors::AssignmentElseNotAllowed { span: e.span.to(bl.span) });
             }
             self.mk_stmt(lo.to(e.span), StmtKind::Expr(e))
         } else {
@@ -147,14 +146,14 @@ impl<'a> Parser<'a> {
             }
 
             let expr = if this.eat(&token::OpenDelim(Delimiter::Brace)) {
-                this.parse_struct_expr(None, path, true)?
+                this.parse_expr_struct(None, path, true)?
             } else {
                 let hi = this.prev_token.span;
                 this.mk_expr(lo.to(hi), ExprKind::Path(None, path))
             };
 
             let expr = this.with_res(Restrictions::STMT_EXPR, |this| {
-                this.parse_dot_or_call_expr_with(expr, lo, attrs)
+                this.parse_expr_dot_or_call_with(expr, lo, attrs)
             })?;
             // `DUMMY_SP` will get overwritten later in this function
             Ok((this.mk_stmt(rustc_span::DUMMY_SP, StmtKind::Expr(expr)), TrailingToken::None))
@@ -164,7 +163,7 @@ impl<'a> Parser<'a> {
             // Perform this outside of the `collect_tokens_trailing_token` closure,
             // since our outer attributes do not apply to this part of the expression
             let expr = self.with_res(Restrictions::STMT_EXPR, |this| {
-                this.parse_assoc_expr_with(
+                this.parse_expr_assoc_with(
                     0,
                     LhsExpr::AlreadyParsed { expr, starts_statement: true },
                 )
@@ -200,8 +199,8 @@ impl<'a> Parser<'a> {
             // Since none of the above applied, this is an expression statement macro.
             let e = self.mk_expr(lo.to(hi), ExprKind::MacCall(mac));
             let e = self.maybe_recover_from_bad_qpath(e)?;
-            let e = self.parse_dot_or_call_expr_with(e, lo, attrs)?;
-            let e = self.parse_assoc_expr_with(
+            let e = self.parse_expr_dot_or_call_with(e, lo, attrs)?;
+            let e = self.parse_expr_assoc_with(
                 0,
                 LhsExpr::AlreadyParsed { expr: e, starts_statement: false },
             )?;
@@ -217,12 +216,12 @@ impl<'a> Parser<'a> {
         && let attrs = attrs.take_for_recovery(self.sess)
         && let attrs @ [.., last] = &*attrs {
             if last.is_doc_comment() {
-                self.sess.emit_err(DocCommentDoesNotDocumentAnything {
+                self.sess.emit_err(errors::DocCommentDoesNotDocumentAnything {
                     span: last.span,
                     missing_comma: None,
                 });
             } else if attrs.iter().any(|a| a.style == AttrStyle::Outer) {
-                self.sess.emit_err(ExpectedStatementAfterOuterAttr { span: last.span });
+                self.sess.emit_err(errors::ExpectedStatementAfterOuterAttr { span: last.span });
             }
         }
     }
@@ -231,7 +230,7 @@ impl<'a> Parser<'a> {
         &mut self,
         lo: Span,
         attrs: AttrWrapper,
-        subdiagnostic: fn(Span) -> InvalidVariableDeclarationSub,
+        subdiagnostic: fn(Span) -> errors::InvalidVariableDeclarationSub,
     ) -> PResult<'a, Stmt> {
         let stmt =
             self.collect_tokens_trailing_token(attrs, ForceCollect::Yes, |this, attrs| {
@@ -242,7 +241,7 @@ impl<'a> Parser<'a> {
                     TrailingToken::None,
                 ))
             })?;
-        self.sess.emit_err(InvalidVariableDeclaration { span: lo, sub: subdiagnostic(lo) });
+        self.sess.emit_err(errors::InvalidVariableDeclaration { span: lo, sub: subdiagnostic(lo) });
         Ok(stmt)
     }
 
@@ -270,12 +269,13 @@ impl<'a> Parser<'a> {
         let lo = self.prev_token.span;
 
         if self.token.is_keyword(kw::Const) && self.look_ahead(1, |t| t.is_ident()) {
-            self.sess.emit_err(ConstLetMutuallyExclusive { span: lo.to(self.token.span) });
+            self.sess.emit_err(errors::ConstLetMutuallyExclusive { span: lo.to(self.token.span) });
             self.bump();
         }
 
         self.report_invalid_identifier_error()?;
-        let (pat, colon) = self.parse_pat_before_ty(None, RecoverComma::Yes, "`let` bindings")?;
+        let (pat, colon) =
+            self.parse_pat_before_ty(None, RecoverComma::Yes, PatternLocation::LetBinding)?;
 
         let (err, ty) = if colon {
             // Save the state of the parser before parsing type normally, in case there is a `:`
@@ -372,7 +372,7 @@ impl<'a> Parser<'a> {
             rustc_ast::MetaItemLit::from_token(&self.token).is_none() &&
             (lit.kind == token::LitKind::Integer || lit.kind == token::LitKind::Float) &&
             self.look_ahead(1, |t| matches!(t.kind, token::Eq) || matches!(t.kind, token::Colon ) ) {
-                return Err(self.sess.create_err(InvalidIdentiferStartsWithNumber { span: self.token.span }));
+                return Err(self.sess.create_err(errors::InvalidIdentiferStartsWithNumber { span: self.token.span }));
         }
         Ok(())
     }
@@ -380,10 +380,10 @@ impl<'a> Parser<'a> {
     fn check_let_else_init_bool_expr(&self, init: &ast::Expr) {
         if let ast::ExprKind::Binary(op, ..) = init.kind {
             if op.node.lazy() {
-                self.sess.emit_err(InvalidExpressionInLetElse {
+                self.sess.emit_err(errors::InvalidExpressionInLetElse {
                     span: init.span,
                     operator: op.node.to_string(),
-                    sugg: WrapExpressionInParentheses {
+                    sugg: errors::WrapExpressionInParentheses {
                         left: init.span.shrink_to_lo(),
                         right: init.span.shrink_to_hi(),
                     },
@@ -394,9 +394,9 @@ impl<'a> Parser<'a> {
 
     fn check_let_else_init_trailing_brace(&self, init: &ast::Expr) {
         if let Some(trailing) = classify::expr_trailing_brace(init) {
-            self.sess.emit_err(InvalidCurlyInLetElse {
+            self.sess.emit_err(errors::InvalidCurlyInLetElse {
                 span: trailing.span.with_lo(trailing.span.hi() - BytePos(1)),
-                sugg: WrapExpressionInParentheses {
+                sugg: errors::WrapExpressionInParentheses {
                     left: trailing.span.shrink_to_lo(),
                     right: trailing.span.shrink_to_hi(),
                 },
@@ -409,7 +409,8 @@ impl<'a> Parser<'a> {
         let eq_consumed = match self.token.kind {
             token::BinOpEq(..) => {
                 // Recover `let x <op>= 1` as `let x = 1`
-                self.sess.emit_err(CompoundAssignmentExpressionInLet { span: self.token.span });
+                self.sess
+                    .emit_err(errors::CompoundAssignmentExpressionInLet { span: self.token.span });
                 self.bump();
                 true
             }
@@ -543,7 +544,7 @@ impl<'a> Parser<'a> {
         s: BlockCheckMode,
         recover: AttemptLocalParseRecovery,
     ) -> PResult<'a, P<Block>> {
-        let mut stmts = vec![];
+        let mut stmts = ThinVec::new();
         let mut snapshot = None;
         while !self.eat(&token::CloseDelim(Delimiter::Brace)) {
             if self.token == token::Eof {
@@ -661,7 +662,12 @@ impl<'a> Parser<'a> {
         Ok(Some(stmt))
     }
 
-    pub(super) fn mk_block(&self, stmts: Vec<Stmt>, rules: BlockCheckMode, span: Span) -> P<Block> {
+    pub(super) fn mk_block(
+        &self,
+        stmts: ThinVec<Stmt>,
+        rules: BlockCheckMode,
+        span: Span,
+    ) -> P<Block> {
         P(Block {
             stmts,
             id: DUMMY_NODE_ID,
@@ -681,6 +687,6 @@ impl<'a> Parser<'a> {
     }
 
     pub(super) fn mk_block_err(&self, span: Span) -> P<Block> {
-        self.mk_block(vec![self.mk_stmt_err(span)], BlockCheckMode::Default, span)
+        self.mk_block(thin_vec![self.mk_stmt_err(span)], BlockCheckMode::Default, span)
     }
 }
diff --git a/compiler/rustc_parse/src/parser/ty.rs b/compiler/rustc_parse/src/parser/ty.rs
index 867974672..6fe4da71f 100644
--- a/compiler/rustc_parse/src/parser/ty.rs
+++ b/compiler/rustc_parse/src/parser/ty.rs
@@ -1,6 +1,12 @@
 use super::{Parser, PathStyle, TokenType};
 
-use crate::errors::{ExpectedFnPathFoundFnKeyword, FnPtrWithGenerics, FnPtrWithGenericsSugg};
+use crate::errors::{
+    DynAfterMut, ExpectedFnPathFoundFnKeyword, ExpectedMutOrConstInRawPointerType,
+    FnPointerCannotBeAsync, FnPointerCannotBeConst, FnPtrWithGenerics, FnPtrWithGenericsSugg,
+    InvalidDynKeyword, LifetimeAfterMut, NeedPlusAfterTraitObjectLifetime,
+    NegativeBoundsNotSupported, NegativeBoundsNotSupportedSugg, NestedCVariadicType,
+    ReturnTypesUseThinArrow,
+};
 use crate::{maybe_recover_from_interpolated_ty_qpath, maybe_whole};
 
 use ast::DUMMY_NODE_ID;
@@ -11,11 +17,11 @@ use rustc_ast::{
     self as ast, BareFnTy, FnRetTy, GenericBound, GenericBounds, GenericParam, Generics, Lifetime,
     MacCall, MutTy, Mutability, PolyTraitRef, TraitBoundModifier, TraitObjectSyntax, Ty, TyKind,
 };
-use rustc_errors::{pluralize, struct_span_err, Applicability, PResult};
+use rustc_errors::{Applicability, PResult};
 use rustc_span::source_map::Span;
 use rustc_span::symbol::{kw, sym, Ident};
 use rustc_span::Symbol;
-use thin_vec::thin_vec;
+use thin_vec::{thin_vec, ThinVec};
 
 /// Any `?` or `~const` modifiers that appear at the start of a bound.
 struct BoundModifiers {
@@ -217,14 +223,7 @@ impl<'a> Parser<'a> {
             // Don't `eat` to prevent `=>` from being added as an expected token which isn't
             // actually expected and could only confuse users
             self.bump();
-            self.struct_span_err(self.prev_token.span, "return types are denoted using `->`")
-                .span_suggestion_short(
-                    self.prev_token.span,
-                    "use `->` instead",
-                    "->",
-                    Applicability::MachineApplicable,
-                )
-                .emit();
+            self.sess.emit_err(ReturnTypesUseThinArrow { span: self.prev_token.span });
             let ty = self.parse_ty_common(
                 allow_plus,
                 AllowCVariadic::No,
@@ -274,7 +273,7 @@ impl<'a> Parser<'a> {
             TyKind::Infer
         } else if self.check_fn_front_matter(false, Case::Sensitive) {
             // Function pointer type
-            self.parse_ty_bare_fn(lo, Vec::new(), None, recover_return_sign)?
+            self.parse_ty_bare_fn(lo, ThinVec::new(), None, recover_return_sign)?
         } else if self.check_keyword(kw::For) {
             // Function pointer type or bound list (trait object type) starting with a poly-trait.
             //   `for<'lt> [unsafe] [extern "ABI"] fn (&'lt S) -> T`
@@ -305,13 +304,14 @@ impl<'a> Parser<'a> {
         } else if self.can_begin_bound() {
             self.parse_bare_trait_object(lo, allow_plus)?
         } else if self.eat(&token::DotDotDot) {
-            if allow_c_variadic == AllowCVariadic::Yes {
-                TyKind::CVarArgs
-            } else {
-                // FIXME(Centril): Should we just allow `...` syntactically
-                // anywhere in a type and use semantic restrictions instead?
-                self.error_illegal_c_varadic_ty(lo);
-                TyKind::Err
+            match allow_c_variadic {
+                AllowCVariadic::Yes => TyKind::CVarArgs,
+                AllowCVariadic::No => {
+                    // FIXME(Centril): Should we just allow `...` syntactically
+                    // anywhere in a type and use semantic restrictions instead?
+                    self.sess.emit_err(NestedCVariadicType { span: lo.to(self.prev_token.span) });
+                    TyKind::Err
+                }
             }
         } else {
             let msg = format!("expected type, found {}", super::token_descr(&self.token));
@@ -325,10 +325,9 @@ impl<'a> Parser<'a> {
         let mut ty = self.mk_ty(span, kind);
 
         // Try to recover from use of `+` with incorrect priority.
-        if matches!(allow_plus, AllowPlus::Yes) {
-            self.maybe_recover_from_bad_type_plus(&ty)?;
-        } else {
-            self.maybe_report_ambiguous_plus(impl_dyn_multi, &ty);
+        match allow_plus {
+            AllowPlus::Yes => self.maybe_recover_from_bad_type_plus(&ty)?,
+            AllowPlus::No => self.maybe_report_ambiguous_plus(impl_dyn_multi, &ty),
         }
         if let RecoverQuestionMark::Yes = recover_question_mark {
             ty = self.maybe_recover_from_question_mark(ty);
@@ -353,7 +352,7 @@ impl<'a> Parser<'a> {
             match ty.kind {
                 // `(TY_BOUND_NOPAREN) + BOUND + ...`.
                 TyKind::Path(None, path) if maybe_bounds => {
-                    self.parse_remaining_bounds_path(Vec::new(), path, lo, true)
+                    self.parse_remaining_bounds_path(ThinVec::new(), path, lo, true)
                 }
                 TyKind::TraitObject(bounds, TraitObjectSyntax::None)
                     if maybe_bounds && bounds.len() == 1 && !trailing_plus =>
@@ -372,15 +371,14 @@ impl<'a> Parser<'a> {
         let lt_no_plus = self.check_lifetime() && !self.look_ahead(1, |t| t.is_like_plus());
         let bounds = self.parse_generic_bounds_common(allow_plus, None)?;
         if lt_no_plus {
-            self.struct_span_err(lo, "lifetime in trait object type must be followed by `+`")
-                .emit();
+            self.sess.emit_err(NeedPlusAfterTraitObjectLifetime { span: lo });
         }
         Ok(TyKind::TraitObject(bounds, TraitObjectSyntax::None))
     }
 
     fn parse_remaining_bounds_path(
         &mut self,
-        generic_params: Vec<GenericParam>,
+        generic_params: ThinVec<GenericParam>,
         path: ast::Path,
         lo: Span,
         parse_plus: bool,
@@ -407,14 +405,10 @@ impl<'a> Parser<'a> {
     fn parse_ty_ptr(&mut self) -> PResult<'a, TyKind> {
         let mutbl = self.parse_const_or_mut().unwrap_or_else(|| {
             let span = self.prev_token.span;
-            self.struct_span_err(span, "expected `mut` or `const` keyword in raw pointer type")
-                .span_suggestions(
-                    span.shrink_to_hi(),
-                    "add `mut` or `const` here",
-                    ["mut ".to_string(), "const ".to_string()],
-                    Applicability::HasPlaceholders,
-                )
-                .emit();
+            self.sess.emit_err(ExpectedMutOrConstInRawPointerType {
+                span,
+                after_asterisk: span.shrink_to_hi(),
+            });
             Mutability::Not
         });
         let ty = self.parse_ty_no_plus()?;
@@ -439,7 +433,7 @@ impl<'a> Parser<'a> {
         };
 
         let ty = if self.eat(&token::Semi) {
-            let mut length = self.parse_anon_const_expr()?;
+            let mut length = self.parse_expr_anon_const()?;
             if let Err(e) = self.expect(&token::CloseDelim(Delimiter::Bracket)) {
                 // Try to recover from `X<Y, ...>` when `X::<Y, ...>` works
                 self.check_mistyped_turbofish_with_multiple_type_params(e, &mut length.value)?;
@@ -456,8 +450,7 @@ impl<'a> Parser<'a> {
 
     fn parse_borrowed_pointee(&mut self) -> PResult<'a, TyKind> {
         let and_span = self.prev_token.span;
-        let mut opt_lifetime =
-            if self.check_lifetime() { Some(self.expect_lifetime()) } else { None };
+        let mut opt_lifetime = self.check_lifetime().then(|| self.expect_lifetime());
         let mut mutbl = self.parse_mutability();
         if self.token.is_lifetime() && mutbl == Mutability::Mut && opt_lifetime.is_none() {
             // A lifetime is invalid here: it would be part of a bare trait bound, which requires
@@ -469,16 +462,13 @@ impl<'a> Parser<'a> {
                 let lifetime_span = self.token.span;
                 let span = and_span.to(lifetime_span);
 
-                let mut err = self.struct_span_err(span, "lifetime must precede `mut`");
-                if let Ok(lifetime_src) = self.span_to_snippet(lifetime_span) {
-                    err.span_suggestion(
-                        span,
-                        "place the lifetime before `mut`",
-                        format!("&{} mut", lifetime_src),
-                        Applicability::MaybeIncorrect,
-                    );
-                }
-                err.emit();
+                let (suggest_lifetime, snippet) =
+                    if let Ok(lifetime_src) = self.span_to_snippet(lifetime_span) {
+                        (Some(span), lifetime_src)
+                    } else {
+                        (None, String::new())
+                    };
+                self.sess.emit_err(LifetimeAfterMut { span, suggest_lifetime, snippet });
 
                 opt_lifetime = Some(self.expect_lifetime());
             }
@@ -488,14 +478,7 @@ impl<'a> Parser<'a> {
         {
             // We have `&dyn mut ...`, which is invalid and should be `&mut dyn ...`.
             let span = and_span.to(self.look_ahead(1, |t| t.span));
-            let mut err = self.struct_span_err(span, "`mut` must precede `dyn`");
-            err.span_suggestion(
-                span,
-                "place `mut` before `dyn`",
-                "&mut dyn",
-                Applicability::MachineApplicable,
-            );
-            err.emit();
+            self.sess.emit_err(DynAfterMut { span });
 
             // Recovery
             mutbl = Mutability::Mut;
@@ -511,7 +494,7 @@ impl<'a> Parser<'a> {
     // To avoid ambiguity, the type is surrounded by parentheses.
     fn parse_typeof_ty(&mut self) -> PResult<'a, TyKind> {
         self.expect(&token::OpenDelim(Delimiter::Parenthesis))?;
-        let expr = self.parse_anon_const_expr()?;
+        let expr = self.parse_expr_anon_const()?;
         self.expect(&token::CloseDelim(Delimiter::Parenthesis))?;
         Ok(TyKind::Typeof(expr))
     }
@@ -528,7 +511,7 @@ impl<'a> Parser<'a> {
     fn parse_ty_bare_fn(
         &mut self,
         lo: Span,
-        mut params: Vec<GenericParam>,
+        mut params: ThinVec<GenericParam>,
         param_insertion_point: Option<Span>,
         recover_return_sign: RecoverReturnSign,
     ) -> PResult<'a, TyKind> {
@@ -549,10 +532,10 @@ impl<'a> Parser<'a> {
             // If we ever start to allow `const fn()`, then update
             // feature gating for `#![feature(const_extern_fn)]` to
             // cover it.
-            self.error_fn_ptr_bad_qualifier(whole_span, span, "const");
+            self.sess.emit_err(FnPointerCannotBeConst { span: whole_span, qualifier: span });
         }
         if let ast::Async::Yes { span, .. } = asyncness {
-            self.error_fn_ptr_bad_qualifier(whole_span, span, "async");
+            self.sess.emit_err(FnPointerCannotBeAsync { span: whole_span, qualifier: span });
         }
         let decl_span = span_start.to(self.token.span);
         Ok(TyKind::BareFn(P(BareFnTy { ext, unsafety, generic_params: params, decl, decl_span })))
@@ -562,13 +545,13 @@ impl<'a> Parser<'a> {
     fn recover_fn_ptr_with_generics(
         &mut self,
         lo: Span,
-        params: &mut Vec<GenericParam>,
+        params: &mut ThinVec<GenericParam>,
         param_insertion_point: Option<Span>,
     ) -> PResult<'a, ()> {
         let generics = self.parse_generics()?;
         let arity = generics.params.len();
 
-        let mut lifetimes: Vec<_> = generics
+        let mut lifetimes: ThinVec<_> = generics
             .params
             .into_iter()
             .filter(|param| matches!(param.kind, ast::GenericParamKind::Lifetime))
@@ -600,19 +583,6 @@ impl<'a> Parser<'a> {
         Ok(())
     }
 
-    /// Emit an error for the given bad function pointer qualifier.
-    fn error_fn_ptr_bad_qualifier(&self, span: Span, qual_span: Span, qual: &str) {
-        self.struct_span_err(span, &format!("an `fn` pointer type cannot be `{}`", qual))
-            .span_label(qual_span, format!("`{}` because of this", qual))
-            .span_suggestion_short(
-                qual_span,
-                &format!("remove the `{}` qualifier", qual),
-                "",
-                Applicability::MaybeIncorrect,
-            )
-            .emit();
-    }
-
     /// Parses an `impl B0 + ... + Bn` type.
     fn parse_impl_ty(&mut self, impl_dyn_multi: &mut bool) -> PResult<'a, TyKind> {
         // Always parse bounds greedily for better error recovery.
@@ -643,7 +613,7 @@ impl<'a> Parser<'a> {
     /// Is a `dyn B0 + ... + Bn` type allowed here?
     fn is_explicit_dyn_type(&mut self) -> bool {
         self.check_keyword(kw::Dyn)
-            && (!self.token.uninterpolated_span().rust_2015()
+            && (self.token.uninterpolated_span().rust_2018()
                 || self.look_ahead(1, |t| {
                     (t.can_begin_bound() || t.kind == TokenKind::BinOp(token::Star))
                         && !can_continue_type_after_non_fn_ident(t)
@@ -692,23 +662,13 @@ impl<'a> Parser<'a> {
             })))
         } else if allow_plus == AllowPlus::Yes && self.check_plus() {
             // `Trait1 + Trait2 + 'a`
-            self.parse_remaining_bounds_path(Vec::new(), path, lo, true)
+            self.parse_remaining_bounds_path(ThinVec::new(), path, lo, true)
         } else {
             // Just a type path.
             Ok(TyKind::Path(None, path))
         }
     }
 
-    fn error_illegal_c_varadic_ty(&self, lo: Span) {
-        struct_span_err!(
-            self.sess.span_diagnostic,
-            lo.to(self.prev_token.span),
-            E0743,
-            "C-variadic type `...` may not be nested inside another type",
-        )
-        .emit();
-    }
-
     pub(super) fn parse_generic_bounds(
         &mut self,
         colon_span: Option<Span>,
@@ -739,15 +699,7 @@ impl<'a> Parser<'a> {
         {
             if self.token.is_keyword(kw::Dyn) {
                 // Account for `&dyn Trait + dyn Other`.
-                self.struct_span_err(self.token.span, "invalid `dyn` keyword")
-                    .help("`dyn` is only needed at the start of a trait `+`-separated list")
-                    .span_suggestion(
-                        self.token.span,
-                        "remove this keyword",
-                        "",
-                        Applicability::MachineApplicable,
-                    )
-                    .emit();
+                self.sess.emit_err(InvalidDynKeyword { span: self.token.span });
                 self.bump();
             }
             match self.parse_generic_bound()? {
@@ -784,11 +736,7 @@ impl<'a> Parser<'a> {
         bounds: &[GenericBound],
         negative_bounds: Vec<Span>,
     ) {
-        let negative_bounds_len = negative_bounds.len();
-        let last_span = *negative_bounds.last().expect("no negative bounds, but still error?");
-        let mut err = self.struct_span_err(negative_bounds, "negative bounds are not supported");
-        err.span_label(last_span, "negative bounds are not supported");
-        if let Some(bound_list) = colon_span {
+        let sub = if let Some(bound_list) = colon_span {
             let bound_list = bound_list.to(self.prev_token.span);
             let mut new_bound_list = String::new();
             if !bounds.is_empty() {
@@ -799,14 +747,18 @@ impl<'a> Parser<'a> {
                 }
                 new_bound_list = new_bound_list.replacen(" +", ":", 1);
             }
-            err.tool_only_span_suggestion(
+
+            Some(NegativeBoundsNotSupportedSugg {
                 bound_list,
-                &format!("remove the bound{}", pluralize!(negative_bounds_len)),
-                new_bound_list,
-                Applicability::MachineApplicable,
-            );
-        }
-        err.emit();
+                num_bounds: negative_bounds.len(),
+                fixed: new_bound_list,
+            })
+        } else {
+            None
+        };
+
+        let last_span = *negative_bounds.last().expect("no negative bounds, but still error?");
+        self.sess.emit_err(NegativeBoundsNotSupported { negative_bounds, last_span, sub });
     }
 
     /// Parses a bound according to the grammar:
@@ -918,7 +870,7 @@ impl<'a> Parser<'a> {
             None
         };
 
-        let maybe = if self.eat(&token::Question) { Some(self.prev_token.span) } else { None };
+        let maybe = self.eat(&token::Question).then_some(self.prev_token.span);
 
         Ok(BoundModifiers { maybe, maybe_const })
     }
@@ -990,7 +942,7 @@ impl<'a> Parser<'a> {
                 self.parse_remaining_bounds(bounds, true)?;
                 self.expect(&token::CloseDelim(Delimiter::Parenthesis))?;
                 let sp = vec![lo, self.prev_token.span];
-                let sugg: Vec<_> = sp.iter().map(|sp| (*sp, String::new())).collect();
+                let sugg = vec![(lo, String::from(" ")), (self.prev_token.span, String::new())];
                 self.struct_span_err(sp, "incorrect braces around trait bounds")
                     .multipart_suggestion(
                         "remove the parentheses",
@@ -1041,7 +993,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Optionally parses `for<$generic_params>`.
-    pub(super) fn parse_late_bound_lifetime_defs(&mut self) -> PResult<'a, Vec<GenericParam>> {
+    pub(super) fn parse_late_bound_lifetime_defs(&mut self) -> PResult<'a, ThinVec<GenericParam>> {
         if self.eat_keyword(kw::For) {
             self.expect_lt()?;
             let params = self.parse_generic_params()?;
@@ -1050,7 +1002,7 @@ impl<'a> Parser<'a> {
             // parameters, and the lifetime parameters must not have bounds.
             Ok(params)
         } else {
-            Ok(Vec::new())
+            Ok(ThinVec::new())
         }
     }
 
@@ -1060,7 +1012,7 @@ impl<'a> Parser<'a> {
     fn recover_fn_trait_with_lifetime_params(
         &mut self,
         fn_path: &mut ast::Path,
-        lifetime_defs: &mut Vec<GenericParam>,
+        lifetime_defs: &mut ThinVec<GenericParam>,
     ) -> PResult<'a, ()> {
         let fn_path_segment = fn_path.segments.last_mut().unwrap();
         let generic_args = if let Some(p_args) = &fn_path_segment.args {
@@ -1094,7 +1046,7 @@ impl<'a> Parser<'a> {
 
         // Parse `(T, U) -> R`.
         let inputs_lo = self.token.span;
-        let inputs: Vec<_> =
+        let inputs: ThinVec<_> =
             self.parse_fn_params(|_| false)?.into_iter().map(|input| input.ty).collect();
         let inputs_span = inputs_lo.to(self.prev_token.span);
         let output = self.parse_ret_ty(AllowPlus::No, RecoverQPath::No, RecoverReturnSign::No)?;
@@ -1120,7 +1072,7 @@ impl<'a> Parser<'a> {
                 kind: ast::GenericParamKind::Lifetime,
                 colon_span: None,
             })
-            .collect::<Vec<GenericParam>>();
+            .collect::<ThinVec<GenericParam>>();
         lifetime_defs.append(&mut generic_params);
 
         let generic_args_span = generic_args.span();
diff --git a/compiler/rustc_parse_format/src/lib.rs b/compiler/rustc_parse_format/src/lib.rs
index 7b016cada..8a3cedfee 100644
--- a/compiler/rustc_parse_format/src/lib.rs
+++ b/compiler/rustc_parse_format/src/lib.rs
@@ -16,7 +16,6 @@
 
 pub use Alignment::*;
 pub use Count::*;
-pub use Flag::*;
 pub use Piece::*;
 pub use Position::*;
 
@@ -111,8 +110,14 @@ pub struct FormatSpec<'a> {
     pub fill: Option<char>,
     /// Optionally specified alignment.
     pub align: Alignment,
-    /// Packed version of various flags provided.
-    pub flags: u32,
+    /// The `+` or `-` flag.
+    pub sign: Option<Sign>,
+    /// The `#` flag.
+    pub alternate: bool,
+    /// The `0` flag.
+    pub zero_pad: bool,
+    /// The `x` or `X` flag. (Only for `Debug`.)
+    pub debug_hex: Option<DebugHex>,
     /// The integer precision to use.
     pub precision: Count<'a>,
     /// The span of the precision formatting flag (for diagnostics).
@@ -162,24 +167,22 @@ pub enum Alignment {
     AlignUnknown,
 }
 
-/// Various flags which can be applied to format strings. The meaning of these
-/// flags is defined by the formatters themselves.
+/// Enum for the sign flags.
 #[derive(Copy, Clone, Debug, PartialEq)]
-pub enum Flag {
-    /// A `+` will be used to denote positive numbers.
-    FlagSignPlus,
-    /// A `-` will be used to denote negative numbers. This is the default.
-    FlagSignMinus,
-    /// An alternate form will be used for the value. In the case of numbers,
-    /// this means that the number will be prefixed with the supplied string.
-    FlagAlternate,
-    /// For numbers, this means that the number will be padded with zeroes,
-    /// and the sign (`+` or `-`) will precede them.
-    FlagSignAwareZeroPad,
-    /// For Debug / `?`, format integers in lower-case hexadecimal.
-    FlagDebugLowerHex,
-    /// For Debug / `?`, format integers in upper-case hexadecimal.
-    FlagDebugUpperHex,
+pub enum Sign {
+    /// The `+` flag.
+    Plus,
+    /// The `-` flag.
+    Minus,
+}
+
+/// Enum for the debug hex flags.
+#[derive(Copy, Clone, Debug, PartialEq)]
+pub enum DebugHex {
+    /// The `x` flag in `{:x?}`.
+    Lower,
+    /// The `X` flag in `{:X?}`.
+    Upper,
 }
 
 /// A count is used for the precision and width parameters of an integer, and
@@ -270,7 +273,13 @@ impl<'a> Iterator for Parser<'a> {
                                 );
                             }
                         } else {
-                            self.suggest_positional_arg_instead_of_captured_arg(arg);
+                            if let Some(&(_, maybe)) = self.cur.peek() {
+                                if maybe == '?' {
+                                    self.suggest_format();
+                                } else {
+                                    self.suggest_positional_arg_instead_of_captured_arg(arg);
+                                }
+                            }
                         }
                         Some(NextArgument(Box::new(arg)))
                     }
@@ -597,7 +606,10 @@ impl<'a> Parser<'a> {
         let mut spec = FormatSpec {
             fill: None,
             align: AlignUnknown,
-            flags: 0,
+            sign: None,
+            alternate: false,
+            zero_pad: false,
+            debug_hex: None,
             precision: CountImplied,
             precision_span: None,
             width: CountImplied,
@@ -626,13 +638,13 @@ impl<'a> Parser<'a> {
         }
         // Sign flags
         if self.consume('+') {
-            spec.flags |= 1 << (FlagSignPlus as u32);
+            spec.sign = Some(Sign::Plus);
         } else if self.consume('-') {
-            spec.flags |= 1 << (FlagSignMinus as u32);
+            spec.sign = Some(Sign::Minus);
         }
         // Alternate marker
         if self.consume('#') {
-            spec.flags |= 1 << (FlagAlternate as u32);
+            spec.alternate = true;
         }
         // Width and precision
         let mut havewidth = false;
@@ -647,7 +659,7 @@ impl<'a> Parser<'a> {
                 spec.width_span = Some(self.span(end - 1, end + 1));
                 havewidth = true;
             } else {
-                spec.flags |= 1 << (FlagSignAwareZeroPad as u32);
+                spec.zero_pad = true;
             }
         }
 
@@ -678,14 +690,14 @@ impl<'a> Parser<'a> {
         // Optional radix followed by the actual format specifier
         if self.consume('x') {
             if self.consume('?') {
-                spec.flags |= 1 << (FlagDebugLowerHex as u32);
+                spec.debug_hex = Some(DebugHex::Lower);
                 spec.ty = "?";
             } else {
                 spec.ty = "x";
             }
         } else if self.consume('X') {
             if self.consume('?') {
-                spec.flags |= 1 << (FlagDebugUpperHex as u32);
+                spec.debug_hex = Some(DebugHex::Upper);
                 spec.ty = "?";
             } else {
                 spec.ty = "X";
@@ -708,7 +720,10 @@ impl<'a> Parser<'a> {
         let mut spec = FormatSpec {
             fill: None,
             align: AlignUnknown,
-            flags: 0,
+            sign: None,
+            alternate: false,
+            zero_pad: false,
+            debug_hex: None,
             precision: CountImplied,
             precision_span: None,
             width: CountImplied,
@@ -820,7 +835,26 @@ impl<'a> Parser<'a> {
             );
         }
 
-        if found { Some(cur) } else { None }
+        found.then_some(cur)
+    }
+
+    fn suggest_format(&mut self) {
+        if let (Some(pos), Some(_)) = (self.consume_pos('?'), self.consume_pos(':')) {
+            let word = self.word();
+            let _end = self.current_pos();
+            let pos = self.to_span_index(pos);
+            self.errors.insert(
+                0,
+                ParseError {
+                    description: "expected format parameter to occur after `:`".to_owned(),
+                    note: Some(format!("`?` comes after `:`, try `{}:{}` instead", word, "?")),
+                    label: "expected `?` to occur after `:`".to_owned(),
+                    span: pos.to(pos),
+                    secondary_label: None,
+                    should_be_replaced_with_positional_argument: false,
+                },
+            );
+        }
     }
 
     fn suggest_positional_arg_instead_of_captured_arg(&mut self, arg: Argument<'a>) {
diff --git a/compiler/rustc_parse_format/src/tests.rs b/compiler/rustc_parse_format/src/tests.rs
index 2992ba845..45314e2fb 100644
--- a/compiler/rustc_parse_format/src/tests.rs
+++ b/compiler/rustc_parse_format/src/tests.rs
@@ -10,7 +10,10 @@ fn fmtdflt() -> FormatSpec<'static> {
     return FormatSpec {
         fill: None,
         align: AlignUnknown,
-        flags: 0,
+        sign: None,
+        alternate: false,
+        zero_pad: false,
+        debug_hex: None,
         precision: CountImplied,
         width: CountImplied,
         precision_span: None,
@@ -126,7 +129,10 @@ fn format_type() {
             format: FormatSpec {
                 fill: None,
                 align: AlignUnknown,
-                flags: 0,
+                sign: None,
+                alternate: false,
+                zero_pad: false,
+                debug_hex: None,
                 precision: CountImplied,
                 width: CountImplied,
                 precision_span: None,
@@ -147,7 +153,10 @@ fn format_align_fill() {
             format: FormatSpec {
                 fill: None,
                 align: AlignRight,
-                flags: 0,
+                sign: None,
+                alternate: false,
+                zero_pad: false,
+                debug_hex: None,
                 precision: CountImplied,
                 width: CountImplied,
                 precision_span: None,
@@ -165,7 +174,10 @@ fn format_align_fill() {
             format: FormatSpec {
                 fill: Some('0'),
                 align: AlignLeft,
-                flags: 0,
+                sign: None,
+                alternate: false,
+                zero_pad: false,
+                debug_hex: None,
                 precision: CountImplied,
                 width: CountImplied,
                 precision_span: None,
@@ -183,7 +195,10 @@ fn format_align_fill() {
             format: FormatSpec {
                 fill: Some('*'),
                 align: AlignLeft,
-                flags: 0,
+                sign: None,
+                alternate: false,
+                zero_pad: false,
+                debug_hex: None,
                 precision: CountImplied,
                 width: CountImplied,
                 precision_span: None,
@@ -204,7 +219,10 @@ fn format_counts() {
             format: FormatSpec {
                 fill: None,
                 align: AlignUnknown,
-                flags: 0,
+                sign: None,
+                alternate: false,
+                zero_pad: false,
+                debug_hex: None,
                 precision: CountImplied,
                 precision_span: None,
                 width: CountIs(10),
@@ -222,7 +240,10 @@ fn format_counts() {
             format: FormatSpec {
                 fill: None,
                 align: AlignUnknown,
-                flags: 0,
+                sign: None,
+                alternate: false,
+                zero_pad: false,
+                debug_hex: None,
                 precision: CountIs(10),
                 precision_span: Some(InnerSpan { start: 6, end: 9 }),
                 width: CountIsParam(10),
@@ -240,7 +261,10 @@ fn format_counts() {
             format: FormatSpec {
                 fill: None,
                 align: AlignUnknown,
-                flags: 0,
+                sign: None,
+                alternate: false,
+                zero_pad: false,
+                debug_hex: None,
                 precision: CountIs(10),
                 precision_span: Some(InnerSpan { start: 6, end: 9 }),
                 width: CountIsParam(0),
@@ -258,7 +282,10 @@ fn format_counts() {
             format: FormatSpec {
                 fill: None,
                 align: AlignUnknown,
-                flags: 0,
+                sign: None,
+                alternate: false,
+                zero_pad: false,
+                debug_hex: None,
                 precision: CountIsStar(0),
                 precision_span: Some(InnerSpan { start: 3, end: 5 }),
                 width: CountImplied,
@@ -276,7 +303,10 @@ fn format_counts() {
             format: FormatSpec {
                 fill: None,
                 align: AlignUnknown,
-                flags: 0,
+                sign: None,
+                alternate: false,
+                zero_pad: false,
+                debug_hex: None,
                 precision: CountIsParam(10),
                 width: CountImplied,
                 precision_span: Some(InnerSpan::new(3, 7)),
@@ -294,7 +324,10 @@ fn format_counts() {
             format: FormatSpec {
                 fill: None,
                 align: AlignUnknown,
-                flags: 0,
+                sign: None,
+                alternate: false,
+                zero_pad: false,
+                debug_hex: None,
                 precision: CountIsName("b", InnerSpan { start: 6, end: 7 }),
                 precision_span: Some(InnerSpan { start: 5, end: 8 }),
                 width: CountIsName("a", InnerSpan { start: 3, end: 4 }),
@@ -312,7 +345,10 @@ fn format_counts() {
             format: FormatSpec {
                 fill: None,
                 align: AlignUnknown,
-                flags: 0,
+                sign: None,
+                alternate: false,
+                zero_pad: false,
+                debug_hex: None,
                 precision: CountIs(4),
                 precision_span: Some(InnerSpan { start: 3, end: 5 }),
                 width: CountImplied,
@@ -333,7 +369,10 @@ fn format_flags() {
             format: FormatSpec {
                 fill: None,
                 align: AlignUnknown,
-                flags: (1 << FlagSignMinus as u32),
+                sign: Some(Sign::Minus),
+                alternate: false,
+                zero_pad: false,
+                debug_hex: None,
                 precision: CountImplied,
                 width: CountImplied,
                 precision_span: None,
@@ -351,7 +390,10 @@ fn format_flags() {
             format: FormatSpec {
                 fill: None,
                 align: AlignUnknown,
-                flags: (1 << FlagSignPlus as u32) | (1 << FlagAlternate as u32),
+                sign: Some(Sign::Plus),
+                alternate: true,
+                zero_pad: false,
+                debug_hex: None,
                 precision: CountImplied,
                 width: CountImplied,
                 precision_span: None,
@@ -374,7 +416,10 @@ fn format_mixture() {
                 format: FormatSpec {
                     fill: None,
                     align: AlignUnknown,
-                    flags: 0,
+                    sign: None,
+                    alternate: false,
+                    zero_pad: false,
+                    debug_hex: None,
                     precision: CountImplied,
                     width: CountImplied,
                     precision_span: None,
diff --git a/compiler/rustc_error_messages/locales/en-US/passes.ftl b/compiler/rustc_passes/locales/en-US.ftl
index 91857dd22..3fa78efc2 100644
--- a/compiler/rustc_error_messages/locales/en-US/passes.ftl
+++ b/compiler/rustc_passes/locales/en-US.ftl
@@ -182,6 +182,18 @@ passes_has_incoherent_inherent_impl =
     `rustc_has_incoherent_inherent_impls` attribute should be applied to types or traits.
     .label = only adts, extern types and traits are supported
 
+passes_both_ffi_const_and_pure =
+    `#[ffi_const]` function cannot be `#[ffi_pure]`
+
+passes_ffi_pure_invalid_target =
+    `#[ffi_pure]` may only be used on foreign functions
+
+passes_ffi_const_invalid_target =
+    `#[ffi_const]` may only be used on foreign functions
+
+passes_ffi_returns_twice_invalid_target =
+    `#[ffi_returns_twice]` may only be used on foreign functions
+
 passes_must_use_async =
     `must_use` attribute on `async` functions applies to the anonymous `Future` returned by the function, not the value within
     .label = this attribute does nothing, the `Future`s returned by async functions are already `must_use`
@@ -390,15 +402,12 @@ passes_invalid_attr_at_crate_level =
     `{$name}` attribute cannot be used at crate level
     .suggestion = perhaps you meant to use an outer attribute
 
-passes_duplicate_diagnostic_item =
-    duplicate diagnostic item found: `{$name}`.
-
 passes_duplicate_diagnostic_item_in_crate =
     duplicate diagnostic item in crate `{$crate_name}`: `{$name}`.
+    .note = the diagnostic item is first defined in crate `{$orig_crate_name}`.
 
 passes_diagnostic_item_first_defined =
     the diagnostic item is first defined here
-    .note = the diagnostic item is first defined in crate `{$orig_crate_name}`.
 
 passes_abi =
     abi: {$abi}
@@ -710,3 +719,30 @@ passes_ignored_derived_impls =
       [one] trait {$trait_list}, but this is
      *[other] traits {$trait_list}, but these are
     } intentionally ignored during dead code analysis
+
+passes_proc_macro_typeerror = mismatched {$kind} signature
+    .label = found {$found}, expected type `proc_macro::TokenStream`
+    .note = {$kind}s must have a signature of `{$expected_signature}`
+
+passes_proc_macro_diff_arg_count = mismatched {$kind} signature
+    .label = found unexpected {$count ->
+      [one] argument
+     *[other] arguments
+    }
+    .note = {$kind}s must have a signature of `{$expected_signature}`
+
+passes_proc_macro_missing_args = mismatched {$kind} signature
+    .label = {$kind} must have {$expected_input_count ->
+      [one] one argument
+     *[other] two arguments
+    } of type `proc_macro::TokenStream`
+
+passes_proc_macro_invalid_abi = proc macro functions may not be `extern "{$abi}"`
+
+passes_proc_macro_unsafe = proc macro functions may not be `unsafe`
+
+passes_skipping_const_checks = skipping const checks
+
+passes_invalid_macro_export_arguments = `{$name}` isn't a valid `#[macro_export]` argument
+
+passes_invalid_macro_export_arguments_too_many_items = `#[macro_export]` can only take 1 or 0 arguments
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index f9f9799d3..5ef3e13ef 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -4,13 +4,10 @@
 //! conflicts between multiple such attributes attached to the same
 //! item.
 
-use crate::errors::{
-    self, AttrApplication, DebugVisualizerUnreadable, InvalidAttrAtCrateLevel, ObjectLifetimeErr,
-    OnlyHasEffectOn, TransparentIncompatible, UnrecognizedReprHint,
-};
+use crate::{errors, fluent_generated as fluent};
 use rustc_ast::{ast, AttrStyle, Attribute, LitKind, MetaItemKind, MetaItemLit, NestedMetaItem};
 use rustc_data_structures::fx::FxHashMap;
-use rustc_errors::{fluent, Applicability, MultiSpan};
+use rustc_errors::{Applicability, IntoDiagnosticArg, MultiSpan};
 use rustc_expand::base::resolve_path;
 use rustc_feature::{AttributeDuplicates, AttributeType, BuiltinAttribute, BUILTIN_ATTRIBUTE_MAP};
 use rustc_hir as hir;
@@ -19,18 +16,21 @@ use rustc_hir::intravisit::{self, Visitor};
 use rustc_hir::{
     self, FnSig, ForeignItem, HirId, Item, ItemKind, TraitItem, CRATE_HIR_ID, CRATE_OWNER_ID,
 };
-use rustc_hir::{MethodKind, Target};
+use rustc_hir::{MethodKind, Target, Unsafety};
 use rustc_middle::hir::nested_filter;
-use rustc_middle::middle::resolve_lifetime::ObjectLifetimeDefault;
+use rustc_middle::middle::resolve_bound_vars::ObjectLifetimeDefault;
+use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams};
 use rustc_middle::ty::query::Providers;
-use rustc_middle::ty::TyCtxt;
+use rustc_middle::ty::{ParamEnv, TyCtxt};
 use rustc_session::lint::builtin::{
-    CONFLICTING_REPR_HINTS, INVALID_DOC_ATTRIBUTES, UNUSED_ATTRIBUTES,
+    CONFLICTING_REPR_HINTS, INVALID_DOC_ATTRIBUTES, INVALID_MACRO_EXPORT_ARGUMENTS,
+    UNUSED_ATTRIBUTES,
 };
 use rustc_session::parse::feature_err;
 use rustc_span::symbol::{kw, sym, Symbol};
 use rustc_span::{Span, DUMMY_SP};
 use rustc_target::spec::abi::Abi;
+use std::cell::Cell;
 use std::collections::hash_map::Entry;
 
 pub(crate) fn target_from_impl_item<'tcx>(
@@ -62,8 +62,29 @@ enum ItemLike<'tcx> {
     ForeignItem,
 }
 
+#[derive(Copy, Clone)]
+pub(crate) enum ProcMacroKind {
+    FunctionLike,
+    Derive,
+    Attribute,
+}
+
+impl IntoDiagnosticArg for ProcMacroKind {
+    fn into_diagnostic_arg(self) -> rustc_errors::DiagnosticArgValue<'static> {
+        match self {
+            ProcMacroKind::Attribute => "attribute proc macro",
+            ProcMacroKind::Derive => "derive proc macro",
+            ProcMacroKind::FunctionLike => "function-like proc macro",
+        }
+        .into_diagnostic_arg()
+    }
+}
+
 struct CheckAttrVisitor<'tcx> {
     tcx: TyCtxt<'tcx>,
+
+    // Whether or not this visitor should abort after finding errors
+    abort: Cell<bool>,
 }
 
 impl CheckAttrVisitor<'_> {
@@ -136,6 +157,7 @@ impl CheckAttrVisitor<'_> {
                 | sym::rustc_dirty
                 | sym::rustc_if_this_changed
                 | sym::rustc_then_this_would_need => self.check_rustc_dirty_clean(&attr),
+                sym::rustc_coinductive => self.check_rustc_coinductive(&attr, span, target),
                 sym::cmse_nonsecure_entry => {
                     self.check_cmse_nonsecure_entry(hir_id, attr, span, target)
                 }
@@ -150,6 +172,9 @@ impl CheckAttrVisitor<'_> {
                 sym::rustc_has_incoherent_inherent_impls => {
                     self.check_has_incoherent_inherent_impls(&attr, span, target)
                 }
+                sym::ffi_pure => self.check_ffi_pure(attr.span, attrs, target),
+                sym::ffi_const => self.check_ffi_const(attr.span, target),
+                sym::ffi_returns_twice => self.check_ffi_returns_twice(attr.span, target),
                 sym::rustc_const_unstable
                 | sym::rustc_const_stable
                 | sym::unstable
@@ -173,7 +198,7 @@ impl CheckAttrVisitor<'_> {
                 sym::path => self.check_generic_attr(hir_id, attr, target, Target::Mod),
                 sym::plugin_registrar => self.check_plugin_registrar(hir_id, attr, target),
                 sym::macro_export => self.check_macro_export(hir_id, attr, target),
-                sym::ignore | sym::should_panic | sym::proc_macro_derive => {
+                sym::ignore | sym::should_panic => {
                     self.check_generic_attr(hir_id, attr, target, Target::Fn)
                 }
                 sym::automatically_derived => {
@@ -183,6 +208,16 @@ impl CheckAttrVisitor<'_> {
                     self.check_generic_attr(hir_id, attr, target, Target::Mod)
                 }
                 sym::rustc_object_lifetime_default => self.check_object_lifetime_default(hir_id),
+                sym::proc_macro => {
+                    self.check_proc_macro(hir_id, target, ProcMacroKind::FunctionLike)
+                }
+                sym::proc_macro_attribute => {
+                    self.check_proc_macro(hir_id, target, ProcMacroKind::Attribute);
+                }
+                sym::proc_macro_derive => {
+                    self.check_generic_attr(hir_id, attr, target, Target::Fn);
+                    self.check_proc_macro(hir_id, target, ProcMacroKind::Derive)
+                }
                 _ => {}
             }
 
@@ -362,7 +397,7 @@ impl CheckAttrVisitor<'_> {
                 UNUSED_ATTRIBUTES,
                 hir_id,
                 attr.span,
-                OnlyHasEffectOn {
+                errors::OnlyHasEffectOn {
                     attr_name: attr.name_or_empty(),
                     target_name: allowed_target.name().replace(' ', "_"),
                 },
@@ -419,7 +454,9 @@ impl CheckAttrVisitor<'_> {
     /// Debugging aid for `object_lifetime_default` query.
     fn check_object_lifetime_default(&self, hir_id: HirId) {
         let tcx = self.tcx;
-        if let Some(generics) = tcx.hir().get_generics(tcx.hir().local_def_id(hir_id)) {
+        if let Some(owner_id) = hir_id.as_owner()
+            && let Some(generics) = tcx.hir().get_generics(owner_id.def_id)
+        {
             for p in generics.params {
                 let hir::GenericParamKind::Type { .. } = p.kind else { continue };
                 let default = tcx.object_lifetime_default(p.def_id);
@@ -429,7 +466,7 @@ impl CheckAttrVisitor<'_> {
                     ObjectLifetimeDefault::Param(def_id) => tcx.item_name(def_id).to_string(),
                     ObjectLifetimeDefault::Ambiguous => "Ambiguous".to_owned(),
                 };
-                tcx.sess.emit_err(ObjectLifetimeErr { span: p.span, repr });
+                tcx.sess.emit_err(errors::ObjectLifetimeErr { span: p.span, repr });
             }
         }
     }
@@ -828,33 +865,39 @@ impl CheckAttrVisitor<'_> {
         target: Target,
         specified_inline: &mut Option<(bool, Span)>,
     ) -> bool {
-        if target == Target::Use || target == Target::ExternCrate {
-            let do_inline = meta.name_or_empty() == sym::inline;
-            if let Some((prev_inline, prev_span)) = *specified_inline {
-                if do_inline != prev_inline {
-                    let mut spans = MultiSpan::from_spans(vec![prev_span, meta.span()]);
-                    spans.push_span_label(prev_span, fluent::passes_doc_inline_conflict_first);
-                    spans.push_span_label(meta.span(), fluent::passes_doc_inline_conflict_second);
-                    self.tcx.sess.emit_err(errors::DocKeywordConflict { spans });
-                    return false;
+        match target {
+            Target::Use | Target::ExternCrate => {
+                let do_inline = meta.name_or_empty() == sym::inline;
+                if let Some((prev_inline, prev_span)) = *specified_inline {
+                    if do_inline != prev_inline {
+                        let mut spans = MultiSpan::from_spans(vec![prev_span, meta.span()]);
+                        spans.push_span_label(prev_span, fluent::passes_doc_inline_conflict_first);
+                        spans.push_span_label(
+                            meta.span(),
+                            fluent::passes_doc_inline_conflict_second,
+                        );
+                        self.tcx.sess.emit_err(errors::DocKeywordConflict { spans });
+                        return false;
+                    }
+                    true
+                } else {
+                    *specified_inline = Some((do_inline, meta.span()));
+                    true
                 }
-                true
-            } else {
-                *specified_inline = Some((do_inline, meta.span()));
-                true
             }
-        } else {
-            self.tcx.emit_spanned_lint(
-                INVALID_DOC_ATTRIBUTES,
-                hir_id,
-                meta.span(),
-                errors::DocInlineOnlyUse {
-                    attr_span: meta.span(),
-                    item_span: (attr.style == AttrStyle::Outer)
-                        .then(|| self.tcx.hir().span(hir_id)),
-                },
-            );
-            false
+            _ => {
+                self.tcx.emit_spanned_lint(
+                    INVALID_DOC_ATTRIBUTES,
+                    hir_id,
+                    meta.span(),
+                    errors::DocInlineOnlyUse {
+                        attr_span: meta.span(),
+                        item_span: (attr.style == AttrStyle::Outer)
+                            .then(|| self.tcx.hir().span(hir_id)),
+                    },
+                );
+                false
+            }
         }
     }
 
@@ -893,15 +936,15 @@ impl CheckAttrVisitor<'_> {
                             src.insert(1, '!');
                             err.span_suggestion_verbose(
                                 attr.span,
-                                fluent::suggestion,
+                                fluent::passes_suggestion,
                                 src,
                                 Applicability::MaybeIncorrect,
                             );
                         } else {
-                            err.span_help(attr.span, fluent::help);
+                            err.span_help(attr.span, fluent::passes_help);
                         }
                     }
-                    err.note(fluent::note);
+                    err.note(fluent::passes_note);
                     err
                 },
             );
@@ -1101,7 +1144,7 @@ impl CheckAttrVisitor<'_> {
                                     errors::DocTestUnknownInclude {
                                         path,
                                         value: value.to_string(),
-                                        inner: if attr.style == AttrStyle::Inner { "!" } else { "" },
+                                        inner: match attr.style { AttrStyle::Inner=>  "!" , AttrStyle::Outer => "" },
                                         sugg: (attr.meta().unwrap().span, applicability),
                                     }
                                 );
@@ -1171,6 +1214,38 @@ impl CheckAttrVisitor<'_> {
         }
     }
 
+    fn check_ffi_pure(&self, attr_span: Span, attrs: &[Attribute], target: Target) -> bool {
+        if target != Target::ForeignFn {
+            self.tcx.sess.emit_err(errors::FfiPureInvalidTarget { attr_span });
+            return false;
+        }
+        if attrs.iter().any(|a| a.has_name(sym::ffi_const)) {
+            // `#[ffi_const]` functions cannot be `#[ffi_pure]`
+            self.tcx.sess.emit_err(errors::BothFfiConstAndPure { attr_span });
+            false
+        } else {
+            true
+        }
+    }
+
+    fn check_ffi_const(&self, attr_span: Span, target: Target) -> bool {
+        if target == Target::ForeignFn {
+            true
+        } else {
+            self.tcx.sess.emit_err(errors::FfiConstInvalidTarget { attr_span });
+            false
+        }
+    }
+
+    fn check_ffi_returns_twice(&self, attr_span: Span, target: Target) -> bool {
+        if target == Target::ForeignFn {
+            true
+        } else {
+            self.tcx.sess.emit_err(errors::FfiReturnsTwiceInvalidTarget { attr_span });
+            false
+        }
+    }
+
     /// Warns against some misuses of `#[must_use]`
     fn check_must_use(&self, hir_id: HirId, attr: &Attribute, target: Target) -> bool {
         if !matches!(
@@ -1535,6 +1610,20 @@ impl CheckAttrVisitor<'_> {
         }
     }
 
+    /// Checks if the `#[rustc_coinductive]` attribute is applied to a trait.
+    fn check_rustc_coinductive(&self, attr: &Attribute, span: Span, target: Target) -> bool {
+        match target {
+            Target::Trait => true,
+            _ => {
+                self.tcx.sess.emit_err(errors::AttrShouldBeAppliedToTrait {
+                    attr_span: attr.span,
+                    defn_span: span,
+                });
+                false
+            }
+        }
+    }
+
     /// Checks if `#[link_section]` is applied to a function or static.
     fn check_link_section(&self, hir_id: HirId, attr: &Attribute, span: Span, target: Target) {
         match target {
@@ -1638,7 +1727,7 @@ impl CheckAttrVisitor<'_> {
                     match target {
                         Target::Struct | Target::Union | Target::Enum => continue,
                         _ => {
-                            self.tcx.sess.emit_err(AttrApplication::StructEnumUnion {
+                            self.tcx.sess.emit_err(errors::AttrApplication::StructEnumUnion {
                                 hint_span: hint.span(),
                                 span,
                             });
@@ -1659,16 +1748,18 @@ impl CheckAttrVisitor<'_> {
                     match target {
                         Target::Struct | Target::Union | Target::Enum | Target::Fn => continue,
                         _ => {
-                            self.tcx.sess.emit_err(AttrApplication::StructEnumFunctionUnion {
-                                hint_span: hint.span(),
-                                span,
-                            });
+                            self.tcx.sess.emit_err(
+                                errors::AttrApplication::StructEnumFunctionUnion {
+                                    hint_span: hint.span(),
+                                    span,
+                                },
+                            );
                         }
                     }
                 }
                 sym::packed => {
                     if target != Target::Struct && target != Target::Union {
-                        self.tcx.sess.emit_err(AttrApplication::StructUnion {
+                        self.tcx.sess.emit_err(errors::AttrApplication::StructUnion {
                             hint_span: hint.span(),
                             span,
                         });
@@ -1679,9 +1770,10 @@ impl CheckAttrVisitor<'_> {
                 sym::simd => {
                     is_simd = true;
                     if target != Target::Struct {
-                        self.tcx
-                            .sess
-                            .emit_err(AttrApplication::Struct { hint_span: hint.span(), span });
+                        self.tcx.sess.emit_err(errors::AttrApplication::Struct {
+                            hint_span: hint.span(),
+                            span,
+                        });
                     } else {
                         continue;
                     }
@@ -1691,7 +1783,7 @@ impl CheckAttrVisitor<'_> {
                     match target {
                         Target::Struct | Target::Union | Target::Enum => continue,
                         _ => {
-                            self.tcx.sess.emit_err(AttrApplication::StructEnumUnion {
+                            self.tcx.sess.emit_err(errors::AttrApplication::StructEnumUnion {
                                 hint_span: hint.span(),
                                 span,
                             });
@@ -1712,15 +1804,16 @@ impl CheckAttrVisitor<'_> {
                 | sym::usize => {
                     int_reprs += 1;
                     if target != Target::Enum {
-                        self.tcx
-                            .sess
-                            .emit_err(AttrApplication::Enum { hint_span: hint.span(), span });
+                        self.tcx.sess.emit_err(errors::AttrApplication::Enum {
+                            hint_span: hint.span(),
+                            span,
+                        });
                     } else {
                         continue;
                     }
                 }
                 _ => {
-                    self.tcx.sess.emit_err(UnrecognizedReprHint { span: hint.span() });
+                    self.tcx.sess.emit_err(errors::UnrecognizedReprHint { span: hint.span() });
                     continue;
                 }
             };
@@ -1733,9 +1826,10 @@ impl CheckAttrVisitor<'_> {
         // Error on repr(transparent, <anything else>).
         if is_transparent && hints.len() > 1 {
             let hint_spans: Vec<_> = hint_spans.clone().collect();
-            self.tcx
-                .sess
-                .emit_err(TransparentIncompatible { hint_spans, target: target.to_string() });
+            self.tcx.sess.emit_err(errors::TransparentIncompatible {
+                hint_spans,
+                target: target.to_string(),
+            });
         }
         // Warn on repr(u8, u16), repr(C, simd), and c-like-enum-repr(C, u8)
         if (int_reprs > 1)
@@ -1888,7 +1982,7 @@ impl CheckAttrVisitor<'_> {
         match std::fs::File::open(&file) {
             Ok(_) => true,
             Err(error) => {
-                self.tcx.sess.emit_err(DebugVisualizerUnreadable {
+                self.tcx.sess.emit_err(errors::DebugVisualizerUnreadable {
                     span: meta_item.span,
                     file: &file,
                     error,
@@ -1909,7 +2003,7 @@ impl CheckAttrVisitor<'_> {
     ) -> bool {
         match target {
             Target::Fn | Target::Method(_)
-                if self.tcx.is_const_fn_raw(self.tcx.hir().local_def_id(hir_id).to_def_id()) =>
+                if self.tcx.is_const_fn_raw(hir_id.expect_owner().to_def_id()) =>
             {
                 true
             }
@@ -2009,7 +2103,33 @@ impl CheckAttrVisitor<'_> {
 
     fn check_macro_export(&self, hir_id: HirId, attr: &Attribute, target: Target) {
         if target != Target::MacroDef {
-            self.tcx.emit_spanned_lint(UNUSED_ATTRIBUTES, hir_id, attr.span, errors::MacroExport);
+            self.tcx.emit_spanned_lint(
+                UNUSED_ATTRIBUTES,
+                hir_id,
+                attr.span,
+                errors::MacroExport::Normal,
+            );
+        } else if let Some(meta_item_list) = attr.meta_item_list() &&
+        !meta_item_list.is_empty() {
+            if meta_item_list.len() > 1 {
+                self.tcx.emit_spanned_lint(
+                    INVALID_MACRO_EXPORT_ARGUMENTS,
+                    hir_id,
+                    attr.span,
+                    errors::MacroExport::TooManyItems,
+                );
+            } else {
+                if meta_item_list[0].name_or_empty() != sym::local_inner_macros {
+                    self.tcx.emit_spanned_lint(
+                        INVALID_MACRO_EXPORT_ARGUMENTS,
+                        hir_id,
+                        meta_item_list[0].span(),
+                        errors::MacroExport::UnknownItem {
+                            name: meta_item_list[0].name_or_empty(),
+                        },
+                    );
+                }
+            }
         }
     }
 
@@ -2063,6 +2183,107 @@ impl CheckAttrVisitor<'_> {
             errors::Unused { attr_span: attr.span, note },
         );
     }
+
+    /// A best effort attempt to create an error for a mismatching proc macro signature.
+    ///
+    /// If this best effort goes wrong, it will just emit a worse error later (see #102923)
+    fn check_proc_macro(&self, hir_id: HirId, target: Target, kind: ProcMacroKind) {
+        let expected_input_count = match kind {
+            ProcMacroKind::Attribute => 2,
+            ProcMacroKind::Derive | ProcMacroKind::FunctionLike => 1,
+        };
+
+        let expected_signature = match kind {
+            ProcMacroKind::Attribute => "fn(TokenStream, TokenStream) -> TokenStream",
+            ProcMacroKind::Derive | ProcMacroKind::FunctionLike => "fn(TokenStream) -> TokenStream",
+        };
+
+        let tcx = self.tcx;
+        if target == Target::Fn {
+            let Some(tokenstream) = tcx.get_diagnostic_item(sym::TokenStream) else {return};
+            let tokenstream = tcx.type_of(tokenstream).subst_identity();
+
+            let id = hir_id.expect_owner();
+            let hir_sig = tcx.hir().fn_sig_by_hir_id(hir_id).unwrap();
+
+            let sig =
+                tcx.liberate_late_bound_regions(id.to_def_id(), tcx.fn_sig(id).subst_identity());
+            let sig = tcx.normalize_erasing_regions(ParamEnv::empty(), sig);
+
+            // We don't currently require that the function signature is equal to
+            // `fn(TokenStream) -> TokenStream`, but instead monomorphizes to
+            // `fn(TokenStream) -> TokenStream` after some substitution of generic arguments.
+            //
+            // Properly checking this means pulling in additional `rustc` crates, so we don't.
+            let drcx = DeepRejectCtxt { treat_obligation_params: TreatParams::AsInfer };
+
+            if sig.abi != Abi::Rust {
+                tcx.sess.emit_err(errors::ProcMacroInvalidAbi {
+                    span: hir_sig.span,
+                    abi: sig.abi.name(),
+                });
+                self.abort.set(true);
+            }
+
+            if sig.unsafety == Unsafety::Unsafe {
+                tcx.sess.emit_err(errors::ProcMacroUnsafe { span: hir_sig.span });
+                self.abort.set(true);
+            }
+
+            let output = sig.output();
+
+            // Typecheck the output
+            if !drcx.types_may_unify(output, tokenstream) {
+                tcx.sess.emit_err(errors::ProcMacroTypeError {
+                    span: hir_sig.decl.output.span(),
+                    found: output,
+                    kind,
+                    expected_signature,
+                });
+                self.abort.set(true);
+            }
+
+            if sig.inputs().len() < expected_input_count {
+                tcx.sess.emit_err(errors::ProcMacroMissingArguments {
+                    expected_input_count,
+                    span: hir_sig.span,
+                    kind,
+                    expected_signature,
+                });
+                self.abort.set(true);
+            }
+
+            // Check that the inputs are correct, if there are enough.
+            if sig.inputs().len() >= expected_input_count {
+                for (arg, input) in
+                    sig.inputs().iter().zip(hir_sig.decl.inputs).take(expected_input_count)
+                {
+                    if !drcx.types_may_unify(*arg, tokenstream) {
+                        tcx.sess.emit_err(errors::ProcMacroTypeError {
+                            span: input.span,
+                            found: *arg,
+                            kind,
+                            expected_signature,
+                        });
+                        self.abort.set(true);
+                    }
+                }
+            }
+
+            // Check that there are not too many arguments
+            let body_id = tcx.hir().body_owned_by(id.def_id);
+            let excess = tcx.hir().body(body_id).params.get(expected_input_count..);
+            if let Some(excess @ [begin @ end] | excess @ [begin, .., end]) = excess {
+                tcx.sess.emit_err(errors::ProcMacroDiffArguments {
+                    span: begin.span.to(end.span),
+                    count: excess.len(),
+                    kind,
+                    expected_signature,
+                });
+                self.abort.set(true);
+            }
+        }
+    }
 }
 
 impl<'tcx> Visitor<'tcx> for CheckAttrVisitor<'tcx> {
@@ -2203,7 +2424,7 @@ fn check_invalid_crate_level_attr(tcx: TyCtxt<'_>, attrs: &[Attribute]) {
         if attr.style == AttrStyle::Inner {
             for attr_to_check in ATTRS_TO_CHECK {
                 if attr.has_name(*attr_to_check) {
-                    tcx.sess.emit_err(InvalidAttrAtCrateLevel {
+                    tcx.sess.emit_err(errors::InvalidAttrAtCrateLevel {
                         span: attr.span,
                         snippet: tcx.sess.source_map().span_to_snippet(attr.span).ok(),
                         name: *attr_to_check,
@@ -2225,12 +2446,15 @@ fn check_non_exported_macro_for_invalid_attrs(tcx: TyCtxt<'_>, item: &Item<'_>)
 }
 
 fn check_mod_attrs(tcx: TyCtxt<'_>, module_def_id: LocalDefId) {
-    let check_attr_visitor = &mut CheckAttrVisitor { tcx };
+    let check_attr_visitor = &mut CheckAttrVisitor { tcx, abort: Cell::new(false) };
     tcx.hir().visit_item_likes_in_module(module_def_id, check_attr_visitor);
     if module_def_id.is_top_level_module() {
         check_attr_visitor.check_attributes(CRATE_HIR_ID, DUMMY_SP, Target::Mod, None);
         check_invalid_crate_level_attr(tcx, tcx.hir().krate_attrs());
     }
+    if check_attr_visitor.abort.get() {
+        tcx.sess.abort_if_errors()
+    }
 }
 
 pub(crate) fn provide(providers: &mut Providers) {
diff --git a/compiler/rustc_passes/src/check_const.rs b/compiler/rustc_passes/src/check_const.rs
index aa726d6cd..526b829bf 100644
--- a/compiler/rustc_passes/src/check_const.rs
+++ b/compiler/rustc_passes/src/check_const.rs
@@ -17,7 +17,7 @@ use rustc_middle::ty::TyCtxt;
 use rustc_session::parse::feature_err;
 use rustc_span::{sym, Span, Symbol};
 
-use crate::errors::ExprNotAllowedInContext;
+use crate::errors::{ExprNotAllowedInContext, SkippingConstChecks};
 
 /// An expression that is not *always* legal in a const context.
 #[derive(Clone, Copy)]
@@ -48,7 +48,7 @@ impl NonConstExpr {
             Self::Match(TryDesugar) => &[sym::const_try],
 
             // All other expressions are allowed.
-            Self::Loop(Loop | While) | Self::Match(Normal) => &[],
+            Self::Loop(Loop | While) | Self::Match(Normal | FormatArgs) => &[],
         };
 
         Some(gates)
@@ -124,7 +124,7 @@ impl<'tcx> CheckConstVisitor<'tcx> {
             // corresponding feature gate. This encourages nightly users to use feature gates when
             // possible.
             None if tcx.sess.opts.unstable_opts.unleash_the_miri_inside_of_you => {
-                tcx.sess.span_warn(span, "skipping const checks");
+                tcx.sess.emit_warning(SkippingConstChecks { span });
                 return;
             }
 
diff --git a/compiler/rustc_passes/src/dead.rs b/compiler/rustc_passes/src/dead.rs
index 94171b4b0..e2f858a34 100644
--- a/compiler/rustc_passes/src/dead.rs
+++ b/compiler/rustc_passes/src/dead.rs
@@ -259,7 +259,7 @@ impl<'tcx> MarkSymbolVisitor<'tcx> {
     /// for discussion).
     fn should_ignore_item(&mut self, def_id: DefId) -> bool {
         if let Some(impl_of) = self.tcx.impl_of_method(def_id) {
-            if !self.tcx.has_attr(impl_of, sym::automatically_derived) {
+            if !self.tcx.is_automatically_derived(impl_of) {
                 return false;
             }
 
@@ -315,7 +315,7 @@ impl<'tcx> MarkSymbolVisitor<'tcx> {
                     //// This is done to handle the case where, for example, the static
                     //// method of a private type is used, but the type itself is never
                     //// called directly.
-                    let self_ty = self.tcx.type_of(item);
+                    let self_ty = self.tcx.type_of(item).subst_identity();
                     match *self_ty.kind() {
                         ty::Adt(def, _) => self.check_def_id(def.did()),
                         ty::Foreign(did) => self.check_def_id(did),
@@ -395,6 +395,9 @@ impl<'tcx> Visitor<'tcx> for MarkSymbolVisitor<'tcx> {
                     self.mark_as_used_if_union(*adt, fields);
                 }
             }
+            hir::ExprKind::Closure(cls) => {
+                self.insert_def_id(cls.def_id.to_def_id());
+            }
             _ => (),
         }
 
@@ -451,47 +454,40 @@ impl<'tcx> Visitor<'tcx> for MarkSymbolVisitor<'tcx> {
         // referenced by it should be considered as used.
         let in_pat = mem::replace(&mut self.in_pat, false);
 
-        self.live_symbols.insert(self.tcx.hir().local_def_id(c.hir_id));
+        self.live_symbols.insert(c.def_id);
         intravisit::walk_anon_const(self, c);
 
         self.in_pat = in_pat;
     }
 }
 
-fn has_allow_dead_code_or_lang_attr_helper(
-    tcx: TyCtxt<'_>,
-    id: hir::HirId,
-    lint: &'static lint::Lint,
-) -> bool {
-    let attrs = tcx.hir().attrs(id);
-    if tcx.sess.contains_name(attrs, sym::lang) {
-        return true;
+fn has_allow_dead_code_or_lang_attr(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
+    fn has_lang_attr(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
+        tcx.has_attr(def_id.to_def_id(), sym::lang)
+            // Stable attribute for #[lang = "panic_impl"]
+            || tcx.has_attr(def_id.to_def_id(), sym::panic_handler)
     }
 
-    // Stable attribute for #[lang = "panic_impl"]
-    if tcx.sess.contains_name(attrs, sym::panic_handler) {
-        return true;
+    fn has_allow_dead_code(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
+        let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+        tcx.lint_level_at_node(lint::builtin::DEAD_CODE, hir_id).0 == lint::Allow
     }
 
-    let def_id = tcx.hir().local_def_id(id);
-    if tcx.def_kind(def_id).has_codegen_attrs() {
-        let cg_attrs = tcx.codegen_fn_attrs(def_id);
+    fn has_used_like_attr(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
+        tcx.def_kind(def_id).has_codegen_attrs() && {
+            let cg_attrs = tcx.codegen_fn_attrs(def_id);
 
-        // #[used], #[no_mangle], #[export_name], etc also keeps the item alive
-        // forcefully, e.g., for placing it in a specific section.
-        if cg_attrs.contains_extern_indicator()
-            || cg_attrs.flags.contains(CodegenFnAttrFlags::USED)
-            || cg_attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER)
-        {
-            return true;
+            // #[used], #[no_mangle], #[export_name], etc also keeps the item alive
+            // forcefully, e.g., for placing it in a specific section.
+            cg_attrs.contains_extern_indicator()
+                || cg_attrs.flags.contains(CodegenFnAttrFlags::USED)
+                || cg_attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER)
         }
     }
 
-    tcx.lint_level_at_node(lint, id).0 == lint::Allow
-}
-
-fn has_allow_dead_code_or_lang_attr(tcx: TyCtxt<'_>, id: hir::HirId) -> bool {
-    has_allow_dead_code_or_lang_attr_helper(tcx, id, lint::builtin::DEAD_CODE)
+    has_allow_dead_code(tcx, def_id)
+        || has_used_like_attr(tcx, def_id)
+        || has_lang_attr(tcx, def_id)
 }
 
 // These check_* functions seeds items that
@@ -513,7 +509,7 @@ fn check_item<'tcx>(
     struct_constructors: &mut FxHashMap<LocalDefId, LocalDefId>,
     id: hir::ItemId,
 ) {
-    let allow_dead_code = has_allow_dead_code_or_lang_attr(tcx, id.hir_id());
+    let allow_dead_code = has_allow_dead_code_or_lang_attr(tcx, id.owner_id.def_id);
     if allow_dead_code {
         worklist.push(id.owner_id.def_id);
     }
@@ -533,10 +529,8 @@ fn check_item<'tcx>(
                 }
             }
         }
-        DefKind::Impl => {
-            let of_trait = tcx.impl_trait_ref(id.owner_id);
-
-            if of_trait.is_some() {
+        DefKind::Impl { of_trait } => {
+            if of_trait {
                 worklist.push(id.owner_id.def_id);
             }
 
@@ -548,9 +542,7 @@ fn check_item<'tcx>(
 
             // And we access the Map here to get HirId from LocalDefId
             for id in local_def_ids {
-                if of_trait.is_some()
-                    || has_allow_dead_code_or_lang_attr(tcx, tcx.hir().local_def_id_to_hir_id(id))
-                {
+                if of_trait || has_allow_dead_code_or_lang_attr(tcx, id) {
                     worklist.push(id);
                 }
             }
@@ -558,9 +550,9 @@ fn check_item<'tcx>(
         DefKind::Struct => {
             let item = tcx.hir().item(id);
             if let hir::ItemKind::Struct(ref variant_data, _) = item.kind
-                && let Some(ctor_hir_id) = variant_data.ctor_hir_id()
+                && let Some(ctor_def_id) = variant_data.ctor_def_id()
             {
-                struct_constructors.insert(tcx.hir().local_def_id(ctor_hir_id), item.owner_id.def_id);
+                struct_constructors.insert(ctor_def_id, item.owner_id.def_id);
             }
         }
         DefKind::GlobalAsm => {
@@ -576,7 +568,7 @@ fn check_trait_item(tcx: TyCtxt<'_>, worklist: &mut Vec<LocalDefId>, id: hir::Tr
     if matches!(tcx.def_kind(id.owner_id), DefKind::AssocConst | DefKind::AssocFn) {
         let trait_item = tcx.hir().trait_item(id);
         if matches!(trait_item.kind, Const(_, Some(_)) | Fn(_, hir::TraitFn::Provided(_)))
-            && has_allow_dead_code_or_lang_attr(tcx, trait_item.hir_id())
+            && has_allow_dead_code_or_lang_attr(tcx, trait_item.owner_id.def_id)
         {
             worklist.push(trait_item.owner_id.def_id);
         }
@@ -585,7 +577,7 @@ fn check_trait_item(tcx: TyCtxt<'_>, worklist: &mut Vec<LocalDefId>, id: hir::Tr
 
 fn check_foreign_item(tcx: TyCtxt<'_>, worklist: &mut Vec<LocalDefId>, id: hir::ForeignItemId) {
     if matches!(tcx.def_kind(id.owner_id), DefKind::Static(_) | DefKind::Fn)
-        && has_allow_dead_code_or_lang_attr(tcx, id.hir_id())
+        && has_allow_dead_code_or_lang_attr(tcx, id.owner_id.def_id)
     {
         worklist.push(id.owner_id.def_id);
     }
@@ -665,7 +657,7 @@ impl<'tcx> DeadVisitor<'tcx> {
         if self.live_symbols.contains(&field.did.expect_local()) {
             return ShouldWarnAboutField::No;
         }
-        let field_type = self.tcx.type_of(field.did);
+        let field_type = self.tcx.type_of(field.did).subst_identity();
         if field_type.is_phantom_data() {
             return ShouldWarnAboutField::No;
         }
@@ -702,7 +694,7 @@ impl<'tcx> DeadVisitor<'tcx> {
             })
             .collect();
 
-        let descr = tcx.def_kind(first_id).descr(first_id.to_def_id());
+        let descr = tcx.def_descr(first_id.to_def_id());
         let num = dead_codes.len();
         let multiple = num > 6;
         let name_list = names.into();
@@ -714,7 +706,7 @@ impl<'tcx> DeadVisitor<'tcx> {
         };
 
         let parent_info = if let Some(parent_item) = parent_item {
-            let parent_descr = tcx.def_kind(parent_item).descr(parent_item.to_def_id());
+            let parent_descr = tcx.def_descr(parent_item.to_def_id());
             Some(ParentInfo {
                 num,
                 descr,
@@ -806,8 +798,7 @@ impl<'tcx> DeadVisitor<'tcx> {
         if self.live_symbols.contains(&def_id) {
             return;
         }
-        let hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id);
-        if has_allow_dead_code_or_lang_attr(self.tcx, hir_id) {
+        if has_allow_dead_code_or_lang_attr(self.tcx, def_id) {
             return;
         }
         let Some(name) = self.tcx.opt_item_name(def_id.to_def_id()) else {
diff --git a/compiler/rustc_passes/src/diagnostic_items.rs b/compiler/rustc_passes/src/diagnostic_items.rs
index 10ffa87ef..110eb210d 100644
--- a/compiler/rustc_passes/src/diagnostic_items.rs
+++ b/compiler/rustc_passes/src/diagnostic_items.rs
@@ -11,19 +11,19 @@
 
 use rustc_ast as ast;
 use rustc_hir::diagnostic_items::DiagnosticItems;
+use rustc_hir::OwnerId;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::TyCtxt;
-use rustc_span::def_id::{CrateNum, DefId, LocalDefId, LOCAL_CRATE};
-use rustc_span::symbol::{kw::Empty, sym, Symbol};
+use rustc_span::def_id::{CrateNum, DefId, LOCAL_CRATE};
+use rustc_span::symbol::{sym, Symbol};
 
-use crate::errors::{DuplicateDiagnosticItem, DuplicateDiagnosticItemInCrate};
+use crate::errors::DuplicateDiagnosticItemInCrate;
 
-fn observe_item(tcx: TyCtxt<'_>, diagnostic_items: &mut DiagnosticItems, def_id: LocalDefId) {
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
-    let attrs = tcx.hir().attrs(hir_id);
+fn observe_item<'tcx>(tcx: TyCtxt<'tcx>, diagnostic_items: &mut DiagnosticItems, owner: OwnerId) {
+    let attrs = tcx.hir().attrs(owner.into());
     if let Some(name) = extract(attrs) {
         // insert into our table
-        collect_item(tcx, diagnostic_items, name, def_id.to_def_id());
+        collect_item(tcx, diagnostic_items, name, owner.to_def_id());
     }
 }
 
@@ -31,26 +31,29 @@ fn collect_item(tcx: TyCtxt<'_>, items: &mut DiagnosticItems, name: Symbol, item
     items.id_to_name.insert(item_def_id, name);
     if let Some(original_def_id) = items.name_to_id.insert(name, item_def_id) {
         if original_def_id != item_def_id {
-            let orig_span = tcx.hir().span_if_local(original_def_id);
-            let orig_crate_name = if orig_span.is_some() {
-                None
-            } else {
-                Some(tcx.crate_name(original_def_id.krate))
-            };
-            match tcx.hir().span_if_local(item_def_id) {
-                Some(span) => tcx.sess.emit_err(DuplicateDiagnosticItem { span, name }),
-                None => tcx.sess.emit_err(DuplicateDiagnosticItemInCrate {
-                    span: orig_span,
-                    orig_crate_name: orig_crate_name.unwrap_or(Empty),
-                    have_orig_crate_name: orig_crate_name.map(|_| ()),
-                    crate_name: tcx.crate_name(item_def_id.krate),
-                    name,
-                }),
-            };
+            report_duplicate_item(tcx, name, original_def_id, item_def_id);
         }
     }
 }
 
+fn report_duplicate_item(
+    tcx: TyCtxt<'_>,
+    name: Symbol,
+    original_def_id: DefId,
+    item_def_id: DefId,
+) {
+    let orig_span = tcx.hir().span_if_local(original_def_id);
+    let duplicate_span = tcx.hir().span_if_local(item_def_id);
+    tcx.sess.emit_err(DuplicateDiagnosticItemInCrate {
+        duplicate_span,
+        orig_span,
+        crate_name: tcx.crate_name(item_def_id.krate),
+        orig_crate_name: tcx.crate_name(original_def_id.krate),
+        different_crates: (item_def_id.krate != original_def_id.krate).then_some(()),
+        name,
+    });
+}
+
 /// Extract the first `rustc_diagnostic_item = "$name"` out of a list of attributes.
 fn extract(attrs: &[ast::Attribute]) -> Option<Symbol> {
     attrs.iter().find_map(|attr| {
@@ -67,21 +70,8 @@ fn diagnostic_items(tcx: TyCtxt<'_>, cnum: CrateNum) -> DiagnosticItems {
 
     // Collect diagnostic items in this crate.
     let crate_items = tcx.hir_crate_items(());
-
-    for id in crate_items.items() {
-        observe_item(tcx, &mut diagnostic_items, id.owner_id.def_id);
-    }
-
-    for id in crate_items.trait_items() {
-        observe_item(tcx, &mut diagnostic_items, id.owner_id.def_id);
-    }
-
-    for id in crate_items.impl_items() {
-        observe_item(tcx, &mut diagnostic_items, id.owner_id.def_id);
-    }
-
-    for id in crate_items.foreign_items() {
-        observe_item(tcx, &mut diagnostic_items, id.owner_id.def_id);
+    for id in crate_items.owners() {
+        observe_item(tcx, &mut diagnostic_items, id);
     }
 
     diagnostic_items
diff --git a/compiler/rustc_passes/src/errors.rs b/compiler/rustc_passes/src/errors.rs
index 9c6519ea4..9f1c0b5a0 100644
--- a/compiler/rustc_passes/src/errors.rs
+++ b/compiler/rustc_passes/src/errors.rs
@@ -3,6 +3,7 @@ use std::{
     path::{Path, PathBuf},
 };
 
+use crate::fluent_generated as fluent;
 use rustc_ast::Label;
 use rustc_errors::{
     error_code, Applicability, DiagnosticSymbolList, ErrorGuaranteed, IntoDiagnostic, MultiSpan,
@@ -12,6 +13,7 @@ use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
 use rustc_middle::ty::{MainDefinition, Ty};
 use rustc_span::{Span, Symbol, DUMMY_SP};
 
+use crate::check_attr::ProcMacroKind;
 use crate::lang_items::Duplicate;
 
 #[derive(Diagnostic)]
@@ -260,7 +262,7 @@ pub struct DocKeywordConflict {
 pub struct DocInlineOnlyUse {
     #[label]
     pub attr_span: Span,
-    #[label(not_a_use_item_label)]
+    #[label(passes_not_a_use_item_label)]
     pub item_span: Option<Span>,
 }
 
@@ -299,7 +301,7 @@ pub struct DocTestUnknownAny {
 #[derive(LintDiagnostic)]
 #[diag(passes_doc_test_unknown_spotlight)]
 #[note]
-#[note(no_op_note)]
+#[note(passes_no_op_note)]
 pub struct DocTestUnknownSpotlight {
     pub path: String,
     #[suggestion(style = "short", applicability = "machine-applicable", code = "notable_trait")]
@@ -347,6 +349,34 @@ pub struct HasIncoherentInherentImpl {
     pub span: Span,
 }
 
+#[derive(Diagnostic)]
+#[diag(passes_both_ffi_const_and_pure, code = "E0757")]
+pub struct BothFfiConstAndPure {
+    #[primary_span]
+    pub attr_span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(passes_ffi_pure_invalid_target, code = "E0755")]
+pub struct FfiPureInvalidTarget {
+    #[primary_span]
+    pub attr_span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(passes_ffi_const_invalid_target, code = "E0756")]
+pub struct FfiConstInvalidTarget {
+    #[primary_span]
+    pub attr_span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(passes_ffi_returns_twice_invalid_target, code = "E0724")]
+pub struct FfiReturnsTwiceInvalidTarget {
+    #[primary_span]
+    pub attr_span: Span,
+}
+
 #[derive(LintDiagnostic)]
 #[diag(passes_must_use_async)]
 pub struct MustUseAsync {
@@ -544,9 +574,9 @@ pub struct DebugVisualizerPlacement {
 
 #[derive(Diagnostic)]
 #[diag(passes_debug_visualizer_invalid)]
-#[note(note_1)]
-#[note(note_2)]
-#[note(note_3)]
+#[note(passes_note_1)]
+#[note(passes_note_2)]
+#[note(passes_note_3)]
 pub struct DebugVisualizerInvalid {
     #[primary_span]
     pub span: Span,
@@ -611,8 +641,16 @@ pub struct MacroUse {
 }
 
 #[derive(LintDiagnostic)]
-#[diag(passes_macro_export)]
-pub struct MacroExport;
+pub enum MacroExport {
+    #[diag(passes_macro_export)]
+    Normal,
+
+    #[diag(passes_invalid_macro_export_arguments)]
+    UnknownItem { name: Symbol },
+
+    #[diag(passes_invalid_macro_export_arguments_too_many_items)]
+    TooManyItems,
+}
 
 #[derive(LintDiagnostic)]
 #[diag(passes_plugin_registrar)]
@@ -753,7 +791,7 @@ impl IntoDiagnostic<'_> for InvalidAttrAtCrateLevel {
         self,
         handler: &'_ rustc_errors::Handler,
     ) -> rustc_errors::DiagnosticBuilder<'_, ErrorGuaranteed> {
-        let mut diag = handler.struct_err(rustc_errors::fluent::passes_invalid_attr_at_crate_level);
+        let mut diag = handler.struct_err(fluent::passes_invalid_attr_at_crate_level);
         diag.set_span(self.span);
         diag.set_arg("name", self.name);
         // Only emit an error with a suggestion if we can create a string out
@@ -762,7 +800,7 @@ impl IntoDiagnostic<'_> for InvalidAttrAtCrateLevel {
             let replacement = src.replace("#!", "#");
             diag.span_suggestion_verbose(
                 self.span,
-                rustc_errors::fluent::suggestion,
+                fluent::passes_suggestion,
                 replacement,
                 rustc_errors::Applicability::MachineApplicable,
             );
@@ -772,22 +810,16 @@ impl IntoDiagnostic<'_> for InvalidAttrAtCrateLevel {
 }
 
 #[derive(Diagnostic)]
-#[diag(passes_duplicate_diagnostic_item)]
-pub struct DuplicateDiagnosticItem {
-    #[primary_span]
-    pub span: Span,
-    pub name: Symbol,
-}
-
-#[derive(Diagnostic)]
 #[diag(passes_duplicate_diagnostic_item_in_crate)]
 pub struct DuplicateDiagnosticItemInCrate {
+    #[primary_span]
+    pub duplicate_span: Option<Span>,
     #[note(passes_diagnostic_item_first_defined)]
-    pub span: Option<Span>,
-    pub orig_crate_name: Symbol,
+    pub orig_span: Option<Span>,
     #[note]
-    pub have_orig_crate_name: Option<()>,
+    pub different_crates: Option<()>,
     pub crate_name: Symbol,
+    pub orig_crate_name: Symbol,
     pub name: Symbol,
 }
 
@@ -888,17 +920,17 @@ impl<'a> IntoDiagnostic<'_> for BreakNonLoop<'a> {
     ) -> rustc_errors::DiagnosticBuilder<'_, ErrorGuaranteed> {
         let mut diag = handler.struct_span_err_with_code(
             self.span,
-            rustc_errors::fluent::passes_break_non_loop,
+            fluent::passes_break_non_loop,
             error_code!(E0571),
         );
         diag.set_arg("kind", self.kind);
-        diag.span_label(self.span, rustc_errors::fluent::label);
+        diag.span_label(self.span, fluent::passes_label);
         if let Some(head) = self.head {
-            diag.span_label(head, rustc_errors::fluent::label2);
+            diag.span_label(head, fluent::passes_label2);
         }
         diag.span_suggestion(
             self.span,
-            rustc_errors::fluent::suggestion,
+            fluent::passes_suggestion,
             self.suggestion,
             Applicability::MaybeIncorrect,
         );
@@ -916,7 +948,7 @@ impl<'a> IntoDiagnostic<'_> for BreakNonLoop<'a> {
                 _ => {
                     diag.span_suggestion(
                         self.break_expr_span,
-                        rustc_errors::fluent::break_expr_suggestion,
+                        fluent::passes_break_expr_suggestion,
                         label.ident,
                         Applicability::MaybeIncorrect,
                     );
@@ -933,7 +965,7 @@ pub struct ContinueLabeledBlock {
     #[primary_span]
     #[label]
     pub span: Span,
-    #[label(block_label)]
+    #[label(passes_block_label)]
     pub block_span: Span,
 }
 
@@ -943,7 +975,7 @@ pub struct BreakInsideClosure<'a> {
     #[primary_span]
     #[label]
     pub span: Span,
-    #[label(closure_label)]
+    #[label(passes_closure_label)]
     pub closure_span: Span,
     pub name: &'a str,
 }
@@ -954,7 +986,7 @@ pub struct BreakInsideAsyncBlock<'a> {
     #[primary_span]
     #[label]
     pub span: Span,
-    #[label(async_block_label)]
+    #[label(passes_async_block_label)]
     pub closure_span: Span,
     pub name: &'a str,
 }
@@ -1027,14 +1059,14 @@ impl IntoDiagnostic<'_> for NakedFunctionsAsmBlock {
     ) -> rustc_errors::DiagnosticBuilder<'_, ErrorGuaranteed> {
         let mut diag = handler.struct_span_err_with_code(
             self.span,
-            rustc_errors::fluent::passes_naked_functions_asm_block,
+            fluent::passes_naked_functions_asm_block,
             error_code!(E0787),
         );
         for span in self.multiple_asms.iter() {
-            diag.span_label(*span, rustc_errors::fluent::label_multiple_asm);
+            diag.span_label(*span, fluent::passes_label_multiple_asm);
         }
         for span in self.non_asms.iter() {
-            diag.span_label(*span, rustc_errors::fluent::label_non_asm);
+            diag.span_label(*span, fluent::passes_label_non_asm);
         }
         diag
     }
@@ -1093,9 +1125,9 @@ pub struct AttrOnlyInFunctions {
 pub struct MultipleRustcMain {
     #[primary_span]
     pub span: Span,
-    #[label(first)]
+    #[label(passes_first)]
     pub first: Span,
-    #[label(additional)]
+    #[label(passes_additional)]
     pub additional: Span,
 }
 
@@ -1106,7 +1138,7 @@ pub struct MultipleStartFunctions {
     pub span: Span,
     #[label]
     pub labeled: Span,
-    #[label(previous)]
+    #[label(passes_previous)]
     pub previous: Span,
 }
 
@@ -1151,7 +1183,7 @@ impl<'a> IntoDiagnostic<'a> for NoMainErr {
     ) -> rustc_errors::DiagnosticBuilder<'a, ErrorGuaranteed> {
         let mut diag = handler.struct_span_err_with_code(
             DUMMY_SP,
-            rustc_errors::fluent::passes_no_main_function,
+            fluent::passes_no_main_function,
             error_code!(E0601),
         );
         diag.set_arg("crate_name", self.crate_name);
@@ -1159,16 +1191,16 @@ impl<'a> IntoDiagnostic<'a> for NoMainErr {
         diag.set_arg("has_filename", self.has_filename);
         let note = if !self.non_main_fns.is_empty() {
             for &span in &self.non_main_fns {
-                diag.span_note(span, rustc_errors::fluent::here_is_main);
+                diag.span_note(span, fluent::passes_here_is_main);
             }
-            diag.note(rustc_errors::fluent::one_or_more_possible_main);
-            diag.help(rustc_errors::fluent::consider_moving_main);
+            diag.note(fluent::passes_one_or_more_possible_main);
+            diag.help(fluent::passes_consider_moving_main);
             // There were some functions named `main` though. Try to give the user a hint.
-            rustc_errors::fluent::main_must_be_defined_at_crate
+            fluent::passes_main_must_be_defined_at_crate
         } else if self.has_filename {
-            rustc_errors::fluent::consider_adding_main_to_file
+            fluent::passes_consider_adding_main_to_file
         } else {
-            rustc_errors::fluent::consider_adding_main_at_crate
+            fluent::passes_consider_adding_main_at_crate
         };
         if self.file_empty {
             diag.note(note);
@@ -1179,11 +1211,11 @@ impl<'a> IntoDiagnostic<'a> for NoMainErr {
 
         if let Some(main_def) = self.main_def_opt && main_def.opt_fn_def_id().is_none(){
             // There is something at `crate::main`, but it is not a function definition.
-            diag.span_label(main_def.span, rustc_errors::fluent::non_function_main);
+            diag.span_label(main_def.span, fluent::passes_non_function_main);
         }
 
         if self.add_teach_note {
-            diag.note(rustc_errors::fluent::teach_note);
+            diag.note(fluent::passes_teach_note);
         }
         diag
     }
@@ -1212,12 +1244,9 @@ impl IntoDiagnostic<'_> for DuplicateLangItem {
     ) -> rustc_errors::DiagnosticBuilder<'_, ErrorGuaranteed> {
         let mut diag = handler.struct_err_with_code(
             match self.duplicate {
-                Duplicate::Plain => rustc_errors::fluent::passes_duplicate_lang_item,
-
-                Duplicate::Crate => rustc_errors::fluent::passes_duplicate_lang_item_crate,
-                Duplicate::CrateDepends => {
-                    rustc_errors::fluent::passes_duplicate_lang_item_crate_depends
-                }
+                Duplicate::Plain => fluent::passes_duplicate_lang_item,
+                Duplicate::Crate => fluent::passes_duplicate_lang_item_crate,
+                Duplicate::CrateDepends => fluent::passes_duplicate_lang_item_crate_depends,
             },
             error_code!(E0152),
         );
@@ -1232,24 +1261,24 @@ impl IntoDiagnostic<'_> for DuplicateLangItem {
             diag.set_span(span);
         }
         if let Some(span) = self.first_defined_span {
-            diag.span_note(span, rustc_errors::fluent::first_defined_span);
+            diag.span_note(span, fluent::passes_first_defined_span);
         } else {
             if self.orig_dependency_of.is_empty() {
-                diag.note(rustc_errors::fluent::first_defined_crate);
+                diag.note(fluent::passes_first_defined_crate);
             } else {
-                diag.note(rustc_errors::fluent::first_defined_crate_depends);
+                diag.note(fluent::passes_first_defined_crate_depends);
             }
 
             if self.orig_is_local {
-                diag.note(rustc_errors::fluent::first_definition_local);
+                diag.note(fluent::passes_first_definition_local);
             } else {
-                diag.note(rustc_errors::fluent::first_definition_path);
+                diag.note(fluent::passes_first_definition_path);
             }
 
             if self.is_local {
-                diag.note(rustc_errors::fluent::second_definition_local);
+                diag.note(fluent::passes_second_definition_local);
             } else {
-                diag.note(rustc_errors::fluent::second_definition_path);
+                diag.note(fluent::passes_second_definition_path);
             }
         }
         diag
@@ -1360,7 +1389,7 @@ pub struct UselessStability {
     #[primary_span]
     #[label]
     pub span: Span,
-    #[label(item)]
+    #[label(passes_item)]
     pub item_sp: Span,
 }
 
@@ -1370,7 +1399,7 @@ pub struct InvalidStability {
     #[primary_span]
     #[label]
     pub span: Span,
-    #[label(item)]
+    #[label(passes_item)]
     pub item_sp: Span,
 }
 
@@ -1380,7 +1409,7 @@ pub struct CannotStabilizeDeprecated {
     #[primary_span]
     #[label]
     pub span: Span,
-    #[label(item)]
+    #[label(passes_item)]
     pub item_sp: Span,
 }
 
@@ -1390,7 +1419,7 @@ pub struct InvalidDeprecationVersion {
     #[primary_span]
     #[label]
     pub span: Span,
-    #[label(item)]
+    #[label(passes_item)]
     pub item_sp: Span,
 }
 
@@ -1515,3 +1544,59 @@ pub struct ChangeFieldsToBeOfUnitType {
     #[suggestion_part(code = "()")]
     pub spans: Vec<Span>,
 }
+
+#[derive(Diagnostic)]
+#[diag(passes_proc_macro_typeerror)]
+#[note]
+pub(crate) struct ProcMacroTypeError<'tcx> {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    pub found: Ty<'tcx>,
+    pub kind: ProcMacroKind,
+    pub expected_signature: &'static str,
+}
+
+#[derive(Diagnostic)]
+#[diag(passes_proc_macro_diff_arg_count)]
+pub(crate) struct ProcMacroDiffArguments {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    pub count: usize,
+    pub kind: ProcMacroKind,
+    pub expected_signature: &'static str,
+}
+
+#[derive(Diagnostic)]
+#[diag(passes_proc_macro_missing_args)]
+pub(crate) struct ProcMacroMissingArguments {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    pub expected_input_count: usize,
+    pub kind: ProcMacroKind,
+    pub expected_signature: &'static str,
+}
+
+#[derive(Diagnostic)]
+#[diag(passes_proc_macro_invalid_abi)]
+pub(crate) struct ProcMacroInvalidAbi {
+    #[primary_span]
+    pub span: Span,
+    pub abi: &'static str,
+}
+
+#[derive(Diagnostic)]
+#[diag(passes_proc_macro_unsafe)]
+pub(crate) struct ProcMacroUnsafe {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(passes_skipping_const_checks)]
+pub struct SkippingConstChecks {
+    #[primary_span]
+    pub span: Span,
+}
diff --git a/compiler/rustc_passes/src/hir_id_validator.rs b/compiler/rustc_passes/src/hir_id_validator.rs
index d143adb2e..de0e50a65 100644
--- a/compiler/rustc_passes/src/hir_id_validator.rs
+++ b/compiler/rustc_passes/src/hir_id_validator.rs
@@ -74,37 +74,26 @@ impl<'a, 'hir> HirIdValidator<'a, 'hir> {
             .expect("owning item has no entry");
 
         if max != self.hir_ids_seen.len() - 1 {
-            // Collect the missing ItemLocalIds
-            let missing: Vec<_> = (0..=max as u32)
-                .filter(|&i| !self.hir_ids_seen.contains(ItemLocalId::from_u32(i)))
-                .collect();
-
-            // Try to map those to something more useful
-            let mut missing_items = Vec::with_capacity(missing.len());
+            let hir = self.tcx.hir();
+            let pretty_owner = hir.def_path(owner.def_id).to_string_no_crate_verbose();
 
-            for local_id in missing {
-                let hir_id = HirId { owner, local_id: ItemLocalId::from_u32(local_id) };
+            let missing_items: Vec<_> = (0..=max as u32)
+                .map(|i| ItemLocalId::from_u32(i))
+                .filter(|&local_id| !self.hir_ids_seen.contains(local_id))
+                .map(|local_id| hir.node_to_string(HirId { owner, local_id }))
+                .collect();
 
-                trace!("missing hir id {:#?}", hir_id);
+            let seen_items: Vec<_> = self
+                .hir_ids_seen
+                .iter()
+                .map(|local_id| hir.node_to_string(HirId { owner, local_id }))
+                .collect();
 
-                missing_items.push(format!(
-                    "[local_id: {}, owner: {}]",
-                    local_id,
-                    self.tcx.hir().def_path(owner.def_id).to_string_no_crate_verbose()
-                ));
-            }
             self.error(|| {
                 format!(
                     "ItemLocalIds not assigned densely in {}. \
-                Max ItemLocalId = {}, missing IDs = {:#?}; seens IDs = {:#?}",
-                    self.tcx.hir().def_path(owner.def_id).to_string_no_crate_verbose(),
-                    max,
-                    missing_items,
-                    self.hir_ids_seen
-                        .iter()
-                        .map(|local_id| HirId { owner, local_id })
-                        .map(|h| format!("({:?} {})", h, self.tcx.hir().node_to_string(h)))
-                        .collect::<Vec<_>>()
+                Max ItemLocalId = {}, missing IDs = {:#?}; seen IDs = {:#?}",
+                    pretty_owner, max, missing_items, seen_items
                 )
             });
         }
diff --git a/compiler/rustc_passes/src/hir_stats.rs b/compiler/rustc_passes/src/hir_stats.rs
index b86d23168..5e2d2d3e5 100644
--- a/compiler/rustc_passes/src/hir_stats.rs
+++ b/compiler/rustc_passes/src/hir_stats.rs
@@ -12,6 +12,7 @@ use rustc_hir::HirId;
 use rustc_middle::hir::map::Map;
 use rustc_middle::ty::TyCtxt;
 use rustc_middle::util::common::to_readable_str;
+use rustc_span::def_id::LocalDefId;
 use rustc_span::Span;
 
 #[derive(Copy, Clone, PartialEq, Eq, Hash)]
@@ -363,7 +364,7 @@ impl<'v> hir_visit::Visitor<'v> for StatCollector<'v> {
         fd: &'v hir::FnDecl<'v>,
         b: hir::BodyId,
         _: Span,
-        id: hir::HirId,
+        id: LocalDefId,
     ) {
         self.record("FnDecl", Id::None, fd);
         hir_visit::walk_fn(self, fk, fd, b, id)
@@ -567,7 +568,7 @@ impl<'v> ast_visit::Visitor<'v> for StatCollector<'v> {
                 Box, Array, ConstBlock, Call, MethodCall, Tup, Binary, Unary, Lit, Cast, Type, Let,
                 If, While, ForLoop, Loop, Match, Closure, Block, Async, Await, TryBlock, Assign,
                 AssignOp, Field, Index, Range, Underscore, Path, AddrOf, Break, Continue, Ret,
-                InlineAsm, MacCall, Struct, Repeat, Paren, Try, Yield, Yeet, IncludedBytes, Err
+                InlineAsm, FormatArgs, MacCall, Struct, Repeat, Paren, Try, Yield, Yeet, IncludedBytes, Err
             ]
         );
         ast_visit::walk_expr(self, e)
@@ -645,7 +646,7 @@ impl<'v> ast_visit::Visitor<'v> for StatCollector<'v> {
     }
 
     // `UseTree` has one inline use (in `ast::ItemKind::Use`) and one
-    // non-inline use (in `ast::UseTreeKind::Nested). The former case is more
+    // non-inline use (in `ast::UseTreeKind::Nested`). The former case is more
     // common, so we don't implement `visit_use_tree` and tolerate the missed
     // coverage in the latter case.
 
diff --git a/compiler/rustc_passes/src/lang_items.rs b/compiler/rustc_passes/src/lang_items.rs
index 9a40b847d..fdd0e5dab 100644
--- a/compiler/rustc_passes/src/lang_items.rs
+++ b/compiler/rustc_passes/src/lang_items.rs
@@ -15,9 +15,9 @@ use crate::weak_lang_items;
 
 use rustc_hir as hir;
 use rustc_hir::def::DefKind;
-use rustc_hir::def_id::DefId;
+use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::lang_items::{extract, GenericRequirement};
-use rustc_hir::{HirId, LangItem, LanguageItems, Target};
+use rustc_hir::{LangItem, LanguageItems, Target};
 use rustc_middle::ty::TyCtxt;
 use rustc_session::cstore::ExternCrate;
 use rustc_span::{symbol::kw::Empty, Span};
@@ -40,13 +40,13 @@ impl<'tcx> LanguageItemCollector<'tcx> {
         LanguageItemCollector { tcx, items: LanguageItems::new() }
     }
 
-    fn check_for_lang(&mut self, actual_target: Target, hir_id: HirId) {
-        let attrs = self.tcx.hir().attrs(hir_id);
+    fn check_for_lang(&mut self, actual_target: Target, def_id: LocalDefId) {
+        let attrs = self.tcx.hir().attrs(self.tcx.hir().local_def_id_to_hir_id(def_id));
         if let Some((name, span)) = extract(&attrs) {
             match LangItem::from_name(name) {
                 // Known lang item with attribute on correct target.
                 Some(lang_item) if actual_target == lang_item.target() => {
-                    self.collect_item_extended(lang_item, hir_id, span);
+                    self.collect_item_extended(lang_item, def_id, span);
                 }
                 // Known lang item with attribute on incorrect target.
                 Some(lang_item) => {
@@ -142,8 +142,7 @@ impl<'tcx> LanguageItemCollector<'tcx> {
 
     // Like collect_item() above, but also checks whether the lang item is declared
     // with the right number of generic arguments.
-    fn collect_item_extended(&mut self, lang_item: LangItem, hir_id: HirId, span: Span) {
-        let item_def_id = self.tcx.hir().local_def_id(hir_id).to_def_id();
+    fn collect_item_extended(&mut self, lang_item: LangItem, item_def_id: LocalDefId, span: Span) {
         let name = lang_item.name();
 
         // Now check whether the lang_item has the expected number of generic
@@ -154,7 +153,8 @@ impl<'tcx> LanguageItemCollector<'tcx> {
         // Some other types like Box and various functions like drop_in_place
         // have minimum requirements.
 
-        if let hir::Node::Item(hir::Item { kind, span: item_span, .. }) = self.tcx.hir().get(hir_id)
+        if let hir::Node::Item(hir::Item { kind, span: item_span, .. }) =
+            self.tcx.hir().get_by_def_id(item_def_id)
         {
             let (actual_num, generics_span) = match kind.generics() {
                 Some(generics) => (generics.params.len(), generics.span),
@@ -191,7 +191,7 @@ impl<'tcx> LanguageItemCollector<'tcx> {
             }
         }
 
-        self.collect_item(lang_item, item_def_id);
+        self.collect_item(lang_item, item_def_id.to_def_id());
     }
 }
 
@@ -211,13 +211,14 @@ fn get_lang_items(tcx: TyCtxt<'_>, (): ()) -> LanguageItems {
     let crate_items = tcx.hir_crate_items(());
 
     for id in crate_items.items() {
-        collector.check_for_lang(Target::from_def_kind(tcx.def_kind(id.owner_id)), id.hir_id());
+        collector
+            .check_for_lang(Target::from_def_kind(tcx.def_kind(id.owner_id)), id.owner_id.def_id);
 
         if matches!(tcx.def_kind(id.owner_id), DefKind::Enum) {
             let item = tcx.hir().item(id);
             if let hir::ItemKind::Enum(def, ..) = &item.kind {
                 for variant in def.variants {
-                    collector.check_for_lang(Target::Variant, variant.hir_id);
+                    collector.check_for_lang(Target::Variant, variant.def_id);
                 }
             }
         }
@@ -226,13 +227,13 @@ fn get_lang_items(tcx: TyCtxt<'_>, (): ()) -> LanguageItems {
     // FIXME: avoid calling trait_item() when possible
     for id in crate_items.trait_items() {
         let item = tcx.hir().trait_item(id);
-        collector.check_for_lang(Target::from_trait_item(item), item.hir_id())
+        collector.check_for_lang(Target::from_trait_item(item), item.owner_id.def_id)
     }
 
     // FIXME: avoid calling impl_item() when possible
     for id in crate_items.impl_items() {
         let item = tcx.hir().impl_item(id);
-        collector.check_for_lang(target_from_impl_item(tcx, item), item.hir_id())
+        collector.check_for_lang(target_from_impl_item(tcx, item), item.owner_id.def_id)
     }
 
     // Extract out the found lang items.
diff --git a/compiler/rustc_passes/src/layout_test.rs b/compiler/rustc_passes/src/layout_test.rs
index 827d86780..047b9b525 100644
--- a/compiler/rustc_passes/src/layout_test.rs
+++ b/compiler/rustc_passes/src/layout_test.rs
@@ -29,7 +29,7 @@ pub fn test_layout(tcx: TyCtxt<'_>) {
 fn dump_layout_of(tcx: TyCtxt<'_>, item_def_id: LocalDefId, attr: &Attribute) {
     let tcx = tcx;
     let param_env = tcx.param_env(item_def_id);
-    let ty = tcx.type_of(item_def_id);
+    let ty = tcx.type_of(item_def_id).subst_identity();
     match tcx.layout_of(param_env.and(ty)) {
         Ok(ty_layout) => {
             // Check out the `#[rustc_layout(..)]` attribute to tell what to dump.
diff --git a/compiler/rustc_passes/src/lib.rs b/compiler/rustc_passes/src/lib.rs
index 6e621b7eb..0cb842408 100644
--- a/compiler/rustc_passes/src/lib.rs
+++ b/compiler/rustc_passes/src/lib.rs
@@ -18,6 +18,8 @@ extern crate rustc_middle;
 #[macro_use]
 extern crate tracing;
 
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
+use rustc_macros::fluent_messages;
 use rustc_middle::ty::query::Providers;
 
 mod check_attr;
@@ -40,6 +42,8 @@ pub mod stability;
 mod upvars;
 mod weak_lang_items;
 
+fluent_messages! { "../locales/en-US.ftl" }
+
 pub fn provide(providers: &mut Providers) {
     check_attr::provide(providers);
     check_const::provide(providers);
diff --git a/compiler/rustc_passes/src/lib_features.rs b/compiler/rustc_passes/src/lib_features.rs
index 4c6a9b23f..f4da1aaec 100644
--- a/compiler/rustc_passes/src/lib_features.rs
+++ b/compiler/rustc_passes/src/lib_features.rs
@@ -4,7 +4,7 @@
 //! but are not declared in one single location (unlike lang features), which means we need to
 //! collect them instead.
 
-use rustc_ast::{Attribute, MetaItemKind};
+use rustc_ast::Attribute;
 use rustc_attr::{rust_version_symbol, VERSION_PLACEHOLDER};
 use rustc_hir::intravisit::Visitor;
 use rustc_middle::hir::nested_filter;
@@ -42,8 +42,7 @@ impl<'tcx> LibFeatureCollector<'tcx> {
         // Find a stability attribute: one of #[stable(…)], #[unstable(…)],
         // #[rustc_const_stable(…)], #[rustc_const_unstable(…)] or #[rustc_default_body_unstable].
         if let Some(stab_attr) = stab_attrs.iter().find(|stab_attr| attr.has_name(**stab_attr)) {
-            let meta_kind = attr.meta_kind();
-            if let Some(MetaItemKind::List(ref metas)) = meta_kind {
+            if let Some(metas) = attr.meta_item_list() {
                 let mut feature = None;
                 let mut since = None;
                 for meta in metas {
diff --git a/compiler/rustc_passes/src/liveness.rs b/compiler/rustc_passes/src/liveness.rs
index 6afdcc37f..df5c8f53e 100644
--- a/compiler/rustc_passes/src/liveness.rs
+++ b/compiler/rustc_passes/src/liveness.rs
@@ -145,7 +145,7 @@ fn check_liveness(tcx: TyCtxt<'_>, def_id: DefId) {
 
     // Don't run unused pass for #[derive()]
     let parent = tcx.local_parent(local_def_id);
-    if let DefKind::Impl = tcx.def_kind(parent)
+    if let DefKind::Impl { .. } = tcx.def_kind(parent)
         && tcx.has_attr(parent.to_def_id(), sym::automatically_derived)
     {
         return;
@@ -475,7 +475,7 @@ impl<'tcx> Visitor<'tcx> for IrMaps<'tcx> {
             | hir::ExprKind::InlineAsm(..)
             | hir::ExprKind::Box(..)
             | hir::ExprKind::Type(..)
-            | hir::ExprKind::Err
+            | hir::ExprKind::Err(_)
             | hir::ExprKind::Path(hir::QPath::TypeRelative(..))
             | hir::ExprKind::Path(hir::QPath::LangItem(..)) => {
                 intravisit::walk_expr(self, expr);
@@ -1129,7 +1129,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
 
             hir::ExprKind::Lit(..)
             | hir::ExprKind::ConstBlock(..)
-            | hir::ExprKind::Err
+            | hir::ExprKind::Err(_)
             | hir::ExprKind::Path(hir::QPath::TypeRelative(..))
             | hir::ExprKind::Path(hir::QPath::LangItem(..)) => succ,
 
@@ -1427,7 +1427,7 @@ fn check_expr<'tcx>(this: &mut Liveness<'_, 'tcx>, expr: &'tcx Expr<'tcx>) {
         | hir::ExprKind::Yield(..)
         | hir::ExprKind::Box(..)
         | hir::ExprKind::Type(..)
-        | hir::ExprKind::Err => {}
+        | hir::ExprKind::Err(_) => {}
     }
 }
 
diff --git a/compiler/rustc_passes/src/liveness/rwu_table.rs b/compiler/rustc_passes/src/liveness/rwu_table.rs
index 6d5983f53..053bf5c23 100644
--- a/compiler/rustc_passes/src/liveness/rwu_table.rs
+++ b/compiler/rustc_passes/src/liveness/rwu_table.rs
@@ -9,7 +9,7 @@ pub(super) struct RWU {
 }
 
 /// Conceptually, this is like a `Vec<Vec<RWU>>`. But the number of
-/// RWU`s can get very large, so it uses a more compact representation.
+/// RWU's can get very large, so it uses a more compact representation.
 pub(super) struct RWUTable {
     /// Total number of live nodes.
     live_nodes: usize,
diff --git a/compiler/rustc_passes/src/naked_functions.rs b/compiler/rustc_passes/src/naked_functions.rs
index acc54e7e1..c5b5cf7f5 100644
--- a/compiler/rustc_passes/src/naked_functions.rs
+++ b/compiler/rustc_passes/src/naked_functions.rs
@@ -219,7 +219,7 @@ impl<'tcx> CheckInlineAssembly<'tcx> {
                 hir::intravisit::walk_expr(self, expr);
             }
 
-            ExprKind::Err => {
+            ExprKind::Err(_) => {
                 self.items.push((ItemKind::Err, span));
             }
         }
diff --git a/compiler/rustc_passes/src/reachable.rs b/compiler/rustc_passes/src/reachable.rs
index ad0952203..051100c56 100644
--- a/compiler/rustc_passes/src/reachable.rs
+++ b/compiler/rustc_passes/src/reachable.rs
@@ -320,31 +320,28 @@ fn check_item<'tcx>(
         worklist.push(id.owner_id.def_id);
     }
 
-    if !matches!(tcx.def_kind(id.owner_id), DefKind::Impl) {
+    if !matches!(tcx.def_kind(id.owner_id), DefKind::Impl { of_trait: true }) {
         return;
     }
 
     // We need only trait impls here, not inherent impls, and only non-exported ones
-    let item = tcx.hir().item(id);
-    if let hir::ItemKind::Impl(hir::Impl { of_trait: Some(ref trait_ref), ref items, .. }) =
-        item.kind
-    {
-        if !effective_visibilities.is_reachable(item.owner_id.def_id) {
-            worklist.extend(items.iter().map(|ii_ref| ii_ref.id.owner_id.def_id));
+    if effective_visibilities.is_reachable(id.owner_id.def_id) {
+        return;
+    }
 
-            let Res::Def(DefKind::Trait, trait_def_id) = trait_ref.path.res else {
-                unreachable!();
-            };
+    let items = tcx.associated_item_def_ids(id.owner_id);
+    worklist.extend(items.iter().map(|ii_ref| ii_ref.expect_local()));
 
-            if !trait_def_id.is_local() {
-                return;
-            }
+    let Some(trait_def_id) = tcx.trait_id_of_impl(id.owner_id.to_def_id()) else {
+        unreachable!();
+    };
 
-            worklist.extend(
-                tcx.provided_trait_methods(trait_def_id).map(|assoc| assoc.def_id.expect_local()),
-            );
-        }
+    if !trait_def_id.is_local() {
+        return;
     }
+
+    worklist
+        .extend(tcx.provided_trait_methods(trait_def_id).map(|assoc| assoc.def_id.expect_local()));
 }
 
 fn has_custom_linkage(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
diff --git a/compiler/rustc_passes/src/stability.rs b/compiler/rustc_passes/src/stability.rs
index 34e1abb78..16194a6f1 100644
--- a/compiler/rustc_passes/src/stability.rs
+++ b/compiler/rustc_passes/src/stability.rs
@@ -1,12 +1,7 @@
 //! A pass that annotates every item and method with its stability level,
 //! propagating default levels lexically from parent to children ast nodes.
 
-use crate::errors::{
-    self, CannotStabilizeDeprecated, DeprecatedAttribute, DuplicateFeatureErr,
-    FeatureOnlyOnNightly, ImpliedFeatureNotExist, InvalidDeprecationVersion, InvalidStability,
-    MissingConstErr, MissingConstStabAttr, MissingStabilityAttr, TraitImplConstStable,
-    UnknownFeature, UselessStability,
-};
+use crate::errors;
 use rustc_attr::{
     self as attr, rust_version_symbol, ConstStability, Stability, StabilityLevel, Unstable,
     UnstableReason, VERSION_PLACEHOLDER,
@@ -125,7 +120,7 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
         if let Some((depr, span)) = &depr {
             is_deprecated = true;
 
-            if kind == AnnotationKind::Prohibited || kind == AnnotationKind::DeprecationProhibited {
+            if matches!(kind, AnnotationKind::Prohibited | AnnotationKind::DeprecationProhibited) {
                 let hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id);
                 self.tcx.emit_spanned_lint(
                     USELESS_DEPRECATED,
@@ -185,7 +180,7 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
                 {
                     self.tcx
                         .sess
-                        .emit_err(MissingConstErr { fn_sig_span: fn_sig.span, const_span });
+                        .emit_err(errors::MissingConstErr { fn_sig_span: fn_sig.span, const_span });
                 }
             }
         }
@@ -203,7 +198,7 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
 
         if let Some((rustc_attr::Deprecation { is_since_rustc_version: true, .. }, span)) = &depr {
             if stab.is_none() {
-                self.tcx.sess.emit_err(DeprecatedAttribute { span: *span });
+                self.tcx.sess.emit_err(errors::DeprecatedAttribute { span: *span });
             }
         }
 
@@ -219,7 +214,7 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
             if kind == AnnotationKind::Prohibited
                 || (kind == AnnotationKind::Container && stab.level.is_stable() && is_deprecated)
             {
-                self.tcx.sess.emit_err(UselessStability { span, item_sp });
+                self.tcx.sess.emit_err(errors::UselessStability { span, item_sp });
             }
 
             debug!("annotate: found {:?}", stab);
@@ -235,15 +230,16 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
                 {
                     match stab_v.parse::<u64>() {
                         Err(_) => {
-                            self.tcx.sess.emit_err(InvalidStability { span, item_sp });
+                            self.tcx.sess.emit_err(errors::InvalidStability { span, item_sp });
                             break;
                         }
                         Ok(stab_vp) => match dep_v.parse::<u64>() {
                             Ok(dep_vp) => match dep_vp.cmp(&stab_vp) {
                                 Ordering::Less => {
-                                    self.tcx
-                                        .sess
-                                        .emit_err(CannotStabilizeDeprecated { span, item_sp });
+                                    self.tcx.sess.emit_err(errors::CannotStabilizeDeprecated {
+                                        span,
+                                        item_sp,
+                                    });
                                     break;
                                 }
                                 Ordering::Equal => continue,
@@ -251,9 +247,10 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
                             },
                             Err(_) => {
                                 if dep_v != "TBD" {
-                                    self.tcx
-                                        .sess
-                                        .emit_err(InvalidDeprecationVersion { span, item_sp });
+                                    self.tcx.sess.emit_err(errors::InvalidDeprecationVersion {
+                                        span,
+                                        item_sp,
+                                    });
                                 }
                                 break;
                             }
@@ -284,7 +281,7 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
         self.recurse_with_stability_attrs(
             depr.map(|(d, _)| DeprecationEntry::local(d, def_id)),
             stab,
-            if inherit_const_stability.yes() { const_stab } else { None },
+            inherit_const_stability.yes().then_some(const_stab).flatten(),
             visit_children,
         );
     }
@@ -526,8 +523,8 @@ impl<'tcx> MissingStabilityAnnotations<'tcx> {
             && stab.is_none()
             && self.effective_visibilities.is_reachable(def_id)
         {
-            let descr = self.tcx.def_kind(def_id).descr(def_id.to_def_id());
-            self.tcx.sess.emit_err(MissingStabilityAttr { span, descr });
+            let descr = self.tcx.def_descr(def_id.to_def_id());
+            self.tcx.sess.emit_err(errors::MissingStabilityAttr { span, descr });
         }
     }
 
@@ -540,7 +537,7 @@ impl<'tcx> MissingStabilityAnnotations<'tcx> {
         // then it would be "stable" at least for the impl.
         // We gate usages of it using `feature(const_trait_impl)` anyways
         // so there is no unstable leakage
-        if self.tcx.is_builtin_derive(def_id.to_def_id()) {
+        if self.tcx.is_automatically_derived(def_id.to_def_id()) {
             return;
         }
 
@@ -554,8 +551,8 @@ impl<'tcx> MissingStabilityAnnotations<'tcx> {
         let is_reachable = self.effective_visibilities.is_reachable(def_id);
 
         if is_const && is_stable && missing_const_stability_attribute && is_reachable {
-            let descr = self.tcx.def_kind(def_id).descr(def_id.to_def_id());
-            self.tcx.sess.emit_err(MissingConstStabAttr { span, descr });
+            let descr = self.tcx.def_descr(def_id.to_def_id());
+            self.tcx.sess.emit_err(errors::MissingConstStabAttr { span, descr });
         }
     }
 }
@@ -751,7 +748,10 @@ impl<'tcx> Visitor<'tcx> for Checker<'tcx> {
                         let mut c = CheckTraitImplStable { tcx: self.tcx, fully_stable: true };
                         c.visit_ty(self_ty);
                         c.visit_trait_ref(t);
-                        if c.fully_stable {
+
+                        // do not lint when the trait isn't resolved, since resolution error should
+                        // be fixed first
+                        if t.path.res != Res::Err && c.fully_stable {
                             self.tcx.struct_span_lint_hir(
                                 INEFFECTIVE_UNSTABLE_TRAIT_IMPL,
                                 item.hir_id(),
@@ -768,7 +768,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'tcx> {
                         && *constness == hir::Constness::Const
                         && const_stab.map_or(false, |(stab, _)| stab.is_const_stable())
                     {
-                        self.tcx.sess.emit_err(TraitImplConstStable { span: item.span });
+                        self.tcx.sess.emit_err(errors::TraitImplConstStable { span: item.span });
                     }
                 }
 
@@ -855,7 +855,8 @@ impl<'tcx> Visitor<'tcx> for Checker<'tcx> {
 /// See issue #94972 for details on why this is a special case
 fn is_unstable_reexport(tcx: TyCtxt<'_>, id: hir::HirId) -> bool {
     // Get the LocalDefId so we can lookup the item to check the kind.
-    let Some(def_id) = tcx.hir().opt_local_def_id(id) else { return false; };
+    let Some(owner) = id.as_owner() else { return false; };
+    let def_id = owner.def_id;
 
     let Some(stab) = tcx.stability().local_stability(def_id) else {
         return false;
@@ -946,7 +947,7 @@ pub fn check_unused_or_stable_features(tcx: TyCtxt<'_>) {
         }
         if !lang_features.insert(feature) {
             // Warn if the user enables a lang feature multiple times.
-            tcx.sess.emit_err(DuplicateFeatureErr { span, feature });
+            tcx.sess.emit_err(errors::DuplicateFeatureErr { span, feature });
         }
     }
 
@@ -954,14 +955,14 @@ pub fn check_unused_or_stable_features(tcx: TyCtxt<'_>) {
     let mut remaining_lib_features = FxIndexMap::default();
     for (feature, span) in declared_lib_features {
         if !tcx.sess.opts.unstable_features.is_nightly_build() {
-            tcx.sess.emit_err(FeatureOnlyOnNightly {
+            tcx.sess.emit_err(errors::FeatureOnlyOnNightly {
                 span: *span,
                 release_channel: env!("CFG_RELEASE_CHANNEL"),
             });
         }
         if remaining_lib_features.contains_key(&feature) {
             // Warn if the user enables a lib feature multiple times.
-            tcx.sess.emit_err(DuplicateFeatureErr { span: *span, feature: *feature });
+            tcx.sess.emit_err(errors::DuplicateFeatureErr { span: *span, feature: *feature });
         }
         remaining_lib_features.insert(feature, *span);
     }
@@ -1062,7 +1063,7 @@ pub fn check_unused_or_stable_features(tcx: TyCtxt<'_>) {
     }
 
     for (feature, span) in remaining_lib_features {
-        tcx.sess.emit_err(UnknownFeature { span, feature: *feature });
+        tcx.sess.emit_err(errors::UnknownFeature { span, feature: *feature });
     }
 
     for (implied_by, feature) in remaining_implications {
@@ -1073,7 +1074,7 @@ pub fn check_unused_or_stable_features(tcx: TyCtxt<'_>) {
             .map(|(_, span)| span)
             .or_else(|| local_defined_features.unstable.get(&feature))
             .expect("feature that implied another does not exist");
-        tcx.sess.emit_err(ImpliedFeatureNotExist { span, feature, implied_by });
+        tcx.sess.emit_err(errors::ImpliedFeatureNotExist { span, feature, implied_by });
     }
 
     // FIXME(#44232): the `used_features` table no longer exists, so we
diff --git a/compiler/rustc_error_messages/locales/en-US/plugin_impl.ftl b/compiler/rustc_plugin_impl/locales/en-US.ftl
index 8db32a42c..8db32a42c 100644
--- a/compiler/rustc_error_messages/locales/en-US/plugin_impl.ftl
+++ b/compiler/rustc_plugin_impl/locales/en-US.ftl
diff --git a/compiler/rustc_plugin_impl/src/lib.rs b/compiler/rustc_plugin_impl/src/lib.rs
index 9ac27c65d..3f03eef9e 100644
--- a/compiler/rustc_plugin_impl/src/lib.rs
+++ b/compiler/rustc_plugin_impl/src/lib.rs
@@ -11,11 +11,15 @@
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
 use rustc_lint::LintStore;
+use rustc_macros::fluent_messages;
 
 mod errors;
 pub mod load;
 
+fluent_messages! { "../locales/en-US.ftl" }
+
 /// Structure used to register plugins.
 ///
 /// A plugin registrar function takes an `&mut Registry` and should call
diff --git a/compiler/rustc_privacy/Cargo.toml b/compiler/rustc_privacy/Cargo.toml
index 832fdc9f0..744cb77dd 100644
--- a/compiler/rustc_privacy/Cargo.toml
+++ b/compiler/rustc_privacy/Cargo.toml
@@ -13,6 +13,5 @@ rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
-rustc_trait_selection = { path = "../rustc_trait_selection" }
 rustc_hir_analysis = { path = "../rustc_hir_analysis" }
 tracing = "0.1"
diff --git a/compiler/rustc_error_messages/locales/en-US/privacy.ftl b/compiler/rustc_privacy/locales/en-US.ftl
index a26d1b2b3..a26d1b2b3 100644
--- a/compiler/rustc_error_messages/locales/en-US/privacy.ftl
+++ b/compiler/rustc_privacy/locales/en-US.ftl
diff --git a/compiler/rustc_privacy/src/errors.rs b/compiler/rustc_privacy/src/errors.rs
index a6c95f1a8..72b53eefa 100644
--- a/compiler/rustc_privacy/src/errors.rs
+++ b/compiler/rustc_privacy/src/errors.rs
@@ -57,7 +57,7 @@ pub struct InPublicInterfaceTraits<'a> {
     pub vis_descr: &'static str,
     pub kind: &'a str,
     pub descr: DiagnosticArgFromDisplay<'a>,
-    #[label(visibility_label)]
+    #[label(privacy_visibility_label)]
     pub vis_span: Span,
 }
 
@@ -71,7 +71,7 @@ pub struct InPublicInterface<'a> {
     pub vis_descr: &'static str,
     pub kind: &'a str,
     pub descr: DiagnosticArgFromDisplay<'a>,
-    #[label(visibility_label)]
+    #[label(privacy_visibility_label)]
     pub vis_span: Span,
 }
 
diff --git a/compiler/rustc_privacy/src/lib.rs b/compiler/rustc_privacy/src/lib.rs
index 9a5d3cceb..50176c802 100644
--- a/compiler/rustc_privacy/src/lib.rs
+++ b/compiler/rustc_privacy/src/lib.rs
@@ -16,11 +16,13 @@ use rustc_ast::MacroDef;
 use rustc_attr as attr;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::intern::Interned;
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
-use rustc_hir::def_id::{DefId, LocalDefId, LocalDefIdSet, CRATE_DEF_ID};
+use rustc_hir::def_id::{DefId, LocalDefId, CRATE_DEF_ID};
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_hir::{AssocItemKind, HirIdSet, ItemId, Node, PatKind};
+use rustc_macros::fluent_messages;
 use rustc_middle::bug;
 use rustc_middle::hir::nested_filter;
 use rustc_middle::middle::privacy::{EffectiveVisibilities, Level};
@@ -44,6 +46,8 @@ use errors::{
     UnnamedItemIsPrivate,
 };
 
+fluent_messages! { "../locales/en-US.ftl" }
+
 ////////////////////////////////////////////////////////////////////////////////
 /// Generic infrastructure used to implement specific visitors below.
 ////////////////////////////////////////////////////////////////////////////////
@@ -81,7 +85,10 @@ trait DefIdVisitor<'tcx> {
             dummy: Default::default(),
         }
     }
-    fn visit(&mut self, ty_fragment: impl TypeVisitable<'tcx>) -> ControlFlow<Self::BreakTy> {
+    fn visit(
+        &mut self,
+        ty_fragment: impl TypeVisitable<TyCtxt<'tcx>>,
+    ) -> ControlFlow<Self::BreakTy> {
         ty_fragment.visit_with(&mut self.skeleton())
     }
     fn visit_trait(&mut self, trait_ref: TraitRef<'tcx>) -> ControlFlow<Self::BreakTy> {
@@ -159,9 +166,21 @@ where
                 _region,
             ))) => ty.visit_with(self),
             ty::PredicateKind::Clause(ty::Clause::RegionOutlives(..)) => ControlFlow::Continue(()),
+            ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(ct, ty)) => {
+                ct.visit_with(self)?;
+                ty.visit_with(self)
+            }
             ty::PredicateKind::ConstEvaluatable(ct) => ct.visit_with(self),
             ty::PredicateKind::WellFormed(arg) => arg.visit_with(self),
-            _ => bug!("unexpected predicate: {:?}", predicate),
+
+            ty::PredicateKind::ObjectSafe(_)
+            | ty::PredicateKind::ClosureKind(_, _, _)
+            | ty::PredicateKind::Subtype(_)
+            | ty::PredicateKind::Coerce(_)
+            | ty::PredicateKind::ConstEquate(_, _)
+            | ty::PredicateKind::TypeWellFormedFromEnv(_)
+            | ty::PredicateKind::Ambiguous
+            | ty::PredicateKind::AliasEq(_, _) => bug!("unexpected predicate: {:?}", predicate),
         }
     }
 
@@ -174,7 +193,7 @@ where
     }
 }
 
-impl<'tcx, V> TypeVisitor<'tcx> for DefIdVisitorSkeleton<'_, 'tcx, V>
+impl<'tcx, V> TypeVisitor<TyCtxt<'tcx>> for DefIdVisitorSkeleton<'_, 'tcx, V>
 where
     V: DefIdVisitor<'tcx> + ?Sized,
 {
@@ -198,7 +217,8 @@ where
                 // Something like `fn() -> Priv {my_func}` is considered a private type even if
                 // `my_func` is public, so we need to visit signatures.
                 if let ty::FnDef(..) = ty.kind() {
-                    tcx.fn_sig(def_id).visit_with(self)?;
+                    // FIXME: this should probably use `substs` from `FnDef`
+                    tcx.fn_sig(def_id).subst_identity().visit_with(self)?;
                 }
                 // Inherent static methods don't have self type in substs.
                 // Something like `fn() {my_method}` type of the method
@@ -206,7 +226,7 @@ where
                 // so we need to visit the self type additionally.
                 if let Some(assoc_item) = tcx.opt_associated_item(def_id) {
                     if let Some(impl_def_id) = assoc_item.impl_container(tcx) {
-                        tcx.type_of(impl_def_id).visit_with(self)?;
+                        tcx.type_of(impl_def_id).subst_identity().visit_with(self)?;
                     }
                 }
             }
@@ -269,9 +289,11 @@ where
             | ty::Ref(..)
             | ty::FnPtr(..)
             | ty::Param(..)
+            | ty::Bound(..)
             | ty::Error(_)
-            | ty::GeneratorWitness(..) => {}
-            ty::Bound(..) | ty::Placeholder(..) | ty::Infer(..) => {
+            | ty::GeneratorWitness(..)
+            | ty::GeneratorWitnessMIR(..) => {}
+            ty::Placeholder(..) | ty::Infer(..) => {
                 bug!("unexpected type: {:?}", ty)
             }
         }
@@ -339,7 +361,7 @@ trait VisibilityLike: Sized {
         effective_visibilities: &EffectiveVisibilities,
     ) -> Self {
         let mut find = FindMin { tcx, effective_visibilities, min: Self::MAX };
-        find.visit(tcx.type_of(def_id));
+        find.visit(tcx.type_of(def_id).subst_identity());
         if let Some(trait_ref) = tcx.impl_trait_ref(def_id) {
             find.visit_trait(trait_ref.subst_identity());
         }
@@ -591,7 +613,7 @@ impl<'tcx> EmbargoVisitor<'tcx> {
             | DefKind::InlineConst
             | DefKind::Field
             | DefKind::GlobalAsm
-            | DefKind::Impl
+            | DefKind::Impl { .. }
             | DefKind::Closure
             | DefKind::Generator => (),
         }
@@ -835,11 +857,11 @@ impl ReachEverythingInTheInterfaceVisitor<'_, '_> {
                 GenericParamDefKind::Lifetime => {}
                 GenericParamDefKind::Type { has_default, .. } => {
                     if has_default {
-                        self.visit(self.ev.tcx.type_of(param.def_id));
+                        self.visit(self.ev.tcx.type_of(param.def_id).subst_identity());
                     }
                 }
                 GenericParamDefKind::Const { has_default } => {
-                    self.visit(self.ev.tcx.type_of(param.def_id));
+                    self.visit(self.ev.tcx.type_of(param.def_id).subst_identity());
                     if has_default {
                         self.visit(self.ev.tcx.const_param_default(param.def_id).subst_identity());
                     }
@@ -855,7 +877,7 @@ impl ReachEverythingInTheInterfaceVisitor<'_, '_> {
     }
 
     fn ty(&mut self) -> &mut Self {
-        self.visit(self.ev.tcx.type_of(self.item_def_id));
+        self.visit(self.ev.tcx.type_of(self.item_def_id).subst_identity());
         self
     }
 
@@ -1266,7 +1288,7 @@ impl<'tcx> Visitor<'tcx> for TypePrivacyVisitor<'tcx> {
                 // Method calls have to be checked specially.
                 self.span = segment.ident.span;
                 if let Some(def_id) = self.typeck_results().type_dependent_def_id(expr.hir_id) {
-                    if self.visit(self.tcx.type_of(def_id)).is_break() {
+                    if self.visit(self.tcx.type_of(def_id).subst_identity()).is_break() {
                         return;
                     }
                 } else {
@@ -1314,7 +1336,7 @@ impl<'tcx> Visitor<'tcx> for TypePrivacyVisitor<'tcx> {
                     hir::QPath::Resolved(_, path) => Some(self.tcx.def_path_str(path.res.def_id())),
                     hir::QPath::TypeRelative(_, segment) => Some(segment.ident.to_string()),
                 };
-                let kind = kind.descr(def_id);
+                let kind = self.tcx.def_descr(def_id);
                 let sess = self.tcx.sess;
                 let _ = match name {
                     Some(name) => {
@@ -1740,12 +1762,12 @@ impl SearchInterfaceForPrivateItemsVisitor<'_> {
                 GenericParamDefKind::Lifetime => {}
                 GenericParamDefKind::Type { has_default, .. } => {
                     if has_default {
-                        self.visit(self.tcx.type_of(param.def_id));
+                        self.visit(self.tcx.type_of(param.def_id).subst_identity());
                     }
                 }
                 // FIXME(generic_const_exprs): May want to look inside const here
                 GenericParamDefKind::Const { .. } => {
-                    self.visit(self.tcx.type_of(param.def_id));
+                    self.visit(self.tcx.type_of(param.def_id).subst_identity());
                 }
             }
         }
@@ -1772,7 +1794,7 @@ impl SearchInterfaceForPrivateItemsVisitor<'_> {
     }
 
     fn ty(&mut self) -> &mut Self {
-        self.visit(self.tcx.type_of(self.item_def_id));
+        self.visit(self.tcx.type_of(self.item_def_id).subst_identity());
         self
     }
 
@@ -1877,7 +1899,7 @@ impl<'tcx> DefIdVisitor<'tcx> for SearchInterfaceForPrivateItemsVisitor<'tcx> {
 
 struct PrivateItemsInPublicInterfacesChecker<'tcx> {
     tcx: TyCtxt<'tcx>,
-    old_error_set_ancestry: LocalDefIdSet,
+    old_error_set_ancestry: HirIdSet,
 }
 
 impl<'tcx> PrivateItemsInPublicInterfacesChecker<'tcx> {
@@ -1890,7 +1912,9 @@ impl<'tcx> PrivateItemsInPublicInterfacesChecker<'tcx> {
             tcx: self.tcx,
             item_def_id: def_id,
             required_visibility,
-            has_old_errors: self.old_error_set_ancestry.contains(&def_id),
+            has_old_errors: self
+                .old_error_set_ancestry
+                .contains(&self.tcx.hir().local_def_id_to_hir_id(def_id)),
             in_assoc_ty: false,
         }
     }
@@ -1993,7 +2017,7 @@ impl<'tcx> PrivateItemsInPublicInterfacesChecker<'tcx> {
             // Subitems of inherent impls have their own publicity.
             // A trait impl is public when both its type and its trait are public
             // Subitems of trait impls have inherited publicity.
-            DefKind::Impl => {
+            DefKind::Impl { .. } => {
                 let item = tcx.hir().item(id);
                 if let hir::ItemKind::Impl(ref impl_) = item.kind {
                     let impl_vis =
@@ -2156,15 +2180,7 @@ fn check_private_in_public(tcx: TyCtxt<'_>, (): ()) {
     }
 
     // Check for private types and traits in public interfaces.
-    let mut checker = PrivateItemsInPublicInterfacesChecker {
-        tcx,
-        // Only definition IDs are ever searched in `old_error_set_ancestry`,
-        // so we can filter away all non-definition IDs at this point.
-        old_error_set_ancestry: old_error_set_ancestry
-            .into_iter()
-            .filter_map(|hir_id| tcx.hir().opt_local_def_id(hir_id))
-            .collect(),
-    };
+    let mut checker = PrivateItemsInPublicInterfacesChecker { tcx, old_error_set_ancestry };
 
     for id in tcx.hir().items() {
         checker.check_item(id);
diff --git a/compiler/rustc_query_impl/Cargo.toml b/compiler/rustc_query_impl/Cargo.toml
index 46e776264..3e8a88c7e 100644
--- a/compiler/rustc_query_impl/Cargo.toml
+++ b/compiler/rustc_query_impl/Cargo.toml
@@ -20,8 +20,7 @@ rustc-rayon-core = { version = "0.4.0", optional = true }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
-rustc_target = { path = "../rustc_target" }
-thin-vec = "0.2.9"
+thin-vec = "0.2.12"
 tracing = "0.1"
 
 [features]
diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs
index 2d243e13c..d7708a3bc 100644
--- a/compiler/rustc_query_impl/src/lib.rs
+++ b/compiler/rustc_query_impl/src/lib.rs
@@ -21,7 +21,9 @@ use rustc_data_structures::sync::AtomicU64;
 use rustc_middle::arena::Arena;
 use rustc_middle::dep_graph::{self, DepKindStruct};
 use rustc_middle::query::Key;
-use rustc_middle::ty::query::{query_keys, query_storage, query_stored, query_values};
+use rustc_middle::ty::query::{
+    query_keys, query_provided, query_provided_to_value, query_storage, query_values,
+};
 use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine};
 use rustc_middle::ty::TyCtxt;
 use rustc_span::Span;
diff --git a/compiler/rustc_query_impl/src/on_disk_cache.rs b/compiler/rustc_query_impl/src/on_disk_cache.rs
index 70c481fb0..46e34462c 100644
--- a/compiler/rustc_query_impl/src/on_disk_cache.rs
+++ b/compiler/rustc_query_impl/src/on_disk_cache.rs
@@ -333,7 +333,7 @@ impl<'sess> rustc_middle::ty::OnDiskCache<'sess> for OnDiskCache<'sess> {
                 },
             );
 
-            // `Encode the file footer.
+            // Encode the file footer.
             let footer_pos = encoder.position() as u64;
             encoder.encode_tagged(
                 TAG_FILE_FOOTER,
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
index 6125ad4ef..a8592bd70 100644
--- a/compiler/rustc_query_impl/src/plumbing.rs
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -53,7 +53,7 @@ impl<'tcx> HasDepContext for QueryCtxt<'tcx> {
 }
 
 impl QueryContext for QueryCtxt<'_> {
-    fn next_job_id(&self) -> QueryJobId {
+    fn next_job_id(self) -> QueryJobId {
         QueryJobId(
             NonZeroU64::new(
                 self.queries.jobs.fetch_add(1, rustc_data_structures::sync::Ordering::Relaxed),
@@ -62,31 +62,31 @@ impl QueryContext for QueryCtxt<'_> {
         )
     }
 
-    fn current_query_job(&self) -> Option<QueryJobId> {
-        tls::with_related_context(**self, |icx| icx.query)
+    fn current_query_job(self) -> Option<QueryJobId> {
+        tls::with_related_context(*self, |icx| icx.query)
     }
 
-    fn try_collect_active_jobs(&self) -> Option<QueryMap<DepKind>> {
-        self.queries.try_collect_active_jobs(**self)
+    fn try_collect_active_jobs(self) -> Option<QueryMap<DepKind>> {
+        self.queries.try_collect_active_jobs(*self)
     }
 
     // Interactions with on_disk_cache
-    fn load_side_effects(&self, prev_dep_node_index: SerializedDepNodeIndex) -> QuerySideEffects {
+    fn load_side_effects(self, prev_dep_node_index: SerializedDepNodeIndex) -> QuerySideEffects {
         self.queries
             .on_disk_cache
             .as_ref()
-            .map(|c| c.load_side_effects(**self, prev_dep_node_index))
+            .map(|c| c.load_side_effects(*self, prev_dep_node_index))
             .unwrap_or_default()
     }
 
-    fn store_side_effects(&self, dep_node_index: DepNodeIndex, side_effects: QuerySideEffects) {
+    fn store_side_effects(self, dep_node_index: DepNodeIndex, side_effects: QuerySideEffects) {
         if let Some(c) = self.queries.on_disk_cache.as_ref() {
             c.store_side_effects(dep_node_index, side_effects)
         }
     }
 
     fn store_side_effects_for_anon_node(
-        &self,
+        self,
         dep_node_index: DepNodeIndex,
         side_effects: QuerySideEffects,
     ) {
@@ -100,7 +100,7 @@ impl QueryContext for QueryCtxt<'_> {
     /// captured during execution and the actual result.
     #[inline(always)]
     fn start_query<R>(
-        &self,
+        self,
         token: QueryJobId,
         depth_limit: bool,
         diagnostics: Option<&Lock<ThinVec<Diagnostic>>>,
@@ -109,14 +109,14 @@ impl QueryContext for QueryCtxt<'_> {
         // The `TyCtxt` stored in TLS has the same global interner lifetime
         // as `self`, so we use `with_related_context` to relate the 'tcx lifetimes
         // when accessing the `ImplicitCtxt`.
-        tls::with_related_context(**self, move |current_icx| {
+        tls::with_related_context(*self, move |current_icx| {
             if depth_limit && !self.recursion_limit().value_within_limit(current_icx.query_depth) {
                 self.depth_limit_error(token);
             }
 
             // Update the `ImplicitCtxt` to point to our new query job.
             let new_icx = ImplicitCtxt {
-                tcx: **self,
+                tcx: *self,
                 query: Some(token),
                 diagnostics,
                 query_depth: current_icx.query_depth + depth_limit as usize,
@@ -124,13 +124,11 @@ impl QueryContext for QueryCtxt<'_> {
             };
 
             // Use the `ImplicitCtxt` while we execute the query.
-            tls::enter_context(&new_icx, |_| {
-                rustc_data_structures::stack::ensure_sufficient_stack(compute)
-            })
+            tls::enter_context(&new_icx, compute)
         })
     }
 
-    fn depth_limit_error(&self, job: QueryJobId) {
+    fn depth_limit_error(self, job: QueryJobId) {
         let mut span = None;
         let mut layout_of_depth = None;
         if let Some(map) = self.try_collect_active_jobs() {
@@ -293,14 +291,14 @@ macro_rules! get_provider {
 }
 
 macro_rules! should_ever_cache_on_disk {
-    ([]) => {{
-        None
+    ([]$yes:tt $no:tt) => {{
+        $no
     }};
-    ([(cache) $($rest:tt)*]) => {{
-        Some($crate::plumbing::try_load_from_disk::<Self::Value>)
+    ([(cache) $($rest:tt)*]$yes:tt $no:tt) => {{
+        $yes
     }};
-    ([$other:tt $($modifiers:tt)*]) => {
-        should_ever_cache_on_disk!([$($modifiers)*])
+    ([$other:tt $($modifiers:tt)*]$yes:tt $no:tt) => {
+        should_ever_cache_on_disk!([$($modifiers)*]$yes $no)
     };
 }
 
@@ -314,11 +312,14 @@ pub(crate) fn create_query_frame<
     kind: DepKind,
     name: &'static str,
 ) -> QueryStackFrame<DepKind> {
-    // Disable visible paths printing for performance reasons.
-    // Showing visible path instead of any path is not that important in production.
-    let description = ty::print::with_no_visible_paths!(
-        // Force filename-line mode to avoid invoking `type_of` query.
-        ty::print::with_forced_impl_filename_line!(do_describe(tcx.tcx, key))
+    // Avoid calling queries while formatting the description
+    let description = ty::print::with_no_queries!(
+        // Disable visible paths printing for performance reasons.
+        // Showing visible path instead of any path is not that important in production.
+        ty::print::with_no_visible_paths!(
+            // Force filename-line mode to avoid invoking `type_of` query.
+            ty::print::with_forced_impl_filename_line!(do_describe(tcx.tcx, key))
+        )
     );
     let description =
         if tcx.sess.verbose() { format!("{description} [{name:?}]") } else { description };
@@ -469,7 +470,6 @@ macro_rules! define_queries {
         $(impl<'tcx> QueryConfig<QueryCtxt<'tcx>> for queries::$name<'tcx> {
             type Key = query_keys::$name<'tcx>;
             type Value = query_values::$name<'tcx>;
-            type Stored = query_stored::$name<'tcx>;
             const NAME: &'static str = stringify!($name);
 
             #[inline]
@@ -490,24 +490,39 @@ macro_rules! define_queries {
             fn query_cache<'a>(tcx: QueryCtxt<'tcx>) -> &'a Self::Cache
                 where 'tcx:'a
             {
-                &tcx.query_caches.$name
+                &tcx.query_system.caches.$name
             }
 
-            fn execute_query(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Stored {
+            fn execute_query(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
                 tcx.$name(key)
             }
 
             #[inline]
-            // key is only sometimes used
             #[allow(unused_variables)]
-            fn compute(qcx: QueryCtxt<'tcx>, key: &Self::Key) -> fn(TyCtxt<'tcx>, Self::Key) -> Self::Value {
-                get_provider!([$($modifiers)*][qcx, $name, key])
+            fn compute(qcx: QueryCtxt<'tcx>, key: Self::Key) -> Self::Value {
+                query_provided_to_value::$name(
+                    qcx.tcx,
+                    get_provider!([$($modifiers)*][qcx, $name, key])(qcx.tcx, key)
+                )
             }
 
             #[inline]
-            fn try_load_from_disk(qcx: QueryCtxt<'tcx>, key: &Self::Key) -> rustc_query_system::query::TryLoadFromDisk<QueryCtxt<'tcx>, Self> {
-                let cache_on_disk = Self::cache_on_disk(qcx.tcx, key);
-                if cache_on_disk { should_ever_cache_on_disk!([$($modifiers)*]) } else { None }
+            fn try_load_from_disk(_qcx: QueryCtxt<'tcx>, _key: &Self::Key) -> rustc_query_system::query::TryLoadFromDisk<QueryCtxt<'tcx>, Self> {
+                should_ever_cache_on_disk!([$($modifiers)*] {
+                    if Self::cache_on_disk(_qcx.tcx, _key) {
+                        Some(|qcx: QueryCtxt<'tcx>, dep_node| {
+                            let value = $crate::plumbing::try_load_from_disk::<query_provided::$name<'tcx>>(
+                                qcx,
+                                dep_node
+                            );
+                            value.map(|value| query_provided_to_value::$name(qcx.tcx, value))
+                        })
+                    } else {
+                        None
+                    }
+                } {
+                    None
+                })
             }
 
             const ANON: bool = is_anon!([$($modifiers)*]);
@@ -630,7 +645,7 @@ macro_rules! define_queries {
                     $crate::profiling_support::alloc_self_profile_query_strings_for_query_cache(
                         tcx,
                         stringify!($name),
-                        &tcx.query_caches.$name,
+                        &tcx.query_system.caches.$name,
                         string_cache,
                     )
                 },
@@ -722,7 +737,7 @@ macro_rules! define_queries_struct {
                 span: Span,
                 key: <queries::$name<'tcx> as QueryConfig<QueryCtxt<'tcx>>>::Key,
                 mode: QueryMode,
-            ) -> Option<query_stored::$name<'tcx>> {
+            ) -> Option<query_values::$name<'tcx>> {
                 let qcx = QueryCtxt { tcx, queries: self };
                 get_query::<queries::$name<'tcx>, _, rustc_middle::dep_graph::DepKind>(qcx, span, key, mode)
             })*
diff --git a/compiler/rustc_query_system/Cargo.toml b/compiler/rustc_query_system/Cargo.toml
index 028756b5a..7d8f75e25 100644
--- a/compiler/rustc_query_system/Cargo.toml
+++ b/compiler/rustc_query_system/Cargo.toml
@@ -22,7 +22,7 @@ rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 rustc_type_ir = { path = "../rustc_type_ir" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-thin-vec = "0.2.9"
+thin-vec = "0.2.12"
 tracing = "0.1"
 
 [features]
diff --git a/compiler/rustc_error_messages/locales/en-US/query_system.ftl b/compiler/rustc_query_system/locales/en-US.ftl
index 870e82403..870e82403 100644
--- a/compiler/rustc_error_messages/locales/en-US/query_system.ftl
+++ b/compiler/rustc_query_system/locales/en-US.ftl
diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs
index 47b2fd8f8..59e0c3597 100644
--- a/compiler/rustc_query_system/src/dep_graph/graph.rs
+++ b/compiler/rustc_query_system/src/dep_graph/graph.rs
@@ -6,6 +6,7 @@ use rustc_data_structures::sharded::{self, Sharded};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::steal::Steal;
 use rustc_data_structures::sync::{AtomicU32, AtomicU64, Lock, Lrc, Ordering};
+use rustc_data_structures::OnDrop;
 use rustc_index::vec::IndexVec;
 use rustc_serialize::opaque::{FileEncodeResult, FileEncoder};
 use smallvec::{smallvec, SmallVec};
@@ -47,7 +48,7 @@ impl DepNodeIndex {
 }
 
 impl From<DepNodeIndex> for QueryInvocationId {
-    #[inline]
+    #[inline(always)]
     fn from(dep_node_index: DepNodeIndex) -> Self {
         QueryInvocationId(dep_node_index.as_u32())
     }
@@ -278,6 +279,7 @@ impl<K: DepKind> DepGraph<K> {
     ///   `arg` parameter.
     ///
     /// [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/incremental-compilation.html
+    #[inline(always)]
     pub fn with_task<Ctxt: HasDepContext<DepKind = K>, A: Debug, R>(
         &self,
         key: DepNode<K>,
@@ -297,6 +299,7 @@ impl<K: DepKind> DepGraph<K> {
         }
     }
 
+    #[inline(always)]
     fn with_task_impl<Ctxt: HasDepContext<DepKind = K>, A: Debug, R>(
         &self,
         key: DepNode<K>,
@@ -597,6 +600,7 @@ impl<K: DepKind> DepGraph<K> {
         self.data.is_some() && self.dep_node_index_of_opt(dep_node).is_some()
     }
 
+    #[inline]
     pub fn prev_fingerprint_of(&self, dep_node: &DepNode<K>) -> Option<Fingerprint> {
         self.data.as_ref().unwrap().previous.fingerprint_of(dep_node)
     }
@@ -671,17 +675,24 @@ impl<K: DepKind> DepGraph<K> {
         let prev_index = data.previous.node_to_index_opt(dep_node)?;
 
         match data.colors.get(prev_index) {
-            Some(DepNodeColor::Green(dep_node_index)) => Some((prev_index, dep_node_index)),
-            Some(DepNodeColor::Red) => None,
-            None => {
-                // This DepNode and the corresponding query invocation existed
-                // in the previous compilation session too, so we can try to
-                // mark it as green by recursively marking all of its
-                // dependencies green.
-                self.try_mark_previous_green(qcx, data, prev_index, &dep_node)
-                    .map(|dep_node_index| (prev_index, dep_node_index))
-            }
+            Some(DepNodeColor::Green(dep_node_index)) => return Some((prev_index, dep_node_index)),
+            Some(DepNodeColor::Red) => return None,
+            None => {}
         }
+
+        let backtrace = backtrace_printer(qcx.dep_context().sess(), data, prev_index);
+
+        // This DepNode and the corresponding query invocation existed
+        // in the previous compilation session too, so we can try to
+        // mark it as green by recursively marking all of its
+        // dependencies green.
+        let ret = self
+            .try_mark_previous_green(qcx, data, prev_index, &dep_node)
+            .map(|dep_node_index| (prev_index, dep_node_index));
+
+        // We succeeded, no backtrace.
+        backtrace.disable();
+        return ret;
     }
 
     #[instrument(skip(self, qcx, data, parent_dep_node_index), level = "debug")]
@@ -794,7 +805,10 @@ impl<K: DepKind> DepGraph<K> {
         let prev_deps = data.previous.edge_targets_from(prev_dep_node_index);
 
         for &dep_dep_node_index in prev_deps {
-            self.try_mark_parent_green(qcx, data, dep_dep_node_index, dep_node)?
+            let backtrace = backtrace_printer(qcx.dep_context().sess(), data, dep_dep_node_index);
+            let success = self.try_mark_parent_green(qcx, data, dep_dep_node_index, dep_node);
+            backtrace.disable();
+            success?;
         }
 
         // If we got here without hitting a `return` that means that all
@@ -1116,6 +1130,7 @@ impl<K: DepKind> CurrentDepGraph<K> {
 
     /// Writes the node to the current dep-graph and allocates a `DepNodeIndex` for it.
     /// Assumes that this is a node that has no equivalent in the previous dep-graph.
+    #[inline(always)]
     fn intern_new_node(
         &self,
         profiler: &SelfProfilerRef,
@@ -1354,6 +1369,7 @@ impl DepNodeColorMap {
         }
     }
 
+    #[inline]
     fn insert(&self, index: SerializedDepNodeIndex, color: DepNodeColor) {
         self.values[index].store(
             match color {
@@ -1364,3 +1380,26 @@ impl DepNodeColorMap {
         )
     }
 }
+
+fn backtrace_printer<'a, K: DepKind>(
+    sess: &'a rustc_session::Session,
+    graph: &'a DepGraphData<K>,
+    node: SerializedDepNodeIndex,
+) -> OnDrop<impl Fn() + 'a> {
+    OnDrop(
+        #[inline(never)]
+        #[cold]
+        move || {
+            let node = graph.previous.index_to_node(node);
+            // Do not try to rely on DepNode's Debug implementation, since it may panic.
+            let diag = rustc_errors::Diagnostic::new(
+                rustc_errors::Level::FailureNote,
+                &format!(
+                    "encountered while trying to mark dependency green: {:?}({})",
+                    node.kind, node.hash
+                ),
+            );
+            sess.diagnostic().force_print_diagnostic(diag);
+        },
+    )
+}
diff --git a/compiler/rustc_query_system/src/dep_graph/mod.rs b/compiler/rustc_query_system/src/dep_graph/mod.rs
index e370c6990..6969f2dbe 100644
--- a/compiler/rustc_query_system/src/dep_graph/mod.rs
+++ b/compiler/rustc_query_system/src/dep_graph/mod.rs
@@ -23,7 +23,7 @@ pub trait DepContext: Copy {
     type DepKind: self::DepKind;
 
     /// Create a hashing context for hashing new results.
-    fn with_stable_hashing_context<R>(&self, f: impl FnOnce(StableHashingContext<'_>) -> R) -> R;
+    fn with_stable_hashing_context<R>(self, f: impl FnOnce(StableHashingContext<'_>) -> R) -> R;
 
     /// Access the DepGraph.
     fn dep_graph(&self) -> &DepGraph<Self::DepKind>;
@@ -37,7 +37,7 @@ pub trait DepContext: Copy {
     fn dep_kind_info(&self, dep_node: Self::DepKind) -> &DepKindStruct<Self>;
 
     #[inline(always)]
-    fn fingerprint_style(&self, kind: Self::DepKind) -> FingerprintStyle {
+    fn fingerprint_style(self, kind: Self::DepKind) -> FingerprintStyle {
         let data = self.dep_kind_info(kind);
         if data.is_anon {
             return FingerprintStyle::Opaque;
@@ -47,7 +47,7 @@ pub trait DepContext: Copy {
 
     #[inline(always)]
     /// Return whether this kind always require evaluation.
-    fn is_eval_always(&self, kind: Self::DepKind) -> bool {
+    fn is_eval_always(self, kind: Self::DepKind) -> bool {
         self.dep_kind_info(kind).is_eval_always
     }
 
diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs
index a81595b24..29513df46 100644
--- a/compiler/rustc_query_system/src/dep_graph/serialized.rs
+++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs
@@ -242,8 +242,7 @@ impl<K: DepKind + Encodable<FileEncoder>> GraphEncoder<K> {
         record_graph: bool,
         record_stats: bool,
     ) -> Self {
-        let record_graph =
-            if record_graph { Some(Lock::new(DepGraphQuery::new(prev_node_count))) } else { None };
+        let record_graph = record_graph.then(|| Lock::new(DepGraphQuery::new(prev_node_count)));
         let status = Lock::new(EncoderState::new(encoder, record_stats));
         GraphEncoder { status, record_graph }
     }
diff --git a/compiler/rustc_query_system/src/ich/hcx.rs b/compiler/rustc_query_system/src/ich/hcx.rs
index 163da59ed..5593a1541 100644
--- a/compiler/rustc_query_system/src/ich/hcx.rs
+++ b/compiler/rustc_query_system/src/ich/hcx.rs
@@ -90,7 +90,7 @@ impl<'a> StableHashingContext<'a> {
         if let Some(def_id) = def_id.as_local() {
             self.local_def_path_hash(def_id)
         } else {
-            self.untracked.cstore.def_path_hash(def_id)
+            self.untracked.cstore.read().def_path_hash(def_id)
         }
     }
 
@@ -146,7 +146,7 @@ impl<'a> rustc_span::HashStableContext for StableHashingContext<'a> {
 
     #[inline]
     fn def_span(&self, def_id: LocalDefId) -> Span {
-        *self.untracked.source_span.get(def_id).unwrap_or(&DUMMY_SP)
+        self.untracked.source_span.get(def_id).unwrap_or(DUMMY_SP)
     }
 
     #[inline]
diff --git a/compiler/rustc_query_system/src/lib.rs b/compiler/rustc_query_system/src/lib.rs
index f47760e9a..6cc4c9a7e 100644
--- a/compiler/rustc_query_system/src/lib.rs
+++ b/compiler/rustc_query_system/src/lib.rs
@@ -3,6 +3,7 @@
 #![feature(hash_raw_entry)]
 #![feature(min_specialization)]
 #![feature(extern_types)]
+#![feature(let_chains)]
 #![allow(rustc::potential_query_instability)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
@@ -14,6 +15,9 @@ extern crate rustc_data_structures;
 #[macro_use]
 extern crate rustc_macros;
 
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
+use rustc_macros::fluent_messages;
+
 pub mod cache;
 pub mod dep_graph;
 mod error;
@@ -25,3 +29,5 @@ pub use error::HandleCycleError;
 pub use error::LayoutOfDepth;
 pub use error::QueryOverflow;
 pub use values::Value;
+
+fluent_messages! { "../locales/en-US.ftl" }
diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs
index 77d0d0314..4b3cd16c2 100644
--- a/compiler/rustc_query_system/src/query/caches.rs
+++ b/compiler/rustc_query_system/src/query/caches.rs
@@ -1,13 +1,10 @@
 use crate::dep_graph::DepNodeIndex;
 
-use rustc_arena::TypedArena;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sharded;
 #[cfg(parallel_compiler)]
 use rustc_data_structures::sharded::Sharded;
-#[cfg(not(parallel_compiler))]
 use rustc_data_structures::sync::Lock;
-use rustc_data_structures::sync::WorkerLocal;
 use rustc_index::vec::{Idx, IndexVec};
 use std::fmt::Debug;
 use std::hash::Hash;
@@ -16,36 +13,23 @@ use std::marker::PhantomData;
 pub trait CacheSelector<'tcx, V> {
     type Cache
     where
-        V: Clone;
-    type ArenaCache;
+        V: Copy;
 }
 
 pub trait QueryStorage {
-    type Value: Debug;
-    type Stored: Clone;
-
-    /// Store a value without putting it in the cache.
-    /// This is meant to be used with cycle errors.
-    fn store_nocache(&self, value: Self::Value) -> Self::Stored;
+    type Value: Copy;
 }
 
 pub trait QueryCache: QueryStorage + Sized {
-    type Key: Hash + Eq + Clone + Debug;
+    type Key: Hash + Eq + Copy + Debug;
 
     /// Checks if the query is already computed and in the cache.
     /// It returns the shard index and a lock guard to the shard,
     /// which will be used if the query is not in the cache and we need
     /// to compute it.
-    fn lookup<R, OnHit>(
-        &self,
-        key: &Self::Key,
-        // `on_hit` can be called while holding a lock to the query state shard.
-        on_hit: OnHit,
-    ) -> Result<R, ()>
-    where
-        OnHit: FnOnce(&Self::Stored, DepNodeIndex) -> R;
+    fn lookup(&self, key: &Self::Key) -> Option<(Self::Value, DepNodeIndex)>;
 
-    fn complete(&self, key: Self::Key, value: Self::Value, index: DepNodeIndex) -> Self::Stored;
+    fn complete(&self, key: Self::Key, value: Self::Value, index: DepNodeIndex);
 
     fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex));
 }
@@ -55,8 +39,7 @@ pub struct DefaultCacheSelector<K>(PhantomData<K>);
 impl<'tcx, K: Eq + Hash, V: 'tcx> CacheSelector<'tcx, V> for DefaultCacheSelector<K> {
     type Cache = DefaultCache<K, V>
     where
-        V: Clone;
-    type ArenaCache = ArenaCache<'tcx, K, V>;
+        V: Copy;
 }
 
 pub struct DefaultCache<K, V> {
@@ -72,29 +55,19 @@ impl<K, V> Default for DefaultCache<K, V> {
     }
 }
 
-impl<K: Eq + Hash, V: Clone + Debug> QueryStorage for DefaultCache<K, V> {
+impl<K: Eq + Hash, V: Copy + Debug> QueryStorage for DefaultCache<K, V> {
     type Value = V;
-    type Stored = V;
-
-    #[inline]
-    fn store_nocache(&self, value: Self::Value) -> Self::Stored {
-        // We have no dedicated storage
-        value
-    }
 }
 
 impl<K, V> QueryCache for DefaultCache<K, V>
 where
-    K: Eq + Hash + Clone + Debug,
-    V: Clone + Debug,
+    K: Eq + Hash + Copy + Debug,
+    V: Copy + Debug,
 {
     type Key = K;
 
     #[inline(always)]
-    fn lookup<R, OnHit>(&self, key: &K, on_hit: OnHit) -> Result<R, ()>
-    where
-        OnHit: FnOnce(&V, DepNodeIndex) -> R,
-    {
+    fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> {
         let key_hash = sharded::make_hash(key);
         #[cfg(parallel_compiler)]
         let lock = self.cache.get_shard_by_hash(key_hash).lock();
@@ -102,24 +75,18 @@ where
         let lock = self.cache.lock();
         let result = lock.raw_entry().from_key_hashed_nocheck(key_hash, key);
 
-        if let Some((_, value)) = result {
-            let hit_result = on_hit(&value.0, value.1);
-            Ok(hit_result)
-        } else {
-            Err(())
-        }
+        if let Some((_, value)) = result { Some(*value) } else { None }
     }
 
     #[inline]
-    fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored {
+    fn complete(&self, key: K, value: V, index: DepNodeIndex) {
         #[cfg(parallel_compiler)]
         let mut lock = self.cache.get_shard_by_value(&key).lock();
         #[cfg(not(parallel_compiler))]
         let mut lock = self.cache.lock();
         // We may be overwriting another value. This is all right, since the dep-graph
         // will check that the fingerprint matches.
-        lock.insert(key, (value.clone(), index));
-        value
+        lock.insert(key, (value, index));
     }
 
     fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
@@ -142,90 +109,46 @@ where
     }
 }
 
-pub struct ArenaCache<'tcx, K, V> {
-    arena: WorkerLocal<TypedArena<(V, DepNodeIndex)>>,
-    #[cfg(parallel_compiler)]
-    cache: Sharded<FxHashMap<K, &'tcx (V, DepNodeIndex)>>,
-    #[cfg(not(parallel_compiler))]
-    cache: Lock<FxHashMap<K, &'tcx (V, DepNodeIndex)>>,
+pub struct SingleCacheSelector;
+
+impl<'tcx, V: 'tcx> CacheSelector<'tcx, V> for SingleCacheSelector {
+    type Cache = SingleCache<V>
+    where
+        V: Copy;
 }
 
-impl<'tcx, K, V> Default for ArenaCache<'tcx, K, V> {
+pub struct SingleCache<V> {
+    cache: Lock<Option<(V, DepNodeIndex)>>,
+}
+
+impl<V> Default for SingleCache<V> {
     fn default() -> Self {
-        ArenaCache { arena: WorkerLocal::new(|_| TypedArena::default()), cache: Default::default() }
+        SingleCache { cache: Lock::new(None) }
     }
 }
 
-impl<'tcx, K: Eq + Hash, V: Debug + 'tcx> QueryStorage for ArenaCache<'tcx, K, V> {
+impl<V: Copy + Debug> QueryStorage for SingleCache<V> {
     type Value = V;
-    type Stored = &'tcx V;
-
-    #[inline]
-    fn store_nocache(&self, value: Self::Value) -> Self::Stored {
-        let value = self.arena.alloc((value, DepNodeIndex::INVALID));
-        let value = unsafe { &*(&value.0 as *const _) };
-        &value
-    }
 }
 
-impl<'tcx, K, V: 'tcx> QueryCache for ArenaCache<'tcx, K, V>
+impl<V> QueryCache for SingleCache<V>
 where
-    K: Eq + Hash + Clone + Debug,
-    V: Debug,
+    V: Copy + Debug,
 {
-    type Key = K;
+    type Key = ();
 
     #[inline(always)]
-    fn lookup<R, OnHit>(&self, key: &K, on_hit: OnHit) -> Result<R, ()>
-    where
-        OnHit: FnOnce(&&'tcx V, DepNodeIndex) -> R,
-    {
-        let key_hash = sharded::make_hash(key);
-        #[cfg(parallel_compiler)]
-        let lock = self.cache.get_shard_by_hash(key_hash).lock();
-        #[cfg(not(parallel_compiler))]
-        let lock = self.cache.lock();
-        let result = lock.raw_entry().from_key_hashed_nocheck(key_hash, key);
-
-        if let Some((_, value)) = result {
-            let hit_result = on_hit(&&value.0, value.1);
-            Ok(hit_result)
-        } else {
-            Err(())
-        }
+    fn lookup(&self, _key: &()) -> Option<(V, DepNodeIndex)> {
+        *self.cache.lock()
     }
 
     #[inline]
-    fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored {
-        let value = self.arena.alloc((value, index));
-        let value = unsafe { &*(value as *const _) };
-        #[cfg(parallel_compiler)]
-        let mut lock = self.cache.get_shard_by_value(&key).lock();
-        #[cfg(not(parallel_compiler))]
-        let mut lock = self.cache.lock();
-        // We may be overwriting another value. This is all right, since the dep-graph
-        // will check that the fingerprint matches.
-        lock.insert(key, value);
-        &value.0
+    fn complete(&self, _key: (), value: V, index: DepNodeIndex) {
+        *self.cache.lock() = Some((value, index));
     }
 
     fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
-        #[cfg(parallel_compiler)]
-        {
-            let shards = self.cache.lock_shards();
-            for shard in shards.iter() {
-                for (k, v) in shard.iter() {
-                    f(k, &v.0, v.1);
-                }
-            }
-        }
-        #[cfg(not(parallel_compiler))]
-        {
-            let map = self.cache.lock();
-            for (k, v) in map.iter() {
-                f(k, &v.0, v.1);
-            }
-        }
+        self.cache.lock().as_ref().map(|value| f(&(), &value.0, value.1));
     }
 }
 
@@ -234,8 +157,7 @@ pub struct VecCacheSelector<K>(PhantomData<K>);
 impl<'tcx, K: Idx, V: 'tcx> CacheSelector<'tcx, V> for VecCacheSelector<K> {
     type Cache = VecCache<K, V>
     where
-        V: Clone;
-    type ArenaCache = VecArenaCache<'tcx, K, V>;
+        V: Copy;
 }
 
 pub struct VecCache<K: Idx, V> {
@@ -251,138 +173,33 @@ impl<K: Idx, V> Default for VecCache<K, V> {
     }
 }
 
-impl<K: Eq + Idx, V: Clone + Debug> QueryStorage for VecCache<K, V> {
+impl<K: Eq + Idx, V: Copy + Debug> QueryStorage for VecCache<K, V> {
     type Value = V;
-    type Stored = V;
-
-    #[inline]
-    fn store_nocache(&self, value: Self::Value) -> Self::Stored {
-        // We have no dedicated storage
-        value
-    }
 }
 
 impl<K, V> QueryCache for VecCache<K, V>
 where
-    K: Eq + Idx + Clone + Debug,
-    V: Clone + Debug,
-{
-    type Key = K;
-
-    #[inline(always)]
-    fn lookup<R, OnHit>(&self, key: &K, on_hit: OnHit) -> Result<R, ()>
-    where
-        OnHit: FnOnce(&V, DepNodeIndex) -> R,
-    {
-        #[cfg(parallel_compiler)]
-        let lock = self.cache.get_shard_by_hash(key.index() as u64).lock();
-        #[cfg(not(parallel_compiler))]
-        let lock = self.cache.lock();
-        if let Some(Some(value)) = lock.get(*key) {
-            let hit_result = on_hit(&value.0, value.1);
-            Ok(hit_result)
-        } else {
-            Err(())
-        }
-    }
-
-    #[inline]
-    fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored {
-        #[cfg(parallel_compiler)]
-        let mut lock = self.cache.get_shard_by_hash(key.index() as u64).lock();
-        #[cfg(not(parallel_compiler))]
-        let mut lock = self.cache.lock();
-        lock.insert(key, (value.clone(), index));
-        value
-    }
-
-    fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
-        #[cfg(parallel_compiler)]
-        {
-            let shards = self.cache.lock_shards();
-            for shard in shards.iter() {
-                for (k, v) in shard.iter_enumerated() {
-                    if let Some(v) = v {
-                        f(&k, &v.0, v.1);
-                    }
-                }
-            }
-        }
-        #[cfg(not(parallel_compiler))]
-        {
-            let map = self.cache.lock();
-            for (k, v) in map.iter_enumerated() {
-                if let Some(v) = v {
-                    f(&k, &v.0, v.1);
-                }
-            }
-        }
-    }
-}
-
-pub struct VecArenaCache<'tcx, K: Idx, V> {
-    arena: WorkerLocal<TypedArena<(V, DepNodeIndex)>>,
-    #[cfg(parallel_compiler)]
-    cache: Sharded<IndexVec<K, Option<&'tcx (V, DepNodeIndex)>>>,
-    #[cfg(not(parallel_compiler))]
-    cache: Lock<IndexVec<K, Option<&'tcx (V, DepNodeIndex)>>>,
-}
-
-impl<'tcx, K: Idx, V> Default for VecArenaCache<'tcx, K, V> {
-    fn default() -> Self {
-        VecArenaCache {
-            arena: WorkerLocal::new(|_| TypedArena::default()),
-            cache: Default::default(),
-        }
-    }
-}
-
-impl<'tcx, K: Eq + Idx, V: Debug + 'tcx> QueryStorage for VecArenaCache<'tcx, K, V> {
-    type Value = V;
-    type Stored = &'tcx V;
-
-    #[inline]
-    fn store_nocache(&self, value: Self::Value) -> Self::Stored {
-        let value = self.arena.alloc((value, DepNodeIndex::INVALID));
-        let value = unsafe { &*(&value.0 as *const _) };
-        &value
-    }
-}
-
-impl<'tcx, K, V: 'tcx> QueryCache for VecArenaCache<'tcx, K, V>
-where
-    K: Eq + Idx + Clone + Debug,
-    V: Debug,
+    K: Eq + Idx + Copy + Debug,
+    V: Copy + Debug,
 {
     type Key = K;
 
     #[inline(always)]
-    fn lookup<R, OnHit>(&self, key: &K, on_hit: OnHit) -> Result<R, ()>
-    where
-        OnHit: FnOnce(&&'tcx V, DepNodeIndex) -> R,
-    {
+    fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> {
         #[cfg(parallel_compiler)]
         let lock = self.cache.get_shard_by_hash(key.index() as u64).lock();
         #[cfg(not(parallel_compiler))]
         let lock = self.cache.lock();
-        if let Some(Some(value)) = lock.get(*key) {
-            let hit_result = on_hit(&&value.0, value.1);
-            Ok(hit_result)
-        } else {
-            Err(())
-        }
+        if let Some(Some(value)) = lock.get(*key) { Some(*value) } else { None }
     }
 
     #[inline]
-    fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored {
-        let value = self.arena.alloc((value, index));
-        let value = unsafe { &*(value as *const _) };
+    fn complete(&self, key: K, value: V, index: DepNodeIndex) {
         #[cfg(parallel_compiler)]
         let mut lock = self.cache.get_shard_by_hash(key.index() as u64).lock();
         #[cfg(not(parallel_compiler))]
         let mut lock = self.cache.lock();
-        lock.insert(key, value);
-        &value.0
+        lock.insert(key, (value, index));
     }
 
     fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
diff --git a/compiler/rustc_query_system/src/query/config.rs b/compiler/rustc_query_system/src/query/config.rs
index 8c0330e43..d56373873 100644
--- a/compiler/rustc_query_system/src/query/config.rs
+++ b/compiler/rustc_query_system/src/query/config.rs
@@ -19,11 +19,12 @@ pub type TryLoadFromDisk<Qcx, Q> =
 pub trait QueryConfig<Qcx: QueryContext> {
     const NAME: &'static str;
 
-    type Key: DepNodeParams<Qcx::DepContext> + Eq + Hash + Clone + Debug;
-    type Value: Debug;
-    type Stored: Debug + Clone + std::borrow::Borrow<Self::Value>;
+    // `Key` and `Value` are `Copy` instead of `Clone` to ensure copying them stays cheap,
+    // but it isn't necessary.
+    type Key: DepNodeParams<Qcx::DepContext> + Eq + Hash + Copy + Debug;
+    type Value: Debug + Copy;
 
-    type Cache: QueryCache<Key = Self::Key, Stored = Self::Stored, Value = Self::Value>;
+    type Cache: QueryCache<Key = Self::Key, Value = Self::Value>;
 
     // Don't use this method to access query results, instead use the methods on TyCtxt
     fn query_state<'a>(tcx: Qcx) -> &'a QueryState<Self::Key, Qcx::DepKind>
@@ -38,9 +39,9 @@ pub trait QueryConfig<Qcx: QueryContext> {
     fn cache_on_disk(tcx: Qcx::DepContext, key: &Self::Key) -> bool;
 
     // Don't use this method to compute query results, instead use the methods on TyCtxt
-    fn execute_query(tcx: Qcx::DepContext, k: Self::Key) -> Self::Stored;
+    fn execute_query(tcx: Qcx::DepContext, k: Self::Key) -> Self::Value;
 
-    fn compute(tcx: Qcx, key: &Self::Key) -> fn(Qcx::DepContext, Self::Key) -> Self::Value;
+    fn compute(tcx: Qcx, key: Self::Key) -> Self::Value;
 
     fn try_load_from_disk(qcx: Qcx, idx: &Self::Key) -> TryLoadFromDisk<Qcx, Self>;
 
diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs
index d308af192..383c63cd2 100644
--- a/compiler/rustc_query_system/src/query/mod.rs
+++ b/compiler/rustc_query_system/src/query/mod.rs
@@ -8,7 +8,8 @@ pub use self::job::{print_query_stack, QueryInfo, QueryJob, QueryJobId, QueryJob
 
 mod caches;
 pub use self::caches::{
-    CacheSelector, DefaultCacheSelector, QueryCache, QueryStorage, VecCacheSelector,
+    CacheSelector, DefaultCacheSelector, QueryCache, QueryStorage, SingleCacheSelector,
+    VecCacheSelector,
 };
 
 mod config;
@@ -100,22 +101,22 @@ impl QuerySideEffects {
 }
 
 pub trait QueryContext: HasDepContext {
-    fn next_job_id(&self) -> QueryJobId;
+    fn next_job_id(self) -> QueryJobId;
 
     /// Get the query information from the TLS context.
-    fn current_query_job(&self) -> Option<QueryJobId>;
+    fn current_query_job(self) -> Option<QueryJobId>;
 
-    fn try_collect_active_jobs(&self) -> Option<QueryMap<Self::DepKind>>;
+    fn try_collect_active_jobs(self) -> Option<QueryMap<Self::DepKind>>;
 
     /// Load side effects associated to the node in the previous session.
-    fn load_side_effects(&self, prev_dep_node_index: SerializedDepNodeIndex) -> QuerySideEffects;
+    fn load_side_effects(self, prev_dep_node_index: SerializedDepNodeIndex) -> QuerySideEffects;
 
     /// Register diagnostics for the given node, for use in next session.
-    fn store_side_effects(&self, dep_node_index: DepNodeIndex, side_effects: QuerySideEffects);
+    fn store_side_effects(self, dep_node_index: DepNodeIndex, side_effects: QuerySideEffects);
 
     /// Register diagnostics for the given node, for use in next session.
     fn store_side_effects_for_anon_node(
-        &self,
+        self,
         dep_node_index: DepNodeIndex,
         side_effects: QuerySideEffects,
     );
@@ -124,12 +125,12 @@ pub trait QueryContext: HasDepContext {
     /// new query job while it executes. It returns the diagnostics
     /// captured during execution and the actual result.
     fn start_query<R>(
-        &self,
+        self,
         token: QueryJobId,
         depth_limit: bool,
         diagnostics: Option<&Lock<ThinVec<Diagnostic>>>,
         compute: impl FnOnce() -> R,
     ) -> R;
 
-    fn depth_limit_error(&self, job: QueryJobId);
+    fn depth_limit_error(self, job: QueryJobId);
 }
diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs
index b3b939eae..5f003fa70 100644
--- a/compiler/rustc_query_system/src/query/plumbing.rs
+++ b/compiler/rustc_query_system/src/query/plumbing.rs
@@ -2,7 +2,7 @@
 //! generate the actual methods on tcx which find and execute the provider,
 //! manage the caches, and so forth.
 
-use crate::dep_graph::{DepContext, DepKind, DepNode, DepNodeIndex};
+use crate::dep_graph::{DepContext, DepKind, DepNode, DepNodeIndex, DepNodeParams};
 use crate::ich::StableHashingContext;
 use crate::query::caches::QueryCache;
 use crate::query::job::{report_cycle, QueryInfo, QueryJob, QueryJobId, QueryJobInfo};
@@ -15,17 +15,16 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::profiling::TimingGuard;
 #[cfg(parallel_compiler)]
 use rustc_data_structures::sharded::Sharded;
+use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_data_structures::sync::Lock;
 use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed, FatalError};
 use rustc_session::Session;
 use rustc_span::{Span, DUMMY_SP};
-use std::borrow::Borrow;
 use std::cell::Cell;
 use std::collections::hash_map::Entry;
 use std::fmt::Debug;
 use std::hash::Hash;
 use std::mem;
-use std::ptr;
 use thin_vec::ThinVec;
 
 use super::QueryConfig;
@@ -49,7 +48,7 @@ enum QueryResult<D: DepKind> {
 
 impl<K, D> QueryState<K, D>
 where
-    K: Eq + Hash + Clone + Debug,
+    K: Eq + Hash + Copy + Debug,
     D: DepKind,
 {
     pub fn all_inactive(&self) -> bool {
@@ -78,7 +77,7 @@ where
             for shard in shards.iter() {
                 for (k, v) in shard.iter() {
                     if let QueryResult::Started(ref job) = *v {
-                        let query = make_query(qcx, k.clone());
+                        let query = make_query(qcx, *k);
                         jobs.insert(job.id, QueryJobInfo { query, job: job.clone() });
                     }
                 }
@@ -92,7 +91,7 @@ where
             // really hurt much.)
             for (k, v) in self.active.try_lock()?.iter() {
                 if let QueryResult::Started(ref job) = *v {
-                    let query = make_query(qcx, k.clone());
+                    let query = make_query(qcx, *k);
                     jobs.insert(job.id, QueryJobInfo { query, job: job.clone() });
                 }
             }
@@ -112,7 +111,7 @@ impl<K, D: DepKind> Default for QueryState<K, D> {
 /// This will poison the relevant query if dropped.
 struct JobOwner<'tcx, K, D: DepKind>
 where
-    K: Eq + Hash + Clone,
+    K: Eq + Hash + Copy,
 {
     state: &'tcx QueryState<K, D>,
     key: K,
@@ -121,20 +120,17 @@ where
 
 #[cold]
 #[inline(never)]
-fn mk_cycle<Qcx, V, R, D: DepKind>(
+fn mk_cycle<Qcx, R, D: DepKind>(
     qcx: Qcx,
     cycle_error: CycleError<D>,
     handler: HandleCycleError,
-    cache: &dyn crate::query::QueryStorage<Value = V, Stored = R>,
 ) -> R
 where
     Qcx: QueryContext + crate::query::HasDepContext<DepKind = D>,
-    V: std::fmt::Debug + Value<Qcx::DepContext, Qcx::DepKind>,
-    R: Clone,
+    R: std::fmt::Debug + Value<Qcx::DepContext, Qcx::DepKind>,
 {
     let error = report_cycle(qcx.dep_context().sess(), &cycle_error);
-    let value = handle_cycle_error(*qcx.dep_context(), &cycle_error, error, handler);
-    cache.store_nocache(value)
+    handle_cycle_error(*qcx.dep_context(), &cycle_error, error, handler)
 }
 
 fn handle_cycle_error<Tcx, V>(
@@ -167,7 +163,7 @@ where
 
 impl<'tcx, K, D: DepKind> JobOwner<'tcx, K, D>
 where
-    K: Eq + Hash + Clone,
+    K: Eq + Hash + Copy,
 {
     /// Either gets a `JobOwner` corresponding the query, allowing us to
     /// start executing the query, or returns with the result of the query.
@@ -192,14 +188,14 @@ where
         #[cfg(not(parallel_compiler))]
         let mut state_lock = state.active.lock();
         let lock = &mut *state_lock;
+        let current_job_id = qcx.current_query_job();
 
         match lock.entry(key) {
             Entry::Vacant(entry) => {
                 let id = qcx.next_job_id();
-                let job = qcx.current_query_job();
-                let job = QueryJob::new(id, span, job);
+                let job = QueryJob::new(id, span, current_job_id);
 
-                let key = entry.key().clone();
+                let key = *entry.key();
                 entry.insert(QueryResult::Started(job));
 
                 let owner = JobOwner { state, id, key };
@@ -216,7 +212,7 @@ where
                         // so we just return the error.
                         return TryGetJob::Cycle(id.find_cycle_in_stack(
                             qcx.try_collect_active_jobs().unwrap(),
-                            &qcx.current_query_job(),
+                            &current_job_id,
                             span,
                         ));
                     }
@@ -234,7 +230,7 @@ where
 
                         // With parallel queries we might just have to wait on some other
                         // thread.
-                        let result = latch.wait_on(qcx.current_query_job(), span);
+                        let result = latch.wait_on(current_job_id, span);
 
                         match result {
                             Ok(()) => TryGetJob::JobCompleted(query_blocked_prof_timer),
@@ -249,40 +245,38 @@ where
 
     /// Completes the query by updating the query cache with the `result`,
     /// signals the waiter and forgets the JobOwner, so it won't poison the query
-    fn complete<C>(self, cache: &C, result: C::Value, dep_node_index: DepNodeIndex) -> C::Stored
+    fn complete<C>(self, cache: &C, result: C::Value, dep_node_index: DepNodeIndex)
     where
         C: QueryCache<Key = K>,
     {
-        // We can move out of `self` here because we `mem::forget` it below
-        let key = unsafe { ptr::read(&self.key) };
+        let key = self.key;
         let state = self.state;
 
         // Forget ourself so our destructor won't poison the query
         mem::forget(self);
 
-        let (job, result) = {
-            let job = {
-                #[cfg(parallel_compiler)]
-                let mut lock = state.active.get_shard_by_value(&key).lock();
-                #[cfg(not(parallel_compiler))]
-                let mut lock = state.active.lock();
-                match lock.remove(&key).unwrap() {
-                    QueryResult::Started(job) => job,
-                    QueryResult::Poisoned => panic!(),
-                }
-            };
-            let result = cache.complete(key, result, dep_node_index);
-            (job, result)
+        // Mark as complete before we remove the job from the active state
+        // so no other thread can re-execute this query.
+        cache.complete(key, result, dep_node_index);
+
+        let job = {
+            #[cfg(parallel_compiler)]
+            let mut lock = state.active.get_shard_by_value(&key).lock();
+            #[cfg(not(parallel_compiler))]
+            let mut lock = state.active.lock();
+            match lock.remove(&key).unwrap() {
+                QueryResult::Started(job) => job,
+                QueryResult::Poisoned => panic!(),
+            }
         };
 
         job.signal_complete();
-        result
     }
 }
 
 impl<'tcx, K, D> Drop for JobOwner<'tcx, K, D>
 where
-    K: Eq + Hash + Clone,
+    K: Eq + Hash + Copy,
     D: DepKind,
 {
     #[inline(never)]
@@ -299,7 +293,7 @@ where
                 QueryResult::Started(job) => job,
                 QueryResult::Poisoned => panic!(),
             };
-            shard.insert(self.key.clone(), QueryResult::Poisoned);
+            shard.insert(self.key, QueryResult::Poisoned);
             job
         };
         // Also signal the completion of the job, so waiters
@@ -318,7 +312,7 @@ pub(crate) struct CycleError<D: DepKind> {
 /// The result of `try_start`.
 enum TryGetJob<'tcx, K, D>
 where
-    K: Eq + Hash + Clone,
+    K: Eq + Hash + Copy,
     D: DepKind,
 {
     /// The query is not yet started. Contains a guard to the cache eventually used to start it.
@@ -339,74 +333,62 @@ where
 /// which will be used if the query is not in the cache and we need
 /// to compute it.
 #[inline]
-pub fn try_get_cached<Tcx, C, R, OnHit>(
-    tcx: Tcx,
-    cache: &C,
-    key: &C::Key,
-    // `on_hit` can be called while holding a lock to the query cache
-    on_hit: OnHit,
-) -> Result<R, ()>
+pub fn try_get_cached<Tcx, C>(tcx: Tcx, cache: &C, key: &C::Key) -> Option<C::Value>
 where
     C: QueryCache,
     Tcx: DepContext,
-    OnHit: FnOnce(&C::Stored) -> R,
 {
-    cache.lookup(&key, |value, index| {
-        if std::intrinsics::unlikely(tcx.profiler().enabled()) {
+    match cache.lookup(&key) {
+        Some((value, index)) => {
             tcx.profiler().query_cache_hit(index.into());
+            tcx.dep_graph().read_index(index);
+            Some(value)
         }
-        tcx.dep_graph().read_index(index);
-        on_hit(value)
-    })
+        None => None,
+    }
 }
 
+#[inline(never)]
 fn try_execute_query<Q, Qcx>(
     qcx: Qcx,
-    state: &QueryState<Q::Key, Qcx::DepKind>,
-    cache: &Q::Cache,
     span: Span,
     key: Q::Key,
     dep_node: Option<DepNode<Qcx::DepKind>>,
-) -> (Q::Stored, Option<DepNodeIndex>)
+) -> (Q::Value, Option<DepNodeIndex>)
 where
     Q: QueryConfig<Qcx>,
     Qcx: QueryContext,
 {
-    match JobOwner::<'_, Q::Key, Qcx::DepKind>::try_start(&qcx, state, span, key.clone()) {
+    let state = Q::query_state(qcx);
+    match JobOwner::<'_, Q::Key, Qcx::DepKind>::try_start(&qcx, state, span, key) {
         TryGetJob::NotYetStarted(job) => {
-            let (result, dep_node_index) =
-                execute_job::<Q, Qcx>(qcx, key.clone(), dep_node, job.id);
+            let (result, dep_node_index) = execute_job::<Q, Qcx>(qcx, key, dep_node, job.id);
+            let cache = Q::query_cache(qcx);
             if Q::FEEDABLE {
-                // We may have put a value inside the cache from inside the execution.
-                // Verify that it has the same hash as what we have now, to ensure consistency.
-                let _ = cache.lookup(&key, |cached_result, _| {
-                    let hasher = Q::HASH_RESULT.expect("feedable forbids no_hash");
-
-                    let old_hash = qcx.dep_context().with_stable_hashing_context(|mut hcx| hasher(&mut hcx, cached_result.borrow()));
-                    let new_hash = qcx.dep_context().with_stable_hashing_context(|mut hcx| hasher(&mut hcx, &result));
-                    debug_assert_eq!(
-                        old_hash, new_hash,
-                        "Computed query value for {:?}({:?}) is inconsistent with fed value,\ncomputed={:#?}\nfed={:#?}",
-                        Q::DEP_KIND, key, result, cached_result,
+                // We should not compute queries that also got a value via feeding.
+                // This can't happen, as query feeding adds the very dependencies to the fed query
+                // as its feeding query had. So if the fed query is red, so is its feeder, which will
+                // get evaluated first, and re-feed the query.
+                if let Some((cached_result, _)) = cache.lookup(&key) {
+                    panic!(
+                        "fed query later has its value computed. The already cached value: {cached_result:?}"
                     );
-                });
+                }
             }
-            let result = job.complete(cache, result, dep_node_index);
+            job.complete(cache, result, dep_node_index);
             (result, Some(dep_node_index))
         }
         TryGetJob::Cycle(error) => {
-            let result = mk_cycle(qcx, error, Q::HANDLE_CYCLE_ERROR, cache);
+            let result = mk_cycle(qcx, error, Q::HANDLE_CYCLE_ERROR);
             (result, None)
         }
         #[cfg(parallel_compiler)]
         TryGetJob::JobCompleted(query_blocked_prof_timer) => {
-            let (v, index) = cache
-                .lookup(&key, |value, index| (value.clone(), index))
-                .unwrap_or_else(|_| panic!("value must be in cache after waiting"));
+            let Some((v, index)) = Q::query_cache(qcx).lookup(&key) else {
+                panic!("value must be in cache after waiting")
+            };
 
-            if std::intrinsics::unlikely(qcx.dep_context().profiler().enabled()) {
-                qcx.dep_context().profiler().query_cache_hit(index.into());
-            }
+            qcx.dep_context().profiler().query_cache_hit(index.into());
             query_blocked_prof_timer.finish_with_query_invocation_id(index.into());
 
             (v, Some(index))
@@ -414,6 +396,7 @@ where
     }
 }
 
+#[inline(always)]
 fn execute_job<Q, Qcx>(
     qcx: Qcx,
     key: Q::Key,
@@ -428,12 +411,27 @@ where
 
     // Fast path for when incr. comp. is off.
     if !dep_graph.is_fully_enabled() {
+        // Fingerprint the key, just to assert that it doesn't
+        // have anything we don't consider hashable
+        if cfg!(debug_assertions) {
+            let _ = key.to_fingerprint(*qcx.dep_context());
+        }
+
         let prof_timer = qcx.dep_context().profiler().query_provider();
-        let result = qcx.start_query(job_id, Q::DEPTH_LIMIT, None, || {
-            Q::compute(qcx, &key)(*qcx.dep_context(), key)
-        });
+        let result = qcx.start_query(job_id, Q::DEPTH_LIMIT, None, || Q::compute(qcx, key));
         let dep_node_index = dep_graph.next_virtual_depnode_index();
         prof_timer.finish_with_query_invocation_id(dep_node_index.into());
+
+        // Similarly, fingerprint the result to assert that
+        // it doesn't have anything not considered hashable.
+        if cfg!(debug_assertions)
+            && let Some(hash_result) = Q::HASH_RESULT
+        {
+            qcx.dep_context().with_stable_hashing_context(|mut hcx| {
+                hash_result(&mut hcx, &result);
+            });
+        }
+
         return (result, dep_node_index);
     }
 
@@ -457,17 +455,15 @@ where
     let (result, dep_node_index) =
         qcx.start_query(job_id, Q::DEPTH_LIMIT, Some(&diagnostics), || {
             if Q::ANON {
-                return dep_graph.with_anon_task(*qcx.dep_context(), Q::DEP_KIND, || {
-                    Q::compute(qcx, &key)(*qcx.dep_context(), key)
-                });
+                return dep_graph
+                    .with_anon_task(*qcx.dep_context(), Q::DEP_KIND, || Q::compute(qcx, key));
             }
 
             // `to_dep_node` is expensive for some `DepKind`s.
             let dep_node =
                 dep_node_opt.unwrap_or_else(|| Q::construct_dep_node(*qcx.dep_context(), &key));
 
-            let task = Q::compute(qcx, &key);
-            dep_graph.with_task(dep_node, *qcx.dep_context(), key, task, Q::HASH_RESULT)
+            dep_graph.with_task(dep_node, qcx, key, Q::compute, Q::HASH_RESULT)
         });
 
     prof_timer.finish_with_query_invocation_id(dep_node_index.into());
@@ -486,6 +482,7 @@ where
     (result, dep_node_index)
 }
 
+#[inline(always)]
 fn try_load_from_disk_and_cache_in_memory<Q, Qcx>(
     qcx: Qcx,
     key: &Q::Key,
@@ -558,7 +555,7 @@ where
     let prof_timer = qcx.dep_context().profiler().query_provider();
 
     // The dep-graph for this computation is already in-place.
-    let result = dep_graph.with_ignore(|| Q::compute(qcx, key)(*qcx.dep_context(), key.clone()));
+    let result = dep_graph.with_ignore(|| Q::compute(qcx, *key));
 
     prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
@@ -576,6 +573,7 @@ where
     Some((result, dep_node_index))
 }
 
+#[inline]
 #[instrument(skip(tcx, result, hash_result), level = "debug")]
 pub(crate) fn incremental_verify_ich<Tcx, V: Debug>(
     tcx: Tcx,
@@ -730,7 +728,8 @@ pub enum QueryMode {
     Ensure,
 }
 
-pub fn get_query<Q, Qcx, D>(qcx: Qcx, span: Span, key: Q::Key, mode: QueryMode) -> Option<Q::Stored>
+#[inline(always)]
+pub fn get_query<Q, Qcx, D>(qcx: Qcx, span: Span, key: Q::Key, mode: QueryMode) -> Option<Q::Value>
 where
     D: DepKind,
     Q: QueryConfig<Qcx>,
@@ -747,14 +746,8 @@ where
         None
     };
 
-    let (result, dep_node_index) = try_execute_query::<Q, Qcx>(
-        qcx,
-        Q::query_state(qcx),
-        Q::query_cache(qcx),
-        span,
-        key,
-        dep_node,
-    );
+    let (result, dep_node_index) =
+        ensure_sufficient_stack(|| try_execute_query::<Q, Qcx>(qcx, span, key, dep_node));
     if let Some(dep_node_index) = dep_node_index {
         qcx.dep_context().dep_graph().read_index(dep_node_index)
     }
@@ -770,20 +763,12 @@ where
 {
     // We may be concurrently trying both execute and force a query.
     // Ensure that only one of them runs the query.
-    let cache = Q::query_cache(qcx);
-    let cached = cache.lookup(&key, |_, index| {
-        if std::intrinsics::unlikely(qcx.dep_context().profiler().enabled()) {
-            qcx.dep_context().profiler().query_cache_hit(index.into());
-        }
-    });
-
-    match cached {
-        Ok(()) => return,
-        Err(()) => {}
+    if let Some((_, index)) = Q::query_cache(qcx).lookup(&key) {
+        qcx.dep_context().profiler().query_cache_hit(index.into());
+        return;
     }
 
-    let state = Q::query_state(qcx);
     debug_assert!(!Q::ANON);
 
-    try_execute_query::<Q, _>(qcx, state, cache, DUMMY_SP, key, Some(dep_node));
+    ensure_sufficient_stack(|| try_execute_query::<Q, _>(qcx, DUMMY_SP, key, Some(dep_node)));
 }
diff --git a/compiler/rustc_resolve/Cargo.toml b/compiler/rustc_resolve/Cargo.toml
index 7c3a0f8f2..5c4ec44d2 100644
--- a/compiler/rustc_resolve/Cargo.toml
+++ b/compiler/rustc_resolve/Cargo.toml
@@ -7,6 +7,7 @@ edition = "2021"
 
 [dependencies]
 bitflags = "1.2.1"
+pulldown-cmark = { version = "0.9.2", default-features = false }
 rustc_arena = { path = "../rustc_arena" }
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
@@ -24,5 +25,5 @@ rustc_query_system = { path = "../rustc_query_system" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-thin-vec = "0.2.8"
+thin-vec = "0.2.12"
 tracing = "0.1"
diff --git a/compiler/rustc_error_messages/locales/en-US/resolve.ftl b/compiler/rustc_resolve/locales/en-US.ftl
index 817bb83ed..817bb83ed 100644
--- a/compiler/rustc_error_messages/locales/en-US/resolve.ftl
+++ b/compiler/rustc_resolve/locales/en-US.ftl
diff --git a/compiler/rustc_resolve/src/build_reduced_graph.rs b/compiler/rustc_resolve/src/build_reduced_graph.rs
index b1b04c92a..b1e023f2c 100644
--- a/compiler/rustc_resolve/src/build_reduced_graph.rs
+++ b/compiler/rustc_resolve/src/build_reduced_graph.rs
@@ -65,7 +65,7 @@ impl<'a, Id: Into<DefId>> ToNameBinding<'a> for (Res, ty::Visibility<Id>, Span,
     }
 }
 
-impl<'a> Resolver<'a> {
+impl<'a, 'tcx> Resolver<'a, 'tcx> {
     /// Defines `name` in namespace `ns` of module `parent` to be `def` if it is not yet defined;
     /// otherwise, reports an error.
     pub(crate) fn define<T>(&mut self, parent: Module<'a>, ident: Ident, ns: Namespace, def: T)
@@ -95,7 +95,7 @@ impl<'a> Resolver<'a> {
     /// Reachable macros with block module parents exist due to `#[macro_export] macro_rules!`,
     /// but they cannot use def-site hygiene, so the assumption holds
     /// (<https://github.com/rust-lang/rust/pull/77984#issuecomment-712445508>).
-    pub fn get_nearest_non_block_module(&mut self, mut def_id: DefId) -> Module<'a> {
+    pub(crate) fn get_nearest_non_block_module(&mut self, mut def_id: DefId) -> Module<'a> {
         loop {
             match self.get_module(def_id) {
                 Some(module) => return module,
@@ -104,7 +104,7 @@ impl<'a> Resolver<'a> {
         }
     }
 
-    pub fn expect_module(&mut self, def_id: DefId) -> Module<'a> {
+    pub(crate) fn expect_module(&mut self, def_id: DefId) -> Module<'a> {
         self.get_module(def_id).expect("argument `DefId` is not a module")
     }
 
@@ -130,11 +130,13 @@ impl<'a> Resolver<'a> {
                         def_key.disambiguated_data.data.get_opt_name().expect("module without name")
                     };
 
+                    let expn_id = self.cstore().module_expansion_untracked(def_id, &self.tcx.sess);
+                    let span = self.cstore().get_span_untracked(def_id, &self.tcx.sess);
                     Some(self.new_module(
                         parent,
                         ModuleKind::Def(def_kind, def_id, name),
-                        self.cstore().module_expansion_untracked(def_id, &self.session),
-                        self.cstore().get_span_untracked(def_id, &self.session),
+                        expn_id,
+                        span,
                         // FIXME: Account for `#[no_implicit_prelude]` attributes.
                         parent.map_or(false, |module| module.no_implicit_prelude),
                     ))
@@ -179,7 +181,8 @@ impl<'a> Resolver<'a> {
             return macro_data.clone();
         }
 
-        let (ext, macro_rules) = match self.cstore().load_macro_untracked(def_id, &self.session) {
+        let load_macro_untracked = self.cstore().load_macro_untracked(def_id, &self.tcx.sess);
+        let (ext, macro_rules) = match load_macro_untracked {
             LoadedMacro::MacroDef(item, edition) => (
                 Lrc::new(self.compile_macro(&item, edition).0),
                 matches!(item.kind, ItemKind::MacroDef(def) if def.macro_rules),
@@ -204,9 +207,9 @@ impl<'a> Resolver<'a> {
     }
 
     pub(crate) fn build_reduced_graph_external(&mut self, module: Module<'a>) {
-        for child in
-            Vec::from_iter(self.cstore().module_children_untracked(module.def_id(), self.session))
-        {
+        let children =
+            Vec::from_iter(self.cstore().module_children_untracked(module.def_id(), self.tcx.sess));
+        for child in children {
             let parent_scope = ParentScope::module(module, self);
             BuildReducedGraphVisitor { r: self, parent_scope }
                 .build_reduced_graph_for_external_crate_res(child);
@@ -214,18 +217,18 @@ impl<'a> Resolver<'a> {
     }
 }
 
-struct BuildReducedGraphVisitor<'a, 'b> {
-    r: &'b mut Resolver<'a>,
+struct BuildReducedGraphVisitor<'a, 'b, 'tcx> {
+    r: &'b mut Resolver<'a, 'tcx>,
     parent_scope: ParentScope<'a>,
 }
 
-impl<'a> AsMut<Resolver<'a>> for BuildReducedGraphVisitor<'a, '_> {
-    fn as_mut(&mut self) -> &mut Resolver<'a> {
+impl<'a, 'tcx> AsMut<Resolver<'a, 'tcx>> for BuildReducedGraphVisitor<'a, '_, 'tcx> {
+    fn as_mut(&mut self) -> &mut Resolver<'a, 'tcx> {
         self.r
     }
 }
 
-impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
+impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
     fn resolve_visibility(&mut self, vis: &ast::Visibility) -> ty::Visibility {
         self.try_resolve_visibility(vis, true).unwrap_or_else(|err| {
             self.r.report_vis_error(err);
@@ -265,7 +268,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
                 let ident = path.segments.get(0).expect("empty path in visibility").ident;
                 let crate_root = if ident.is_path_segment_keyword() {
                     None
-                } else if ident.span.rust_2015() {
+                } else if ident.span.is_rust_2015() {
                     Some(Segment::from_ident(Ident::new(
                         kw::PathRoot,
                         path.span.shrink_to_lo().with_ctxt(ident.span.ctxt()),
@@ -298,14 +301,15 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
                             self.r.record_partial_res(id, PartialRes::new(res));
                         }
                         if module.is_normal() {
-                            if res == Res::Err {
-                                Ok(ty::Visibility::Public)
-                            } else {
-                                let vis = ty::Visibility::Restricted(res.def_id());
-                                if self.r.is_accessible_from(vis, parent_scope.module) {
-                                    Ok(vis.expect_local())
-                                } else {
-                                    Err(VisResolutionError::AncestorOnly(path.span))
+                            match res {
+                                Res::Err => Ok(ty::Visibility::Public),
+                                _ => {
+                                    let vis = ty::Visibility::Restricted(res.def_id());
+                                    if self.r.is_accessible_from(vis, parent_scope.module) {
+                                        Ok(vis.expect_local())
+                                    } else {
+                                        Err(VisResolutionError::AncestorOnly(path.span))
+                                    }
                                 }
                             }
                         } else {
@@ -345,7 +349,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
 
     fn insert_field_names_extern(&mut self, def_id: DefId) {
         let field_names =
-            self.r.cstore().struct_field_names_untracked(def_id, self.r.session).collect();
+            self.r.cstore().struct_field_names_untracked(def_id, self.r.tcx.sess).collect();
         self.r.field_names.insert(def_id, field_names);
     }
 
@@ -434,10 +438,10 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
         // appears, so imports in braced groups can have roots prepended independently.
         let is_glob = matches!(use_tree.kind, ast::UseTreeKind::Glob);
         let crate_root = match prefix_iter.peek() {
-            Some(seg) if !seg.ident.is_path_segment_keyword() && seg.ident.span.rust_2015() => {
+            Some(seg) if !seg.ident.is_path_segment_keyword() && seg.ident.span.is_rust_2015() => {
                 Some(seg.ident.span.ctxt())
             }
-            None if is_glob && use_tree.span.rust_2015() => Some(use_tree.span.ctxt()),
+            None if is_glob && use_tree.span.is_rust_2015() => Some(use_tree.span.ctxt()),
             _ => None,
         }
         .map(|ctxt| {
@@ -538,14 +542,15 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
                         }
 
                         self.r
-                            .session
+                            .tcx
+                            .sess
                             .struct_span_err(item.span, "`$crate` may not be imported")
                             .emit();
                     }
                 }
 
                 if ident.name == kw::Crate {
-                    self.r.session.span_err(
+                    self.r.tcx.sess.span_err(
                         ident.span,
                         "crate root imports need to be explicitly named: \
                          `use crate as name;`",
@@ -574,7 +579,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
             }
             ast::UseTreeKind::Glob => {
                 let kind = ImportKind::Glob {
-                    is_prelude: self.r.session.contains_name(&item.attrs, sym::prelude_import),
+                    is_prelude: self.r.tcx.sess.contains_name(&item.attrs, sym::prelude_import),
                     max_vis: Cell::new(None),
                     id,
                 };
@@ -689,7 +694,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
                     expansion.to_expn_id(),
                     item.span,
                     parent.no_implicit_prelude
-                        || self.r.session.contains_name(&item.attrs, sym::no_implicit_prelude),
+                        || self.r.tcx.sess.contains_name(&item.attrs, sym::no_implicit_prelude),
                 );
                 self.r.define(parent, ident, TypeNS, (module, vis, sp, expansion));
 
@@ -754,7 +759,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
                     // If the structure is marked as non_exhaustive then lower the visibility
                     // to within the crate.
                     let mut ctor_vis = if vis.is_public()
-                        && self.r.session.contains_name(&item.attrs, sym::non_exhaustive)
+                        && self.r.tcx.sess.contains_name(&item.attrs, sym::non_exhaustive)
                     {
                         ty::Visibility::Restricted(CRATE_DEF_ID)
                     } else {
@@ -836,7 +841,8 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
 
         let (used, module, binding) = if orig_name.is_none() && ident.name == kw::SelfLower {
             self.r
-                .session
+                .tcx
+                .sess
                 .struct_span_err(item.span, "`extern crate self;` requires renaming")
                 .span_suggestion(
                     item.span,
@@ -849,7 +855,10 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
         } else if orig_name == Some(kw::SelfLower) {
             Some(self.r.graph_root)
         } else {
-            let crate_id = self.r.crate_loader().process_extern_crate(item, local_def_id);
+            let tcx = self.r.tcx;
+            let crate_id = self.r.crate_loader(|c| {
+                c.process_extern_crate(item, local_def_id, &tcx.definitions_untracked())
+            });
             crate_id.map(|crate_id| {
                 self.r.extern_crate_map.insert(local_def_id, crate_id);
                 self.r.expect_module(crate_id.as_def_id())
@@ -886,7 +895,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
                 {
                     let msg = "macro-expanded `extern crate` items cannot \
                                        shadow names passed with `--extern`";
-                    self.r.session.span_err(item.span, msg);
+                    self.r.tcx.sess.span_err(item.span, msg);
                 }
             }
             let entry = self.r.extern_prelude.entry(ident.normalize_to_macros_2_0()).or_insert(
@@ -986,7 +995,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
                 | DefKind::LifetimeParam
                 | DefKind::GlobalAsm
                 | DefKind::Closure
-                | DefKind::Impl
+                | DefKind::Impl { .. }
                 | DefKind::Generator,
                 _,
             )
@@ -997,23 +1006,26 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
             | Res::Err => bug!("unexpected resolution: {:?}", res),
         }
         // Record some extra data for better diagnostics.
-        let cstore = self.r.cstore();
         match res {
             Res::Def(DefKind::Struct, def_id) => {
+                let cstore = self.r.cstore();
                 if let Some((ctor_kind, ctor_def_id)) = cstore.ctor_untracked(def_id) {
                     let ctor_res = Res::Def(DefKind::Ctor(CtorOf::Struct, ctor_kind), ctor_def_id);
                     let ctor_vis = cstore.visibility_untracked(ctor_def_id);
                     let field_visibilities =
                         cstore.struct_field_visibilities_untracked(def_id).collect();
+                    drop(cstore);
                     self.r
                         .struct_constructors
                         .insert(def_id, (ctor_res, ctor_vis, field_visibilities));
+                } else {
+                    drop(cstore);
                 }
                 self.insert_field_names_extern(def_id)
             }
             Res::Def(DefKind::Union, def_id) => self.insert_field_names_extern(def_id),
             Res::Def(DefKind::AssocFn, def_id) => {
-                if cstore.fn_has_self_parameter_untracked(def_id, self.r.session) {
+                if self.r.cstore().fn_has_self_parameter_untracked(def_id, self.r.tcx.sess) {
                     self.r.has_self.insert(def_id);
                 }
             }
@@ -1032,7 +1044,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
             let msg = format!("`{}` is already in scope", name);
             let note =
                 "macro-expanded `#[macro_use]`s may not shadow existing macros (see RFC 1560)";
-            self.r.session.struct_span_err(span, &msg).note(note).emit();
+            self.r.tcx.sess.struct_span_err(span, &msg).note(note).emit();
         }
     }
 
@@ -1044,7 +1056,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
             if attr.has_name(sym::macro_use) {
                 if self.parent_scope.module.parent.is_some() {
                     struct_span_err!(
-                        self.r.session,
+                        self.r.tcx.sess,
                         item.span,
                         E0468,
                         "an `extern crate` loading macros must be at the crate root"
@@ -1054,7 +1066,8 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
                 if let ItemKind::ExternCrate(Some(orig_name)) = item.kind {
                     if orig_name == kw::SelfLower {
                         self.r
-                            .session
+                            .tcx
+                            .sess
                             .struct_span_err(
                                 attr.span,
                                 "`#[macro_use]` is not supported on `extern crate self`",
@@ -1063,7 +1076,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
                     }
                 }
                 let ill_formed = |span| {
-                    struct_span_err!(self.r.session, span, E0466, "bad macro import").emit();
+                    struct_span_err!(self.r.tcx.sess, span, E0466, "bad macro import").emit();
                 };
                 match attr.meta() {
                     Some(meta) => match meta.kind {
@@ -1134,8 +1147,13 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
                         allow_shadowing,
                     );
                 } else {
-                    struct_span_err!(self.r.session, ident.span, E0469, "imported macro not found")
-                        .emit();
+                    struct_span_err!(
+                        self.r.tcx.sess,
+                        ident.span,
+                        E0469,
+                        "imported macro not found"
+                    )
+                    .emit();
                 }
             }
         }
@@ -1147,7 +1165,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
         for attr in attrs {
             if attr.has_name(sym::macro_escape) {
                 let msg = "`#[macro_escape]` is a deprecated synonym for `#[macro_use]`";
-                let mut err = self.r.session.struct_span_warn(attr.span, msg);
+                let mut err = self.r.tcx.sess.struct_span_warn(attr.span, msg);
                 if let ast::AttrStyle::Inner = attr.style {
                     err.help("try an outer attribute: `#[macro_use]`").emit();
                 } else {
@@ -1158,7 +1176,10 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
             }
 
             if !attr.is_word() {
-                self.r.session.span_err(attr.span, "arguments to `macro_use` are not allowed here");
+                self.r
+                    .tcx
+                    .sess
+                    .span_err(attr.span, "arguments to `macro_use` are not allowed here");
             }
             return true;
         }
@@ -1182,11 +1203,11 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
     }
 
     fn proc_macro_stub(&self, item: &ast::Item) -> Option<(MacroKind, Ident, Span)> {
-        if self.r.session.contains_name(&item.attrs, sym::proc_macro) {
+        if self.r.tcx.sess.contains_name(&item.attrs, sym::proc_macro) {
             return Some((MacroKind::Bang, item.ident, item.span));
-        } else if self.r.session.contains_name(&item.attrs, sym::proc_macro_attribute) {
+        } else if self.r.tcx.sess.contains_name(&item.attrs, sym::proc_macro_attribute) {
             return Some((MacroKind::Attr, item.ident, item.span));
-        } else if let Some(attr) = self.r.session.find_by_name(&item.attrs, sym::proc_macro_derive)
+        } else if let Some(attr) = self.r.tcx.sess.find_by_name(&item.attrs, sym::proc_macro_derive)
         {
             if let Some(nested_meta) = attr.meta_item_list().and_then(|list| list.get(0).cloned()) {
                 if let Some(ident) = nested_meta.ident() {
@@ -1221,7 +1242,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
         let def_id = self.r.local_def_id(item.id);
         let (ext, ident, span, macro_rules, rule_spans) = match &item.kind {
             ItemKind::MacroDef(def) => {
-                let (ext, rule_spans) = self.r.compile_macro(item, self.r.session.edition());
+                let (ext, rule_spans) = self.r.compile_macro(item, self.r.tcx.sess.edition());
                 let ext = Lrc::new(ext);
                 (ext, item.ident, item.span, def.macro_rules, rule_spans)
             }
@@ -1242,7 +1263,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
         if macro_rules {
             let ident = ident.normalize_to_macros_2_0();
             self.r.macro_names.insert(ident);
-            let is_macro_export = self.r.session.contains_name(&item.attrs, sym::macro_export);
+            let is_macro_export = self.r.tcx.sess.contains_name(&item.attrs, sym::macro_export);
             let vis = if is_macro_export {
                 ty::Visibility::Public
             } else {
@@ -1250,6 +1271,7 @@ impl<'a, 'b> BuildReducedGraphVisitor<'a, 'b> {
             };
             let binding = (res, vis, span, expansion).to_name_binding(self.r.arenas);
             self.r.set_binding_parent_module(binding, parent_scope.module);
+            self.r.all_macro_rules.insert(ident.name, res);
             if is_macro_export {
                 let import = self.r.arenas.alloc_import(Import {
                     kind: ImportKind::MacroExport,
@@ -1313,7 +1335,7 @@ macro_rules! method {
     };
 }
 
-impl<'a, 'b> Visitor<'b> for BuildReducedGraphVisitor<'a, 'b> {
+impl<'a, 'b, 'tcx> Visitor<'b> for BuildReducedGraphVisitor<'a, 'b, 'tcx> {
     method!(visit_expr: ast::Expr, ast::ExprKind::MacCall, walk_expr);
     method!(visit_pat: ast::Pat, ast::PatKind::MacCall, walk_pat);
     method!(visit_ty: ast::Ty, ast::TyKind::MacCall, walk_ty);
@@ -1505,7 +1527,7 @@ impl<'a, 'b> Visitor<'b> for BuildReducedGraphVisitor<'a, 'b> {
 
         // If the variant is marked as non_exhaustive then lower the visibility to within the crate.
         let ctor_vis = if vis.is_public()
-            && self.r.session.contains_name(&variant.attrs, sym::non_exhaustive)
+            && self.r.tcx.sess.contains_name(&variant.attrs, sym::non_exhaustive)
         {
             ty::Visibility::Restricted(CRATE_DEF_ID)
         } else {
diff --git a/compiler/rustc_resolve/src/check_unused.rs b/compiler/rustc_resolve/src/check_unused.rs
index eae4c9992..b2578e4c4 100644
--- a/compiler/rustc_resolve/src/check_unused.rs
+++ b/compiler/rustc_resolve/src/check_unused.rs
@@ -29,11 +29,12 @@ use crate::Resolver;
 
 use rustc_ast as ast;
 use rustc_ast::visit::{self, Visitor};
-use rustc_data_structures::fx::FxIndexMap;
+use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
 use rustc_data_structures::unord::UnordSet;
 use rustc_errors::{pluralize, MultiSpan};
-use rustc_session::lint::builtin::{MACRO_USE_EXTERN_CRATE, UNUSED_IMPORTS};
+use rustc_session::lint::builtin::{MACRO_USE_EXTERN_CRATE, UNUSED_EXTERN_CRATES, UNUSED_IMPORTS};
 use rustc_session::lint::BuiltinLintDiagnostics;
+use rustc_span::symbol::Ident;
 use rustc_span::{Span, DUMMY_SP};
 
 struct UnusedImport<'a> {
@@ -49,16 +50,33 @@ impl<'a> UnusedImport<'a> {
     }
 }
 
-struct UnusedImportCheckVisitor<'a, 'b> {
-    r: &'a mut Resolver<'b>,
+struct UnusedImportCheckVisitor<'a, 'b, 'tcx> {
+    r: &'a mut Resolver<'b, 'tcx>,
     /// All the (so far) unused imports, grouped path list
     unused_imports: FxIndexMap<ast::NodeId, UnusedImport<'a>>,
+    extern_crate_items: Vec<ExternCrateToLint>,
     base_use_tree: Option<&'a ast::UseTree>,
     base_id: ast::NodeId,
     item_span: Span,
 }
 
-impl<'a, 'b> UnusedImportCheckVisitor<'a, 'b> {
+struct ExternCrateToLint {
+    id: ast::NodeId,
+    /// Span from the item
+    span: Span,
+    /// Span to use to suggest complete removal.
+    span_with_attributes: Span,
+    /// Span of the visibility, if any.
+    vis_span: Span,
+    /// Whether the item has attrs.
+    has_attrs: bool,
+    /// Name used to refer to the crate.
+    ident: Ident,
+    /// Whether the statement renames the crate `extern crate orig_name as new_name;`.
+    renames: bool,
+}
+
+impl<'a, 'b, 'tcx> UnusedImportCheckVisitor<'a, 'b, 'tcx> {
     // We have information about whether `use` (import) items are actually
     // used now. If an import is not used at all, we signal a lint error.
     fn check_import(&mut self, id: ast::NodeId) {
@@ -94,20 +112,29 @@ impl<'a, 'b> UnusedImportCheckVisitor<'a, 'b> {
     }
 }
 
-impl<'a, 'b> Visitor<'a> for UnusedImportCheckVisitor<'a, 'b> {
+impl<'a, 'b, 'tcx> Visitor<'a> for UnusedImportCheckVisitor<'a, 'b, 'tcx> {
     fn visit_item(&mut self, item: &'a ast::Item) {
-        self.item_span = item.span_with_attributes();
-
-        // Ignore is_public import statements because there's no way to be sure
-        // whether they're used or not. Also ignore imports with a dummy span
-        // because this means that they were generated in some fashion by the
-        // compiler and we don't need to consider them.
-        if let ast::ItemKind::Use(..) = item.kind {
-            if item.vis.kind.is_pub() || item.span.is_dummy() {
-                return;
+        match item.kind {
+            // Ignore is_public import statements because there's no way to be sure
+            // whether they're used or not. Also ignore imports with a dummy span
+            // because this means that they were generated in some fashion by the
+            // compiler and we don't need to consider them.
+            ast::ItemKind::Use(..) if item.vis.kind.is_pub() || item.span.is_dummy() => return,
+            ast::ItemKind::ExternCrate(orig_name) => {
+                self.extern_crate_items.push(ExternCrateToLint {
+                    id: item.id,
+                    span: item.span,
+                    vis_span: item.vis.span,
+                    span_with_attributes: item.span_with_attributes(),
+                    has_attrs: !item.attrs.is_empty(),
+                    ident: item.ident,
+                    renames: orig_name.is_some(),
+                });
             }
+            _ => {}
         }
 
+        self.item_span = item.span_with_attributes();
         visit::walk_item(self, item);
     }
 
@@ -222,8 +249,11 @@ fn calc_unused_spans(
     }
 }
 
-impl Resolver<'_> {
+impl Resolver<'_, '_> {
     pub(crate) fn check_unused(&mut self, krate: &ast::Crate) {
+        let tcx = self.tcx;
+        let mut maybe_unused_extern_crates = FxHashMap::default();
+
         for import in self.potentially_unused_imports.iter() {
             match import.kind {
                 _ if import.used.get()
@@ -246,7 +276,14 @@ impl Resolver<'_> {
                 }
                 ImportKind::ExternCrate { id, .. } => {
                     let def_id = self.local_def_id(id);
-                    self.maybe_unused_extern_crates.push((def_id, import.span));
+                    if self.extern_crate_map.get(&def_id).map_or(true, |&cnum| {
+                        !tcx.is_compiler_builtins(cnum)
+                            && !tcx.is_panic_runtime(cnum)
+                            && !tcx.has_global_allocator(cnum)
+                            && !tcx.has_panic_handler(cnum)
+                    }) {
+                        maybe_unused_extern_crates.insert(id, import.span);
+                    }
                 }
                 ImportKind::MacroUse => {
                     let msg = "unused `#[macro_use]` import";
@@ -259,6 +296,7 @@ impl Resolver<'_> {
         let mut visitor = UnusedImportCheckVisitor {
             r: self,
             unused_imports: Default::default(),
+            extern_crate_items: Default::default(),
             base_use_tree: None,
             base_id: ast::DUMMY_NODE_ID,
             item_span: DUMMY_SP,
@@ -290,7 +328,7 @@ impl Resolver<'_> {
             let ms = MultiSpan::from_spans(spans.clone());
             let mut span_snippets = spans
                 .iter()
-                .filter_map(|s| match visitor.r.session.source_map().span_to_snippet(*s) {
+                .filter_map(|s| match tcx.sess.source_map().span_to_snippet(*s) {
                     Ok(s) => Some(format!("`{}`", s)),
                     _ => None,
                 })
@@ -317,7 +355,7 @@ impl Resolver<'_> {
             // If we are in the `--test` mode, suppress a help that adds the `#[cfg(test)]`
             // attribute; however, if not, suggest adding the attribute. There is no way to
             // retrieve attributes here because we do not have a `TyCtxt` yet.
-            let test_module_span = if visitor.r.session.opts.test {
+            let test_module_span = if tcx.sess.opts.test {
                 None
             } else {
                 let parent_module = visitor.r.get_nearest_non_block_module(
@@ -346,5 +384,74 @@ impl Resolver<'_> {
                 BuiltinLintDiagnostics::UnusedImports(fix_msg.into(), fixes, test_module_span),
             );
         }
+
+        for extern_crate in visitor.extern_crate_items {
+            let warn_if_unused = !extern_crate.ident.name.as_str().starts_with('_');
+
+            // If the crate is fully unused, we suggest removing it altogether.
+            // We do this in any edition.
+            if warn_if_unused {
+                if let Some(&span) = maybe_unused_extern_crates.get(&extern_crate.id) {
+                    visitor.r.lint_buffer.buffer_lint_with_diagnostic(
+                        UNUSED_EXTERN_CRATES,
+                        extern_crate.id,
+                        span,
+                        "unused extern crate",
+                        BuiltinLintDiagnostics::UnusedExternCrate {
+                            removal_span: extern_crate.span_with_attributes,
+                        },
+                    );
+                    continue;
+                }
+            }
+
+            // If we are not in Rust 2018 edition, then we don't make any further
+            // suggestions.
+            if !tcx.sess.rust_2018() {
+                continue;
+            }
+
+            // If the extern crate has any attributes, they may have funky
+            // semantics we can't faithfully represent using `use` (most
+            // notably `#[macro_use]`). Ignore it.
+            if extern_crate.has_attrs {
+                continue;
+            }
+
+            // If the extern crate is renamed, then we cannot suggest replacing it with a use as this
+            // would not insert the new name into the prelude, where other imports in the crate may be
+            // expecting it.
+            if extern_crate.renames {
+                continue;
+            }
+
+            // If the extern crate isn't in the extern prelude,
+            // there is no way it can be written as a `use`.
+            if !visitor
+                .r
+                .extern_prelude
+                .get(&extern_crate.ident)
+                .map_or(false, |entry| !entry.introduced_by_item)
+            {
+                continue;
+            }
+
+            let vis_span = extern_crate
+                .vis_span
+                .find_ancestor_inside(extern_crate.span)
+                .unwrap_or(extern_crate.vis_span);
+            let ident_span = extern_crate
+                .ident
+                .span
+                .find_ancestor_inside(extern_crate.span)
+                .unwrap_or(extern_crate.ident.span);
+            visitor.r.lint_buffer.buffer_lint_with_diagnostic(
+                UNUSED_EXTERN_CRATES,
+                extern_crate.id,
+                extern_crate.span,
+                "`extern crate` is not idiomatic in the new edition",
+                BuiltinLintDiagnostics::ExternCrateNotIdiomatic { vis_span, ident_span },
+            );
+        }
     }
 }
diff --git a/compiler/rustc_resolve/src/def_collector.rs b/compiler/rustc_resolve/src/def_collector.rs
index 2764a6c28..e7ff236f8 100644
--- a/compiler/rustc_resolve/src/def_collector.rs
+++ b/compiler/rustc_resolve/src/def_collector.rs
@@ -9,7 +9,7 @@ use rustc_span::symbol::sym;
 use rustc_span::Span;
 
 pub(crate) fn collect_definitions(
-    resolver: &mut Resolver<'_>,
+    resolver: &mut Resolver<'_, '_>,
     fragment: &AstFragment,
     expansion: LocalExpnId,
 ) {
@@ -18,14 +18,14 @@ pub(crate) fn collect_definitions(
 }
 
 /// Creates `DefId`s for nodes in the AST.
-struct DefCollector<'a, 'b> {
-    resolver: &'a mut Resolver<'b>,
+struct DefCollector<'a, 'b, 'tcx> {
+    resolver: &'a mut Resolver<'b, 'tcx>,
     parent_def: LocalDefId,
     impl_trait_context: ImplTraitContext,
     expansion: LocalExpnId,
 }
 
-impl<'a, 'b> DefCollector<'a, 'b> {
+impl<'a, 'b, 'tcx> DefCollector<'a, 'b, 'tcx> {
     fn create_def(&mut self, node_id: NodeId, data: DefPathData, span: Span) -> LocalDefId {
         let parent_def = self.parent_def;
         debug!("create_def(node_id={:?}, data={:?}, parent_def={:?})", node_id, data, parent_def);
@@ -81,7 +81,7 @@ impl<'a, 'b> DefCollector<'a, 'b> {
     }
 }
 
-impl<'a, 'b> visit::Visitor<'a> for DefCollector<'a, 'b> {
+impl<'a, 'b, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'b, 'tcx> {
     fn visit_item(&mut self, i: &'a Item) {
         debug!("visit_item: {:?}", i);
 
diff --git a/compiler/rustc_resolve/src/diagnostics.rs b/compiler/rustc_resolve/src/diagnostics.rs
index 366086152..7add59ac6 100644
--- a/compiler/rustc_resolve/src/diagnostics.rs
+++ b/compiler/rustc_resolve/src/diagnostics.rs
@@ -12,25 +12,24 @@ use rustc_errors::{struct_span_err, SuggestionStyle};
 use rustc_feature::BUILTIN_ATTRIBUTES;
 use rustc_hir::def::Namespace::{self, *};
 use rustc_hir::def::{self, CtorKind, CtorOf, DefKind, NonMacroAttrKind, PerNS};
-use rustc_hir::def_id::{DefId, LocalDefId, CRATE_DEF_ID, LOCAL_CRATE};
+use rustc_hir::def_id::{DefId, CRATE_DEF_ID, LOCAL_CRATE};
 use rustc_hir::PrimTy;
-use rustc_index::vec::IndexVec;
 use rustc_middle::bug;
-use rustc_middle::ty::DefIdTree;
+use rustc_middle::ty::{DefIdTree, TyCtxt};
 use rustc_session::lint::builtin::ABSOLUTE_PATHS_NOT_STARTING_WITH_CRATE;
 use rustc_session::lint::builtin::MACRO_EXPANDED_MACRO_EXPORTS_ACCESSED_BY_ABSOLUTE_PATHS;
 use rustc_session::lint::BuiltinLintDiagnostics;
 use rustc_session::Session;
+use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::edition::Edition;
 use rustc_span::hygiene::MacroKind;
-use rustc_span::lev_distance::find_best_match_for_name;
 use rustc_span::source_map::SourceMap;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{BytePos, Span, SyntaxContext};
 use thin_vec::ThinVec;
 
 use crate::errors as errs;
-use crate::imports::{Import, ImportKind, ImportResolver};
+use crate::imports::{Import, ImportKind};
 use crate::late::{PatternSource, Rib};
 use crate::path_names_to_string;
 use crate::{AmbiguityError, AmbiguityErrorMisc, AmbiguityKind, BindingError, Finalize};
@@ -114,7 +113,7 @@ fn reduce_impl_span_to_impl_keyword(sm: &SourceMap, impl_span: Span) -> Span {
     sm.span_until_whitespace(impl_span)
 }
 
-impl<'a> Resolver<'a> {
+impl<'a, 'tcx> Resolver<'a, 'tcx> {
     pub(crate) fn report_errors(&mut self, krate: &Crate) {
         self.report_with_use_injections(krate);
 
@@ -154,8 +153,7 @@ impl<'a> Resolver<'a> {
 
             if !candidates.is_empty() {
                 show_candidates(
-                    &self.session,
-                    &self.untracked.source_span,
+                    self.tcx,
                     &mut err,
                     span,
                     &candidates,
@@ -191,6 +189,8 @@ impl<'a> Resolver<'a> {
         }
 
         let container = match parent.kind {
+            // Avoid using TyCtxt::def_kind_descr in the resolver, because it
+            // indirectly *calls* the resolver, and would cause a query cycle.
             ModuleKind::Def(kind, _, _) => kind.descr(parent.def_id()),
             ModuleKind::Block => "block",
         };
@@ -206,7 +206,7 @@ impl<'a> Resolver<'a> {
         };
 
         let (name, span) =
-            (ident.name, self.session.source_map().guess_head_span(new_binding.span));
+            (ident.name, self.tcx.sess.source_map().guess_head_span(new_binding.span));
 
         if let Some(s) = self.name_already_seen.get(&name) {
             if s == &span {
@@ -226,15 +226,15 @@ impl<'a> Resolver<'a> {
         let msg = format!("the name `{}` is defined multiple times", name);
 
         let mut err = match (old_binding.is_extern_crate(), new_binding.is_extern_crate()) {
-            (true, true) => struct_span_err!(self.session, span, E0259, "{}", msg),
+            (true, true) => struct_span_err!(self.tcx.sess, span, E0259, "{}", msg),
             (true, _) | (_, true) => match new_binding.is_import() && old_binding.is_import() {
-                true => struct_span_err!(self.session, span, E0254, "{}", msg),
-                false => struct_span_err!(self.session, span, E0260, "{}", msg),
+                true => struct_span_err!(self.tcx.sess, span, E0254, "{}", msg),
+                false => struct_span_err!(self.tcx.sess, span, E0260, "{}", msg),
             },
             _ => match (old_binding.is_import_user_facing(), new_binding.is_import_user_facing()) {
-                (false, false) => struct_span_err!(self.session, span, E0428, "{}", msg),
-                (true, true) => struct_span_err!(self.session, span, E0252, "{}", msg),
-                _ => struct_span_err!(self.session, span, E0255, "{}", msg),
+                (false, false) => struct_span_err!(self.tcx.sess, span, E0428, "{}", msg),
+                (true, true) => struct_span_err!(self.tcx.sess, span, E0252, "{}", msg),
+                _ => struct_span_err!(self.tcx.sess, span, E0255, "{}", msg),
             },
         };
 
@@ -248,7 +248,7 @@ impl<'a> Resolver<'a> {
         err.span_label(span, format!("`{}` re{} here", name, new_participle));
         if !old_binding.span.is_dummy() && old_binding.span != span {
             err.span_label(
-                self.session.source_map().guess_head_span(old_binding.span),
+                self.tcx.sess.source_map().guess_head_span(old_binding.span),
                 format!("previous {} of the {} `{}` here", old_noun, old_kind, name),
             );
         }
@@ -352,7 +352,7 @@ impl<'a> Resolver<'a> {
                 if let Some(pos) =
                     source.span.hi().0.checked_sub(binding_span.lo().0).map(|pos| pos as usize)
                 {
-                    if let Ok(snippet) = self.session.source_map().span_to_snippet(binding_span) {
+                    if let Ok(snippet) = self.tcx.sess.source_map().span_to_snippet(binding_span) {
                         if pos <= snippet.len() {
                             suggestion = Some(format!(
                                 "{} as {}{}",
@@ -426,12 +426,12 @@ impl<'a> Resolver<'a> {
         //   `a` and `import.use_span` is `issue_52891::{d, e, a};`.
 
         let (found_closing_brace, span) =
-            find_span_of_binding_until_next_binding(self.session, binding_span, import.use_span);
+            find_span_of_binding_until_next_binding(self.tcx.sess, binding_span, import.use_span);
 
         // If there was a closing brace then identify the span to remove any trailing commas from
         // previous imports.
         if found_closing_brace {
-            if let Some(span) = extend_span_to_previous_binding(self.session, span) {
+            if let Some(span) = extend_span_to_previous_binding(self.tcx.sess, span) {
                 err.tool_only_span_suggestion(span, message, "", Applicability::MaybeIncorrect);
             } else {
                 // Remove the entire line if we cannot extend the span back, this indicates an
@@ -462,7 +462,9 @@ impl<'a> Resolver<'a> {
 
         let first_name = match path.get(0) {
             // In the 2018 edition this lint is a hard error, so nothing to do
-            Some(seg) if seg.ident.span.rust_2015() && self.session.rust_2015() => seg.ident.name,
+            Some(seg) if seg.ident.span.is_rust_2015() && self.tcx.sess.is_rust_2015() => {
+                seg.ident.name
+            }
             _ => return,
         };
 
@@ -539,14 +541,14 @@ impl<'a> Resolver<'a> {
         match resolution_error {
             ResolutionError::GenericParamsFromOuterFunction(outer_res, has_generic_params) => {
                 let mut err = struct_span_err!(
-                    self.session,
+                    self.tcx.sess,
                     span,
                     E0401,
                     "can't use generic parameters from outer function",
                 );
                 err.span_label(span, "use of generic parameter from outer function");
 
-                let sm = self.session.source_map();
+                let sm = self.tcx.sess.source_map();
                 let def_id = match outer_res {
                     Res::SelfTyParam { .. } => {
                         err.span_label(span, "can't use `Self` here");
@@ -603,10 +605,11 @@ impl<'a> Resolver<'a> {
                 err
             }
             ResolutionError::NameAlreadyUsedInParameterList(name, first_use_span) => self
-                .session
+                .tcx
+                .sess
                 .create_err(errs::NameAlreadyUsedInParameterList { span, first_use_span, name }),
             ResolutionError::MethodNotMemberOfTrait(method, trait_, candidate) => {
-                self.session.create_err(errs::MethodNotMemberOfTrait {
+                self.tcx.sess.create_err(errs::MethodNotMemberOfTrait {
                     span,
                     method,
                     trait_,
@@ -617,7 +620,7 @@ impl<'a> Resolver<'a> {
                 })
             }
             ResolutionError::TypeNotMemberOfTrait(type_, trait_, candidate) => {
-                self.session.create_err(errs::TypeNotMemberOfTrait {
+                self.tcx.sess.create_err(errs::TypeNotMemberOfTrait {
                     span,
                     type_,
                     trait_,
@@ -628,7 +631,7 @@ impl<'a> Resolver<'a> {
                 })
             }
             ResolutionError::ConstNotMemberOfTrait(const_, trait_, candidate) => {
-                self.session.create_err(errs::ConstNotMemberOfTrait {
+                self.tcx.sess.create_err(errs::ConstNotMemberOfTrait {
                     span,
                     const_,
                     trait_,
@@ -646,7 +649,7 @@ impl<'a> Resolver<'a> {
 
                 let msp = MultiSpan::from_spans(target_sp.clone());
                 let mut err = struct_span_err!(
-                    self.session,
+                    self.tcx.sess,
                     msp,
                     E0408,
                     "variable `{}` is not bound in all patterns",
@@ -684,8 +687,7 @@ impl<'a> Resolver<'a> {
                         err.span_help(span, &help_msg);
                     }
                     show_candidates(
-                        &self.session,
-                        &self.untracked.source_span,
+                        self.tcx,
                         &mut err,
                         Some(span),
                         &import_suggestions,
@@ -699,17 +701,19 @@ impl<'a> Resolver<'a> {
                 err
             }
             ResolutionError::VariableBoundWithDifferentMode(variable_name, first_binding_span) => {
-                self.session.create_err(errs::VariableBoundWithDifferentMode {
+                self.tcx.sess.create_err(errs::VariableBoundWithDifferentMode {
                     span,
                     first_binding_span,
                     variable_name,
                 })
             }
             ResolutionError::IdentifierBoundMoreThanOnceInParameterList(identifier) => self
-                .session
+                .tcx
+                .sess
                 .create_err(errs::IdentifierBoundMoreThanOnceInParameterList { span, identifier }),
             ResolutionError::IdentifierBoundMoreThanOnceInSamePattern(identifier) => self
-                .session
+                .tcx
+                .sess
                 .create_err(errs::IdentifierBoundMoreThanOnceInSamePattern { span, identifier }),
             ResolutionError::UndeclaredLabel { name, suggestion } => {
                 let ((sub_reachable, sub_reachable_suggestion), sub_unreachable) = match suggestion
@@ -735,7 +739,7 @@ impl<'a> Resolver<'a> {
                     // No similarly-named labels exist.
                     None => ((None, None), None),
                 };
-                self.session.create_err(errs::UndeclaredLabel {
+                self.tcx.sess.create_err(errs::UndeclaredLabel {
                     span,
                     name,
                     sub_reachable,
@@ -760,21 +764,22 @@ impl<'a> Resolver<'a> {
                     };
                     (Some(suggestion), Some(mpart_suggestion))
                 };
-                self.session.create_err(errs::SelfImportsOnlyAllowedWithin {
+                self.tcx.sess.create_err(errs::SelfImportsOnlyAllowedWithin {
                     span,
                     suggestion,
                     mpart_suggestion,
                 })
             }
             ResolutionError::SelfImportCanOnlyAppearOnceInTheList => {
-                self.session.create_err(errs::SelfImportCanOnlyAppearOnceInTheList { span })
-            }
-            ResolutionError::SelfImportOnlyInImportListWithNonEmptyPrefix => {
-                self.session.create_err(errs::SelfImportOnlyInImportListWithNonEmptyPrefix { span })
+                self.tcx.sess.create_err(errs::SelfImportCanOnlyAppearOnceInTheList { span })
             }
+            ResolutionError::SelfImportOnlyInImportListWithNonEmptyPrefix => self
+                .tcx
+                .sess
+                .create_err(errs::SelfImportOnlyInImportListWithNonEmptyPrefix { span }),
             ResolutionError::FailedToResolve { label, suggestion } => {
                 let mut err =
-                    struct_span_err!(self.session, span, E0433, "failed to resolve: {}", &label);
+                    struct_span_err!(self.tcx.sess, span, E0433, "failed to resolve: {}", &label);
                 err.span_label(span, label);
 
                 if let Some((suggestions, msg, applicability)) = suggestion {
@@ -788,7 +793,7 @@ impl<'a> Resolver<'a> {
                 err
             }
             ResolutionError::CannotCaptureDynamicEnvironmentInFnItem => {
-                self.session.create_err(errs::CannotCaptureDynamicEnvironmentInFnItem { span })
+                self.tcx.sess.create_err(errs::CannotCaptureDynamicEnvironmentInFnItem { span })
             }
             ResolutionError::AttemptToUseNonConstantValueInConstant(ident, suggestion, current) => {
                 // let foo =...
@@ -800,12 +805,13 @@ impl<'a> Resolver<'a> {
                 // the further the two are apart, the higher the chance of the suggestion being wrong
 
                 let sp = self
-                    .session
+                    .tcx
+                    .sess
                     .source_map()
                     .span_extend_to_prev_str(ident.span, current, true, false);
 
                 let ((with, with_label), without) = match sp {
-                    Some(sp) if !self.session.source_map().is_multiline(sp) => {
+                    Some(sp) if !self.tcx.sess.source_map().is_multiline(sp) => {
                         let sp = sp.with_lo(BytePos(sp.lo().0 - (current.len() as u32)));
                         (
                         (Some(errs::AttemptToUseNonConstantValueInConstantWithSuggestion {
@@ -826,7 +832,7 @@ impl<'a> Resolver<'a> {
                     ),
                 };
 
-                self.session.create_err(errs::AttemptToUseNonConstantValueInConstant {
+                self.tcx.sess.create_err(errs::AttemptToUseNonConstantValueInConstant {
                     span,
                     with,
                     with_label,
@@ -840,7 +846,7 @@ impl<'a> Resolver<'a> {
                 article,
                 shadowed_binding,
                 shadowed_binding_span,
-            } => self.session.create_err(errs::BindingShadowsSomethingUnacceptable {
+            } => self.tcx.sess.create_err(errs::BindingShadowsSomethingUnacceptable {
                 span,
                 shadowing_binding,
                 shadowed_binding,
@@ -857,13 +863,13 @@ impl<'a> Resolver<'a> {
                 name,
             }),
             ResolutionError::ForwardDeclaredGenericParam => {
-                self.session.create_err(errs::ForwardDeclaredGenericParam { span })
+                self.tcx.sess.create_err(errs::ForwardDeclaredGenericParam { span })
             }
             ResolutionError::ParamInTyOfConstParam(name) => {
-                self.session.create_err(errs::ParamInTyOfConstParam { span, name })
+                self.tcx.sess.create_err(errs::ParamInTyOfConstParam { span, name })
             }
             ResolutionError::ParamInNonTrivialAnonConst { name, is_type } => {
-                self.session.create_err(errs::ParamInNonTrivialAnonConst {
+                self.tcx.sess.create_err(errs::ParamInNonTrivialAnonConst {
                     span,
                     name,
                     sub_is_type: if is_type {
@@ -872,13 +878,14 @@ impl<'a> Resolver<'a> {
                         errs::ParamInNonTrivialAnonConstIsType::NotAType { name }
                     },
                     help: self
-                        .session
+                        .tcx
+                        .sess
                         .is_nightly_build()
                         .then_some(errs::ParamInNonTrivialAnonConstHelp),
                 })
             }
             ResolutionError::SelfInGenericParamDefault => {
-                self.session.create_err(errs::SelfInGenericParamDefault { span })
+                self.tcx.sess.create_err(errs::SelfInGenericParamDefault { span })
             }
             ResolutionError::UnreachableLabel { name, definition_span, suggestion } => {
                 let ((sub_suggestion_label, sub_suggestion), sub_unreachable_label) =
@@ -906,7 +913,7 @@ impl<'a> Resolver<'a> {
                         // No similarly-named labels exist.
                         None => ((None, None), None),
                     };
-                self.session.create_err(errs::UnreachableLabel {
+                self.tcx.sess.create_err(errs::UnreachableLabel {
                     span,
                     name,
                     definition_span,
@@ -922,7 +929,7 @@ impl<'a> Resolver<'a> {
                 trait_item_span,
                 trait_path,
             } => {
-                let mut err = self.session.struct_span_err_with_code(
+                let mut err = self.tcx.sess.struct_span_err_with_code(
                     span,
                     &format!(
                         "item `{}` is an associated {}, which doesn't match its trait `{}`",
@@ -935,9 +942,12 @@ impl<'a> Resolver<'a> {
                 err
             }
             ResolutionError::TraitImplDuplicate { name, trait_item_span, old_span } => self
-                .session
+                .tcx
+                .sess
                 .create_err(errs::TraitImplDuplicate { span, name, trait_item_span, old_span }),
-            ResolutionError::InvalidAsmSym => self.session.create_err(errs::InvalidAsmSym { span }),
+            ResolutionError::InvalidAsmSym => {
+                self.tcx.sess.create_err(errs::InvalidAsmSym { span })
+            }
         }
     }
 
@@ -947,7 +957,7 @@ impl<'a> Resolver<'a> {
     ) -> ErrorGuaranteed {
         match vis_resolution_error {
             VisResolutionError::Relative2018(span, path) => {
-                self.session.create_err(errs::Relative2018 {
+                self.tcx.sess.create_err(errs::Relative2018 {
                     span,
                     path_span: path.span,
                     // intentionally converting to String, as the text would also be used as
@@ -956,18 +966,20 @@ impl<'a> Resolver<'a> {
                 })
             }
             VisResolutionError::AncestorOnly(span) => {
-                self.session.create_err(errs::AncestorOnly(span))
+                self.tcx.sess.create_err(errs::AncestorOnly(span))
             }
             VisResolutionError::FailedToResolve(span, label, suggestion) => {
                 self.into_struct_error(span, ResolutionError::FailedToResolve { label, suggestion })
             }
             VisResolutionError::ExpectedFound(span, path_str, res) => {
-                self.session.create_err(errs::ExpectedFound { span, res, path_str })
+                self.tcx.sess.create_err(errs::ExpectedFound { span, res, path_str })
             }
             VisResolutionError::Indeterminate(span) => {
-                self.session.create_err(errs::Indeterminate(span))
+                self.tcx.sess.create_err(errs::Indeterminate(span))
+            }
+            VisResolutionError::ModuleOnly(span) => {
+                self.tcx.sess.create_err(errs::ModuleOnly(span))
             }
-            VisResolutionError::ModuleOnly(span) => self.session.create_err(errs::ModuleOnly(span)),
         }
         .emit()
     }
@@ -1204,7 +1216,7 @@ impl<'a> Resolver<'a> {
                             // a note about editions
                             let note = if let Some(did) = did {
                                 let requires_note = !did.is_local()
-                                    && this.cstore().item_attrs_untracked(did, this.session).any(
+                                    && this.cstore().item_attrs_untracked(did, this.tcx.sess).any(
                                         |attr| {
                                             if attr.has_name(sym::rustc_diagnostic_item) {
                                                 [sym::TryInto, sym::TryFrom, sym::FromIterator]
@@ -1302,7 +1314,7 @@ impl<'a> Resolver<'a> {
                     // otherwise cause duplicate suggestions.
                     continue;
                 }
-                let crate_id = self.crate_loader().maybe_process_path_extern(ident.name);
+                let crate_id = self.crate_loader(|c| c.maybe_process_path_extern(ident.name));
                 if let Some(crate_id) = crate_id {
                     let crate_root = self.expect_module(crate_id.as_def_id());
                     suggestions.extend(self.lookup_import_candidates_from_module(
@@ -1339,8 +1351,7 @@ impl<'a> Resolver<'a> {
         let import_suggestions =
             self.lookup_import_candidates(ident, Namespace::MacroNS, parent_scope, is_expected);
         show_candidates(
-            &self.session,
-            &self.untracked.source_span,
+            self.tcx,
             err,
             None,
             &import_suggestions,
@@ -1364,7 +1375,7 @@ impl<'a> Resolver<'a> {
             && let ModuleKind::Def(DefKind::Enum, def_id, _) = parent_scope.module.kind
             && let Some(span) = self.opt_span(def_id)
         {
-            let source_map = self.session.source_map();
+            let source_map = self.tcx.sess.source_map();
             let head_span = source_map.guess_head_span(span);
             if let Ok(head) = source_map.span_to_snippet(head_span) {
                 err.span_suggestion(head_span, "consider adding a derive", format!("#[derive(Default)]\n{head}"), Applicability::MaybeIncorrect);
@@ -1441,7 +1452,7 @@ impl<'a> Resolver<'a> {
         };
         let def_span = suggestion.res.opt_def_id().and_then(|def_id| match def_id.krate {
             LOCAL_CRATE => self.opt_span(def_id),
-            _ => Some(self.cstore().get_span_untracked(def_id, self.session)),
+            _ => Some(self.cstore().get_span_untracked(def_id, self.tcx.sess)),
         });
         if let Some(def_span) = def_span {
             if span.overlaps(def_span) {
@@ -1471,7 +1482,7 @@ impl<'a> Resolver<'a> {
             };
 
             err.span_label(
-                self.session.source_map().guess_head_span(def_span),
+                self.tcx.sess.source_map().guess_head_span(def_span),
                 &format!(
                     "{}{} `{}` defined here",
                     prefix,
@@ -1496,7 +1507,7 @@ impl<'a> Resolver<'a> {
 
     fn binding_description(&self, b: &NameBinding<'_>, ident: Ident, from_prelude: bool) -> String {
         let res = b.res();
-        if b.span.is_dummy() || !self.session.source_map().is_span_accessible(b.span) {
+        if b.span.is_dummy() || !self.tcx.sess.source_map().is_span_accessible(b.span) {
             // These already contain the "built-in" prefix or look bad with it.
             let add_built_in =
                 !matches!(b.res(), Res::NonMacroAttr(..) | Res::PrimTy(..) | Res::ToolMod);
@@ -1504,7 +1515,7 @@ impl<'a> Resolver<'a> {
                 ("", " from prelude")
             } else if b.is_extern_crate()
                 && !b.is_import()
-                && self.session.opts.externs.get(ident.as_str()).is_some()
+                && self.tcx.sess.opts.externs.get(ident.as_str()).is_some()
             {
                 ("", " passed with `--extern`")
             } else if add_built_in {
@@ -1530,7 +1541,7 @@ impl<'a> Resolver<'a> {
             (b1, b2, misc1, misc2, false)
         };
 
-        let mut err = struct_span_err!(self.session, ident.span, E0659, "`{ident}` is ambiguous");
+        let mut err = struct_span_err!(self.tcx.sess, ident.span, E0659, "`{ident}` is ambiguous");
         err.span_label(ident.span, "ambiguous name");
         err.note(&format!("ambiguous because of {}", kind.descr()));
 
@@ -1552,12 +1563,12 @@ impl<'a> Resolver<'a> {
             if b.is_extern_crate() && ident.span.rust_2018() {
                 help_msgs.push(format!("use `::{ident}` to refer to this {thing} unambiguously"))
             }
-            if misc == AmbiguityErrorMisc::SuggestCrate {
-                help_msgs
-                    .push(format!("use `crate::{ident}` to refer to this {thing} unambiguously"))
-            } else if misc == AmbiguityErrorMisc::SuggestSelf {
-                help_msgs
-                    .push(format!("use `self::{ident}` to refer to this {thing} unambiguously"))
+            match misc {
+                AmbiguityErrorMisc::SuggestCrate => help_msgs
+                    .push(format!("use `crate::{ident}` to refer to this {thing} unambiguously")),
+                AmbiguityErrorMisc::SuggestSelf => help_msgs
+                    .push(format!("use `self::{ident}` to refer to this {thing} unambiguously")),
+                AmbiguityErrorMisc::FromPrelude | AmbiguityErrorMisc::None => {}
             }
 
             err.span_note(b.span, &note_msg);
@@ -1602,7 +1613,7 @@ impl<'a> Resolver<'a> {
         // Print the primary message.
         let descr = get_descr(binding);
         let mut err =
-            struct_span_err!(self.session, ident.span, E0603, "{} `{}` is private", descr, ident);
+            struct_span_err!(self.tcx.sess, ident.span, E0603, "{} `{}` is private", descr, ident);
         err.span_label(ident.span, &format!("private {}", descr));
         if let Some(span) = ctor_fields_span {
             err.span_label(span, "a constructor is private if any of the fields is private");
@@ -1648,7 +1659,7 @@ impl<'a> Resolver<'a> {
                 which = if first { "" } else { " which" },
                 dots = if next_binding.is_some() { "..." } else { "" },
             );
-            let def_span = self.session.source_map().guess_head_span(binding.span);
+            let def_span = self.tcx.sess.source_map().guess_head_span(binding.span);
             let mut note_span = MultiSpan::from_span(def_span);
             if !first && binding.vis.is_public() {
                 note_span.push_span_label(def_span, "consider importing it directly");
@@ -1717,7 +1728,7 @@ impl<'a> Resolver<'a> {
                         Applicability::MaybeIncorrect,
                     )),
                 )
-            } else if self.session.edition() == Edition::Edition2015 {
+            } else if self.tcx.sess.is_rust_2015() {
                 (
                     format!("maybe a missing crate `{ident}`?"),
                     Some((
@@ -1736,7 +1747,7 @@ impl<'a> Resolver<'a> {
             let parent = match parent {
                 // ::foo is mounted at the crate root for 2015, and is the extern
                 // prelude for 2018+
-                kw::PathRoot if self.session.edition() > Edition::Edition2015 => {
+                kw::PathRoot if self.tcx.sess.edition() > Edition::Edition2015 => {
                     "the list of imported crates".to_owned()
                 }
                 kw::PathRoot | kw::Crate => "the crate root".to_owned(),
@@ -1795,6 +1806,8 @@ impl<'a> Resolver<'a> {
                         found("module")
                     } else {
                         match binding.res() {
+                            // Avoid using TyCtxt::def_kind_descr in the resolver, because it
+                            // indirectly *calls* the resolver, and would cause a query cycle.
                             Res::Def(kind, id) => found(kind.descr(id)),
                             _ => found(ns_to_try.descr()),
                         }
@@ -1879,15 +1892,13 @@ impl<'a> Resolver<'a> {
             (format!("use of undeclared crate or module `{}`", ident), suggestion)
         }
     }
-}
 
-impl<'a, 'b> ImportResolver<'a, 'b> {
     /// Adds suggestions for a path that cannot be resolved.
     pub(crate) fn make_path_suggestion(
         &mut self,
         span: Span,
         mut path: Vec<Segment>,
-        parent_scope: &ParentScope<'b>,
+        parent_scope: &ParentScope<'a>,
     ) -> Option<(Vec<Segment>, Option<String>)> {
         debug!("make_path_suggestion: span={:?} path={:?}", span, path);
 
@@ -1922,11 +1933,11 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
     fn make_missing_self_suggestion(
         &mut self,
         mut path: Vec<Segment>,
-        parent_scope: &ParentScope<'b>,
+        parent_scope: &ParentScope<'a>,
     ) -> Option<(Vec<Segment>, Option<String>)> {
         // Replace first ident with `self` and check if that is valid.
         path[0].ident.name = kw::SelfLower;
-        let result = self.r.maybe_resolve_path(&path, None, parent_scope);
+        let result = self.maybe_resolve_path(&path, None, parent_scope);
         debug!("make_missing_self_suggestion: path={:?} result={:?}", path, result);
         if let PathResult::Module(..) = result { Some((path, None)) } else { None }
     }
@@ -1941,11 +1952,11 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
     fn make_missing_crate_suggestion(
         &mut self,
         mut path: Vec<Segment>,
-        parent_scope: &ParentScope<'b>,
+        parent_scope: &ParentScope<'a>,
     ) -> Option<(Vec<Segment>, Option<String>)> {
         // Replace first ident with `crate` and check if that is valid.
         path[0].ident.name = kw::Crate;
-        let result = self.r.maybe_resolve_path(&path, None, parent_scope);
+        let result = self.maybe_resolve_path(&path, None, parent_scope);
         debug!("make_missing_crate_suggestion:  path={:?} result={:?}", path, result);
         if let PathResult::Module(..) = result {
             Some((
@@ -1972,11 +1983,11 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
     fn make_missing_super_suggestion(
         &mut self,
         mut path: Vec<Segment>,
-        parent_scope: &ParentScope<'b>,
+        parent_scope: &ParentScope<'a>,
     ) -> Option<(Vec<Segment>, Option<String>)> {
         // Replace first ident with `crate` and check if that is valid.
         path[0].ident.name = kw::Super;
-        let result = self.r.maybe_resolve_path(&path, None, parent_scope);
+        let result = self.maybe_resolve_path(&path, None, parent_scope);
         debug!("make_missing_super_suggestion:  path={:?} result={:?}", path, result);
         if let PathResult::Module(..) = result { Some((path, None)) } else { None }
     }
@@ -1994,9 +2005,9 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
     fn make_external_crate_suggestion(
         &mut self,
         mut path: Vec<Segment>,
-        parent_scope: &ParentScope<'b>,
+        parent_scope: &ParentScope<'a>,
     ) -> Option<(Vec<Segment>, Option<String>)> {
-        if path[1].ident.span.rust_2015() {
+        if path[1].ident.span.is_rust_2015() {
             return None;
         }
 
@@ -2004,13 +2015,13 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
         // 1) some consistent ordering for emitted diagnostics, and
         // 2) `std` suggestions before `core` suggestions.
         let mut extern_crate_names =
-            self.r.extern_prelude.iter().map(|(ident, _)| ident.name).collect::<Vec<_>>();
+            self.extern_prelude.iter().map(|(ident, _)| ident.name).collect::<Vec<_>>();
         extern_crate_names.sort_by(|a, b| b.as_str().partial_cmp(a.as_str()).unwrap());
 
         for name in extern_crate_names.into_iter() {
             // Replace first ident with a crate name and check if that is valid.
             path[0].ident.name = name;
-            let result = self.r.maybe_resolve_path(&path, None, parent_scope);
+            let result = self.maybe_resolve_path(&path, None, parent_scope);
             debug!(
                 "make_external_crate_suggestion: name={:?} path={:?} result={:?}",
                 name, path, result
@@ -2037,8 +2048,8 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
     /// ```
     pub(crate) fn check_for_module_export_macro(
         &mut self,
-        import: &'b Import<'b>,
-        module: ModuleOrUniformRoot<'b>,
+        import: &'a Import<'a>,
+        module: ModuleOrUniformRoot<'a>,
         ident: Ident,
     ) -> Option<(Option<Suggestion>, Option<String>)> {
         let ModuleOrUniformRoot::Module(mut crate_module) = module else {
@@ -2055,8 +2066,8 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
             return None;
         }
 
-        let resolutions = self.r.resolutions(crate_module).borrow();
-        let resolution = resolutions.get(&self.r.new_key(ident, MacroNS))?;
+        let resolutions = self.resolutions(crate_module).borrow();
+        let resolution = resolutions.get(&self.new_key(ident, MacroNS))?;
         let binding = resolution.borrow().binding()?;
         if let Res::Def(DefKind::Macro(MacroKind::Bang), _) = binding.res() {
             let module_name = crate_module.kind.name().unwrap();
@@ -2077,7 +2088,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
                 //   ie. `use a::b::{c, d, e};`
                 //                      ^^^
                 let (found_closing_brace, binding_span) = find_span_of_binding_until_next_binding(
-                    self.r.session,
+                    self.tcx.sess,
                     import.span,
                     import.use_span,
                 );
@@ -2096,7 +2107,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
                     //   ie. `use a::b::{c, d};`
                     //                    ^^^
                     if let Some(previous_span) =
-                        extend_span_to_previous_binding(self.r.session, binding_span)
+                        extend_span_to_previous_binding(self.tcx.sess, binding_span)
                     {
                         debug!("check_for_module_export_macro: previous_span={:?}", previous_span);
                         removal_span = removal_span.with_lo(previous_span.lo());
@@ -2114,7 +2125,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
                 //   or  `use a::{b, c, d}};`
                 //               ^^^^^^^^^^^
                 let (has_nested, after_crate_name) = find_span_immediately_after_crate_name(
-                    self.r.session,
+                    self.tcx.sess,
                     module_name,
                     import.use_span,
                 );
@@ -2123,7 +2134,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
                     has_nested, after_crate_name
                 );
 
-                let source_map = self.r.session.source_map();
+                let source_map = self.tcx.sess.source_map();
 
                 // Make sure this is actually crate-relative.
                 let is_definitely_crate = import
@@ -2345,8 +2356,7 @@ pub(crate) enum DiagnosticMode {
 }
 
 pub(crate) fn import_candidates(
-    session: &Session,
-    source_span: &IndexVec<LocalDefId, Span>,
+    tcx: TyCtxt<'_>,
     err: &mut Diagnostic,
     // This is `None` if all placement locations are inside expansions
     use_placement_span: Option<Span>,
@@ -2355,8 +2365,7 @@ pub(crate) fn import_candidates(
     append: &str,
 ) {
     show_candidates(
-        session,
-        source_span,
+        tcx,
         err,
         use_placement_span,
         candidates,
@@ -2372,8 +2381,7 @@ pub(crate) fn import_candidates(
 /// entities with that name in all crates. This method allows outputting the
 /// results of this search in a programmer-friendly way
 fn show_candidates(
-    session: &Session,
-    source_span: &IndexVec<LocalDefId, Span>,
+    tcx: TyCtxt<'_>,
     err: &mut Diagnostic,
     // This is `None` if all placement locations are inside expansions
     use_placement_span: Option<Span>,
@@ -2498,8 +2506,8 @@ fn show_candidates(
             );
 
             if let Some(local_def_id) = def_id.and_then(|did| did.as_local()) {
-                let span = source_span[local_def_id];
-                let span = session.source_map().guess_head_span(span);
+                let span = tcx.source_span(local_def_id);
+                let span = tcx.sess.source_map().guess_head_span(span);
                 let mut multi_span = MultiSpan::from_span(span);
                 multi_span.push_span_label(span, "not accessible");
                 err.span_note(multi_span, &msg);
@@ -2529,8 +2537,8 @@ fn show_candidates(
             let mut spans = Vec::new();
             for (name, _, def_id, _) in &inaccessible_path_strings {
                 if let Some(local_def_id) = def_id.and_then(|did| did.as_local()) {
-                    let span = source_span[local_def_id];
-                    let span = session.source_map().guess_head_span(span);
+                    let span = tcx.source_span(local_def_id);
+                    let span = tcx.sess.source_map().guess_head_span(span);
                     spans.push((name, span));
                 } else {
                     if !has_colon {
diff --git a/compiler/rustc_resolve/src/effective_visibilities.rs b/compiler/rustc_resolve/src/effective_visibilities.rs
index b8efa3f8b..7bd90d7e3 100644
--- a/compiler/rustc_resolve/src/effective_visibilities.rs
+++ b/compiler/rustc_resolve/src/effective_visibilities.rs
@@ -1,4 +1,4 @@
-use crate::{NameBinding, NameBindingKind, Resolver, ResolverTree};
+use crate::{NameBinding, NameBindingKind, Resolver};
 use rustc_ast::ast;
 use rustc_ast::visit;
 use rustc_ast::visit::Visitor;
@@ -7,8 +7,8 @@ use rustc_ast::EnumDef;
 use rustc_data_structures::intern::Interned;
 use rustc_hir::def_id::LocalDefId;
 use rustc_hir::def_id::CRATE_DEF_ID;
+use rustc_middle::middle::privacy::Level;
 use rustc_middle::middle::privacy::{EffectiveVisibilities, EffectiveVisibility};
-use rustc_middle::middle::privacy::{IntoDefIdTree, Level};
 use rustc_middle::ty::{DefIdTree, Visibility};
 use std::mem;
 
@@ -29,8 +29,8 @@ impl ParentId<'_> {
     }
 }
 
-pub struct EffectiveVisibilitiesVisitor<'r, 'a> {
-    r: &'r mut Resolver<'a>,
+pub(crate) struct EffectiveVisibilitiesVisitor<'r, 'a, 'tcx> {
+    r: &'r mut Resolver<'a, 'tcx>,
     def_effective_visibilities: EffectiveVisibilities,
     /// While walking import chains we need to track effective visibilities per-binding, and def id
     /// keys in `Resolver::effective_visibilities` are not enough for that, because multiple
@@ -41,7 +41,7 @@ pub struct EffectiveVisibilitiesVisitor<'r, 'a> {
     changed: bool,
 }
 
-impl Resolver<'_> {
+impl Resolver<'_, '_> {
     fn nearest_normal_mod(&mut self, def_id: LocalDefId) -> LocalDefId {
         self.get_nearest_non_block_module(def_id.to_def_id()).nearest_parent_mod().expect_local()
     }
@@ -67,18 +67,14 @@ impl Resolver<'_> {
     }
 }
 
-impl<'a, 'b> IntoDefIdTree for &'b mut Resolver<'a> {
-    type Tree = &'b Resolver<'a>;
-    fn tree(self) -> Self::Tree {
-        self
-    }
-}
-
-impl<'r, 'a> EffectiveVisibilitiesVisitor<'r, 'a> {
+impl<'r, 'a, 'tcx> EffectiveVisibilitiesVisitor<'r, 'a, 'tcx> {
     /// Fills the `Resolver::effective_visibilities` table with public & exported items
     /// For now, this doesn't resolve macros (FIXME) and cannot resolve Impl, as we
     /// need access to a TyCtxt for that.
-    pub fn compute_effective_visibilities<'c>(r: &'r mut Resolver<'a>, krate: &'c Crate) {
+    pub(crate) fn compute_effective_visibilities<'c>(
+        r: &'r mut Resolver<'a, 'tcx>,
+        krate: &'c Crate,
+    ) {
         let mut visitor = EffectiveVisibilitiesVisitor {
             r,
             def_effective_visibilities: Default::default(),
@@ -104,11 +100,7 @@ impl<'r, 'a> EffectiveVisibilitiesVisitor<'r, 'a> {
         for (binding, eff_vis) in visitor.import_effective_visibilities.iter() {
             let NameBindingKind::Import { import, .. } = binding.kind else { unreachable!() };
             if let Some(node_id) = import.id() {
-                r.effective_visibilities.update_eff_vis(
-                    r.local_def_id(node_id),
-                    eff_vis,
-                    ResolverTree(&r.untracked),
-                )
+                r.effective_visibilities.update_eff_vis(r.local_def_id(node_id), eff_vis, r.tcx)
             }
         }
 
@@ -164,26 +156,28 @@ impl<'r, 'a> EffectiveVisibilitiesVisitor<'r, 'a> {
         let nominal_vis = binding.vis.expect_local();
         let private_vis = self.cheap_private_vis(parent_id);
         let inherited_eff_vis = self.effective_vis_or_private(parent_id);
+        let tcx = self.r.tcx;
         self.changed |= self.import_effective_visibilities.update(
             binding,
             nominal_vis,
-            |r| (private_vis.unwrap_or_else(|| r.private_vis_import(binding)), r),
+            || private_vis.unwrap_or_else(|| self.r.private_vis_import(binding)),
             inherited_eff_vis,
             parent_id.level(),
-            &mut *self.r,
+            tcx,
         );
     }
 
     fn update_def(&mut self, def_id: LocalDefId, nominal_vis: Visibility, parent_id: ParentId<'a>) {
         let private_vis = self.cheap_private_vis(parent_id);
         let inherited_eff_vis = self.effective_vis_or_private(parent_id);
+        let tcx = self.r.tcx;
         self.changed |= self.def_effective_visibilities.update(
             def_id,
             nominal_vis,
-            |r| (private_vis.unwrap_or_else(|| r.private_vis_def(def_id)), r),
+            || private_vis.unwrap_or_else(|| self.r.private_vis_def(def_id)),
             inherited_eff_vis,
             parent_id.level(),
-            &mut *self.r,
+            tcx,
         );
     }
 
@@ -192,7 +186,7 @@ impl<'r, 'a> EffectiveVisibilitiesVisitor<'r, 'a> {
     }
 }
 
-impl<'r, 'ast> Visitor<'ast> for EffectiveVisibilitiesVisitor<'ast, 'r> {
+impl<'r, 'ast, 'tcx> Visitor<'ast> for EffectiveVisibilitiesVisitor<'ast, 'r, 'tcx> {
     fn visit_item(&mut self, item: &'ast ast::Item) {
         let def_id = self.r.local_def_id(item.id);
         // Update effective visibilities of nested items.
diff --git a/compiler/rustc_resolve/src/errors.rs b/compiler/rustc_resolve/src/errors.rs
index 2c4427746..867363f42 100644
--- a/compiler/rustc_resolve/src/errors.rs
+++ b/compiler/rustc_resolve/src/errors.rs
@@ -38,7 +38,7 @@ pub(crate) struct NameAlreadyUsedInParameterList {
     #[primary_span]
     #[label]
     pub(crate) span: Span,
-    #[label(first_use_of_name)]
+    #[label(resolve_first_use_of_name)]
     pub(crate) first_use_span: Span,
     pub(crate) name: Symbol,
 }
@@ -121,7 +121,7 @@ pub(crate) struct VariableBoundWithDifferentMode {
     #[primary_span]
     #[label]
     pub(crate) span: Span,
-    #[label(first_binding_span)]
+    #[label(resolve_first_binding_span)]
     pub(crate) first_binding_span: Span,
     pub(crate) variable_name: Symbol,
 }
@@ -293,7 +293,7 @@ pub(crate) struct BindingShadowsSomethingUnacceptable<'a> {
     pub(crate) article: &'a str,
     #[subdiagnostic]
     pub(crate) sub_suggestion: Option<BindingShadowsSomethingUnacceptableSuggestion>,
-    #[label(label_shadowed_binding)]
+    #[label(resolve_label_shadowed_binding)]
     pub(crate) shadowed_binding_span: Span,
     pub(crate) participle: &'a str,
     pub(crate) name: Symbol,
@@ -369,7 +369,7 @@ pub(crate) struct UnreachableLabel {
     #[label]
     pub(crate) span: Span,
     pub(crate) name: Symbol,
-    #[label(label_definition_span)]
+    #[label(resolve_label_definition_span)]
     pub(crate) definition_span: Span,
     #[subdiagnostic]
     pub(crate) sub_suggestion: Option<UnreachableLabelSubSuggestion>,
@@ -413,7 +413,7 @@ pub(crate) struct TraitImplMismatch {
     pub(crate) span: Span,
     pub(crate) name: Symbol,
     pub(crate) kind: String,
-    #[label(label_trait_item)]
+    #[label(resolve_label_trait_item)]
     pub(crate) trait_item_span: Span,
     pub(crate) trait_path: String,
     pub(crate) code: String,
@@ -434,9 +434,9 @@ pub(crate) struct TraitImplDuplicate {
     #[primary_span]
     #[label]
     pub(crate) span: Span,
-    #[label(old_span_label)]
+    #[label(resolve_old_span_label)]
     pub(crate) old_span: Span,
-    #[label(trait_item_span)]
+    #[label(resolve_trait_item_span)]
     pub(crate) trait_item_span: Span,
     pub(crate) name: Symbol,
 }
diff --git a/compiler/rustc_resolve/src/ident.rs b/compiler/rustc_resolve/src/ident.rs
index a84652a31..52f0b65fa 100644
--- a/compiler/rustc_resolve/src/ident.rs
+++ b/compiler/rustc_resolve/src/ident.rs
@@ -7,7 +7,6 @@ use rustc_middle::ty;
 use rustc_session::lint::builtin::PROC_MACRO_DERIVE_RESOLUTION_FALLBACK;
 use rustc_session::lint::BuiltinLintDiagnostics;
 use rustc_span::def_id::LocalDefId;
-use rustc_span::edition::Edition;
 use rustc_span::hygiene::{ExpnId, ExpnKind, LocalExpnId, MacroKind, SyntaxContext};
 use rustc_span::symbol::{kw, Ident};
 use rustc_span::{Span, DUMMY_SP};
@@ -29,7 +28,7 @@ use RibKind::*;
 
 type Visibility = ty::Visibility<LocalDefId>;
 
-impl<'a> Resolver<'a> {
+impl<'a, 'tcx> Resolver<'a, 'tcx> {
     /// A generic scope visitor.
     /// Visits scopes in order to resolve some identifier in them or perform other actions.
     /// If the callback returns `Some` result, we stop visiting scopes and return it.
@@ -86,7 +85,7 @@ impl<'a> Resolver<'a> {
         // 4c. Standard library prelude (de-facto closed, controlled).
         // 6. Language prelude: builtin attributes (closed, controlled).
 
-        let rust_2015 = ctxt.edition() == Edition::Edition2015;
+        let rust_2015 = ctxt.edition().is_rust_2015();
         let (ns, macro_kind, is_absolute_path) = match scope_set {
             ScopeSet::All(ns, _) => (ns, None, false),
             ScopeSet::AbsolutePath(ns) => (ns, None, true),
@@ -369,7 +368,7 @@ impl<'a> Resolver<'a> {
     /// This is a variation of `fn resolve_ident_in_lexical_scope` that can be run during
     /// expansion and import resolution (perhaps they can be merged in the future).
     /// The function is used for resolving initial segments of macro paths (e.g., `foo` in
-    /// `foo::bar!(); or `foo!();`) and also for import paths on 2018 edition.
+    /// `foo::bar!();` or `foo!();`) and also for import paths on 2018 edition.
     #[instrument(level = "debug", skip(self, scope_set))]
     pub(crate) fn early_resolve_ident_in_lexical_scope(
         &mut self,
@@ -1180,7 +1179,7 @@ impl<'a> Resolver<'a> {
                         }
 
                         ConstantItemRibKind(trivial, _) => {
-                            let features = self.session.features_untracked();
+                            let features = self.tcx.sess.features_untracked();
                             // HACK(min_const_generics): We currently only allow `N` or `{ N }`.
                             if !(trivial == ConstantHasGenerics::Yes
                                 || features.generic_const_exprs)
@@ -1209,7 +1208,7 @@ impl<'a> Resolver<'a> {
                                                 is_type: true,
                                             },
                                         );
-                                        self.session.delay_span_bug(span, CG_BUG_STR);
+                                        self.tcx.sess.delay_span_bug(span, CG_BUG_STR);
                                     }
 
                                     return Res::Err;
@@ -1256,7 +1255,7 @@ impl<'a> Resolver<'a> {
                         | ForwardGenericParamBanRibKind => continue,
 
                         ConstantItemRibKind(trivial, _) => {
-                            let features = self.session.features_untracked();
+                            let features = self.tcx.sess.features_untracked();
                             // HACK(min_const_generics): We currently only allow `N` or `{ N }`.
                             if !(trivial == ConstantHasGenerics::Yes
                                 || features.generic_const_exprs)
@@ -1269,7 +1268,7 @@ impl<'a> Resolver<'a> {
                                             is_type: false,
                                         },
                                     );
-                                    self.session.delay_span_bug(span, CG_BUG_STR);
+                                    self.tcx.sess.delay_span_bug(span, CG_BUG_STR);
                                 }
 
                                 return Res::Err;
@@ -1398,7 +1397,10 @@ impl<'a> Resolver<'a> {
                         module = Some(ModuleOrUniformRoot::ExternPrelude);
                         continue;
                     }
-                    if name == kw::PathRoot && ident.span.rust_2015() && self.session.rust_2018() {
+                    if name == kw::PathRoot
+                        && ident.span.is_rust_2015()
+                        && self.tcx.sess.rust_2018()
+                    {
                         // `::a::b` from 2015 macro on 2018 global edition
                         module = Some(ModuleOrUniformRoot::CrateRootAndExternPrelude);
                         continue;
@@ -1494,7 +1496,8 @@ impl<'a> Resolver<'a> {
                         record_segment_res(self, res);
                     } else if res == Res::ToolMod && i + 1 != path.len() {
                         if binding.is_import() {
-                            self.session
+                            self.tcx
+                                .sess
                                 .struct_span_err(
                                     ident.span,
                                     "cannot use a tool module through an import",
diff --git a/compiler/rustc_resolve/src/imports.rs b/compiler/rustc_resolve/src/imports.rs
index 00f65ac37..4dab0836d 100644
--- a/compiler/rustc_resolve/src/imports.rs
+++ b/compiler/rustc_resolve/src/imports.rs
@@ -21,8 +21,8 @@ use rustc_middle::span_bug;
 use rustc_middle::ty;
 use rustc_session::lint::builtin::{PUB_USE_OF_PRIVATE_EXTERN_CRATE, UNUSED_IMPORTS};
 use rustc_session::lint::BuiltinLintDiagnostics;
+use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::hygiene::LocalExpnId;
-use rustc_span::lev_distance::find_best_match_for_name;
 use rustc_span::symbol::{kw, Ident, Symbol};
 use rustc_span::Span;
 
@@ -33,7 +33,7 @@ type Res = def::Res<NodeId>;
 
 /// Contains data for specific kinds of imports.
 #[derive(Clone)]
-pub enum ImportKind<'a> {
+pub(crate) enum ImportKind<'a> {
     Single {
         /// `source` in `use prefix::source as target`.
         source: Ident,
@@ -157,11 +157,11 @@ pub(crate) struct Import<'a> {
 }
 
 impl<'a> Import<'a> {
-    pub fn is_glob(&self) -> bool {
+    pub(crate) fn is_glob(&self) -> bool {
         matches!(self.kind, ImportKind::Glob { .. })
     }
 
-    pub fn is_nested(&self) -> bool {
+    pub(crate) fn is_nested(&self) -> bool {
         match self.kind {
             ImportKind::Single { nested, .. } => nested,
             _ => false,
@@ -210,6 +210,17 @@ impl<'a> NameResolution<'a> {
     }
 }
 
+/// An error that may be transformed into a diagnostic later. Used to combine multiple unresolved
+/// import errors within the same use tree into a single diagnostic.
+#[derive(Debug, Clone)]
+struct UnresolvedImportError {
+    span: Span,
+    label: Option<String>,
+    note: Option<String>,
+    suggestion: Option<Suggestion>,
+    candidates: Option<Vec<ImportSuggestion>>,
+}
+
 // Reexports of the form `pub use foo as bar;` where `foo` is `extern crate foo;`
 // are permitted for backward-compatibility under a deprecation lint.
 fn pub_use_of_private_extern_crate_hack(import: &Import<'_>, binding: &NameBinding<'_>) -> bool {
@@ -225,7 +236,7 @@ fn pub_use_of_private_extern_crate_hack(import: &Import<'_>, binding: &NameBindi
     }
 }
 
-impl<'a> Resolver<'a> {
+impl<'a, 'tcx> Resolver<'a, 'tcx> {
     /// Given a binding and an import that resolves to it,
     /// return the corresponding binding defined by the import.
     pub(crate) fn import(
@@ -333,7 +344,7 @@ impl<'a> Resolver<'a> {
     // If the resolution becomes a success, define it in the module's glob importers.
     fn update_resolution<T, F>(&mut self, module: Module<'a>, key: BindingKey, f: F) -> T
     where
-        F: FnOnce(&mut Resolver<'a>, &mut NameResolution<'a>) -> T,
+        F: FnOnce(&mut Resolver<'a, 'tcx>, &mut NameResolution<'a>) -> T,
     {
         // Ensure that `resolution` isn't borrowed when defining in the module's glob importers,
         // during which the resolution might end up getting re-defined via a glob cycle.
@@ -392,24 +403,7 @@ impl<'a> Resolver<'a> {
             }
         }
     }
-}
-
-/// An error that may be transformed into a diagnostic later. Used to combine multiple unresolved
-/// import errors within the same use tree into a single diagnostic.
-#[derive(Debug, Clone)]
-struct UnresolvedImportError {
-    span: Span,
-    label: Option<String>,
-    note: Option<String>,
-    suggestion: Option<Suggestion>,
-    candidates: Option<Vec<ImportSuggestion>>,
-}
-
-pub struct ImportResolver<'a, 'b> {
-    pub r: &'a mut Resolver<'b>,
-}
 
-impl<'a, 'b> ImportResolver<'a, 'b> {
     // Import resolution
     //
     // This is a fixed-point algorithm. We resolve imports until our efforts
@@ -420,29 +414,29 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
 
     /// Resolves all imports for the crate. This method performs the fixed-
     /// point iteration.
-    pub fn resolve_imports(&mut self) {
-        let mut prev_num_indeterminates = self.r.indeterminate_imports.len() + 1;
-        while self.r.indeterminate_imports.len() < prev_num_indeterminates {
-            prev_num_indeterminates = self.r.indeterminate_imports.len();
-            for import in mem::take(&mut self.r.indeterminate_imports) {
+    pub(crate) fn resolve_imports(&mut self) {
+        let mut prev_num_indeterminates = self.indeterminate_imports.len() + 1;
+        while self.indeterminate_imports.len() < prev_num_indeterminates {
+            prev_num_indeterminates = self.indeterminate_imports.len();
+            for import in mem::take(&mut self.indeterminate_imports) {
                 match self.resolve_import(&import) {
-                    true => self.r.determined_imports.push(import),
-                    false => self.r.indeterminate_imports.push(import),
+                    true => self.determined_imports.push(import),
+                    false => self.indeterminate_imports.push(import),
                 }
             }
         }
     }
 
-    pub fn finalize_imports(&mut self) {
-        for module in self.r.arenas.local_modules().iter() {
+    pub(crate) fn finalize_imports(&mut self) {
+        for module in self.arenas.local_modules().iter() {
             self.finalize_resolutions_in(module);
         }
 
         let mut seen_spans = FxHashSet::default();
         let mut errors = vec![];
         let mut prev_root_id: NodeId = NodeId::from_u32(0);
-        let determined_imports = mem::take(&mut self.r.determined_imports);
-        let indeterminate_imports = mem::take(&mut self.r.indeterminate_imports);
+        let determined_imports = mem::take(&mut self.determined_imports);
+        let indeterminate_imports = mem::take(&mut self.indeterminate_imports);
 
         for (is_indeterminate, import) in determined_imports
             .into_iter()
@@ -453,7 +447,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
 
             // If this import is unresolved then create a dummy import
             // resolution for it so that later resolve stages won't complain.
-            self.r.import_dummy_binding(import);
+            self.import_dummy_binding(import);
 
             if let Some(err) = unresolved_import_error {
                 if let ImportKind::Single { source, ref source_bindings, .. } = import.kind {
@@ -526,7 +520,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
             .collect::<Vec<_>>();
         let msg = format!("unresolved import{} {}", pluralize!(paths.len()), paths.join(", "),);
 
-        let mut diag = struct_span_err!(self.r.session, span, E0432, "{}", &msg);
+        let mut diag = struct_span_err!(self.tcx.sess, span, E0432, "{}", &msg);
 
         if let Some((_, UnresolvedImportError { note: Some(note), .. })) = errors.iter().last() {
             diag.note(note);
@@ -548,8 +542,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
             if let Some(candidates) = &err.candidates {
                 match &import.kind {
                     ImportKind::Single { nested: false, source, target, .. } => import_candidates(
-                        self.r.session,
-                        &self.r.untracked.source_span,
+                        self.tcx,
                         &mut diag,
                         Some(err.span),
                         &candidates,
@@ -561,8 +554,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
                     ),
                     ImportKind::Single { nested: true, source, target, .. } => {
                         import_candidates(
-                            self.r.session,
-                            &self.r.untracked.source_span,
+                            self.tcx,
                             &mut diag,
                             None,
                             &candidates,
@@ -583,7 +575,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
 
     /// Attempts to resolve the given import, returning true if its resolution is determined.
     /// If successful, the resolved bindings are written into the module.
-    fn resolve_import(&mut self, import: &'b Import<'b>) -> bool {
+    fn resolve_import(&mut self, import: &'a Import<'a>) -> bool {
         debug!(
             "(resolving import for module) resolving import `{}::...` in `{}`",
             Segment::names_to_string(&import.module_path),
@@ -596,8 +588,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
             // For better failure detection, pretend that the import will
             // not define any names while resolving its module path.
             let orig_vis = import.vis.take();
-            let path_res =
-                self.r.maybe_resolve_path(&import.module_path, None, &import.parent_scope);
+            let path_res = self.maybe_resolve_path(&import.module_path, None, &import.parent_scope);
             import.vis.set(orig_vis);
 
             match path_res {
@@ -625,7 +616,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
         };
 
         let mut indeterminate = false;
-        self.r.per_ns(|this, ns| {
+        self.per_ns(|this, ns| {
             if !type_ns_only || ns == TypeNS {
                 if let Err(Undetermined) = source_bindings[ns].get() {
                     // For better failure detection, pretend that the import will
@@ -658,7 +649,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
                     source_binding @ (Ok(..) | Err(Determined)) => {
                         if source_binding.is_ok() {
                             let msg = format!("`{}` is not directly importable", target);
-                            struct_span_err!(this.session, import.span, E0253, "{}", &msg)
+                            struct_span_err!(this.tcx.sess, import.span, E0253, "{}", &msg)
                                 .span_label(import.span, "cannot be imported directly")
                                 .emit();
                         }
@@ -678,15 +669,15 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
     ///
     /// Optionally returns an unresolved import error. This error is buffered and used to
     /// consolidate multiple unresolved import errors into a single diagnostic.
-    fn finalize_import(&mut self, import: &'b Import<'b>) -> Option<UnresolvedImportError> {
+    fn finalize_import(&mut self, import: &'a Import<'a>) -> Option<UnresolvedImportError> {
         let orig_vis = import.vis.take();
         let ignore_binding = match &import.kind {
             ImportKind::Single { target_bindings, .. } => target_bindings[TypeNS].get(),
             _ => None,
         };
-        let prev_ambiguity_errors_len = self.r.ambiguity_errors.len();
+        let prev_ambiguity_errors_len = self.ambiguity_errors.len();
         let finalize = Finalize::with_root_span(import.root_id, import.span, import.root_span);
-        let path_res = self.r.resolve_path(
+        let path_res = self.resolve_path(
             &import.module_path,
             None,
             &import.parent_scope,
@@ -694,7 +685,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
             ignore_binding,
         );
 
-        let no_ambiguity = self.r.ambiguity_errors.len() == prev_ambiguity_errors_len;
+        let no_ambiguity = self.ambiguity_errors.len() == prev_ambiguity_errors_len;
         import.vis.set(orig_vis);
         let module = match path_res {
             PathResult::Module(module) => {
@@ -703,10 +694,10 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
                     if !ModuleOrUniformRoot::same_def(module, initial_module) && no_ambiguity {
                         span_bug!(import.span, "inconsistent resolution for an import");
                     }
-                } else if self.r.privacy_errors.is_empty() {
+                } else if self.privacy_errors.is_empty() {
                     let msg = "cannot determine resolution for the import";
                     let msg_note = "import resolution is stuck, try simplifying other imports";
-                    self.r.session.struct_span_err(import.span, msg).note(msg_note).emit();
+                    self.tcx.sess.struct_span_err(import.span, msg).note(msg_note).emit();
                 }
 
                 module
@@ -714,8 +705,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
             PathResult::Failed { is_error_from_last_segment: false, span, label, suggestion } => {
                 if no_ambiguity {
                     assert!(import.imported_module.get().is_none());
-                    self.r
-                        .report_error(span, ResolutionError::FailedToResolve { label, suggestion });
+                    self.report_error(span, ResolutionError::FailedToResolve { label, suggestion });
                 }
                 return None;
             }
@@ -777,7 +767,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
                         // 2 segments, so the `resolve_path` above won't trigger it.
                         let mut full_path = import.module_path.clone();
                         full_path.push(Segment::from_ident(Ident::empty()));
-                        self.r.lint_if_path_starts_with_module(Some(finalize), &full_path, None);
+                        self.lint_if_path_starts_with_module(Some(finalize), &full_path, None);
                     }
 
                     if let ModuleOrUniformRoot::Module(module) = module {
@@ -796,10 +786,10 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
                     }
                     if !is_prelude
                     && let Some(max_vis) = max_vis.get()
-                    && !max_vis.is_at_least(import.expect_vis(), &*self.r)
+                    && !max_vis.is_at_least(import.expect_vis(), &*self)
                 {
                     let msg = "glob import doesn't reexport anything because no candidate is public enough";
-                    self.r.lint_buffer.buffer_lint(UNUSED_IMPORTS, id, import.span, msg);
+                    self.lint_buffer.buffer_lint(UNUSED_IMPORTS, id, import.span, msg);
                 }
                     return None;
                 }
@@ -807,7 +797,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
             };
 
         let mut all_ns_err = true;
-        self.r.per_ns(|this, ns| {
+        self.per_ns(|this, ns| {
             if !type_ns_only || ns == TypeNS {
                 let orig_vis = import.vis.take();
                 let binding = this.resolve_ident_in_module(
@@ -859,7 +849,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
                             let msg = "cannot determine resolution for the import";
                             let msg_note =
                                 "import resolution is stuck, try simplifying other imports";
-                            this.session.struct_span_err(import.span, msg).note(msg_note).emit();
+                            this.tcx.sess.struct_span_err(import.span, msg).note(msg_note).emit();
                         }
                     }
                     Err(..) => {
@@ -876,7 +866,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
 
         if all_ns_err {
             let mut all_ns_failed = true;
-            self.r.per_ns(|this, ns| {
+            self.per_ns(|this, ns| {
                 if !type_ns_only || ns == TypeNS {
                     let binding = this.resolve_ident_in_module(
                         module,
@@ -894,9 +884,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
 
             return if all_ns_failed {
                 let resolutions = match module {
-                    ModuleOrUniformRoot::Module(module) => {
-                        Some(self.r.resolutions(module).borrow())
-                    }
+                    ModuleOrUniformRoot::Module(module) => Some(self.resolutions(module).borrow()),
                     _ => None,
                 };
                 let resolutions = resolutions.as_ref().into_iter().flat_map(|r| r.iter());
@@ -965,7 +953,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
                 };
 
                 let parent_suggestion =
-                    self.r.lookup_import_candidates(ident, TypeNS, &import.parent_scope, |_| true);
+                    self.lookup_import_candidates(ident, TypeNS, &import.parent_scope, |_| true);
 
                 Some(UnresolvedImportError {
                     span: import.span,
@@ -987,7 +975,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
         let mut reexport_error = None;
         let mut any_successful_reexport = false;
         let mut crate_private_reexport = false;
-        self.r.per_ns(|this, ns| {
+        self.per_ns(|this, ns| {
             if let Ok(binding) = source_bindings[ns].get() {
                 if !binding.vis.is_at_least(import.expect_vis(), &*this) {
                     reexport_error = Some((ns, binding));
@@ -1012,7 +1000,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
                                    `pub`",
                     ident
                 );
-                self.r.lint_buffer.buffer_lint(
+                self.lint_buffer.buffer_lint(
                     PUB_USE_OF_PRIVATE_EXTERN_CRATE,
                     import_id,
                     import.span,
@@ -1035,17 +1023,17 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
                         format!("re-export of private `{}`", ident)
                     };
 
-                    struct_span_err!(self.r.session, import.span, E0365, "{}", error_msg)
+                    struct_span_err!(self.tcx.sess, import.span, E0365, "{}", error_msg)
                         .span_label(import.span, label_msg)
                         .note(&format!("consider declaring type or module `{}` with `pub`", ident))
                         .emit();
                 } else {
                     let mut err =
-                        struct_span_err!(self.r.session, import.span, E0364, "{error_msg}");
+                        struct_span_err!(self.tcx.sess, import.span, E0364, "{error_msg}");
                     match binding.kind {
                         NameBindingKind::Res(Res::Def(DefKind::Macro(_), def_id))
                             // exclude decl_macro
-                            if self.r.get_macro_by_def_id(def_id).macro_rules =>
+                            if self.get_macro_by_def_id(def_id).macro_rules =>
                         {
                             err.span_help(
                                 binding.span,
@@ -1071,7 +1059,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
             // 2 segments, so the `resolve_path` above won't trigger it.
             let mut full_path = import.module_path.clone();
             full_path.push(Segment::from_ident(ident));
-            self.r.per_ns(|this, ns| {
+            self.per_ns(|this, ns| {
                 if let Ok(binding) = source_bindings[ns].get() {
                     this.lint_if_path_starts_with_module(Some(finalize), &full_path, Some(binding));
                 }
@@ -1081,7 +1069,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
         // Record what this import resolves to for later uses in documentation,
         // this may resolve to either a value or a type, but for documentation
         // purposes it's good enough to just favor one over the other.
-        self.r.per_ns(|this, ns| {
+        self.per_ns(|this, ns| {
             if let Ok(binding) = source_bindings[ns].get() {
                 this.import_res_map.entry(import_id).or_default()[ns] = Some(binding.res());
             }
@@ -1096,9 +1084,9 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
     fn check_for_redundant_imports(
         &mut self,
         ident: Ident,
-        import: &'b Import<'b>,
-        source_bindings: &PerNS<Cell<Result<&'b NameBinding<'b>, Determinacy>>>,
-        target_bindings: &PerNS<Cell<Option<&'b NameBinding<'b>>>>,
+        import: &'a Import<'a>,
+        source_bindings: &PerNS<Cell<Result<&'a NameBinding<'a>, Determinacy>>>,
+        target_bindings: &PerNS<Cell<Option<&'a NameBinding<'a>>>>,
         target: Ident,
     ) {
         // This function is only called for single imports.
@@ -1119,7 +1107,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
 
         let mut redundant_span = PerNS { value_ns: None, type_ns: None, macro_ns: None };
 
-        self.r.per_ns(|this, ns| {
+        self.per_ns(|this, ns| {
             if let Ok(binding) = source_bindings[ns].get() {
                 if binding.res() == Res::Err {
                     return;
@@ -1149,7 +1137,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
             let mut redundant_spans: Vec<_> = redundant_span.present_items().collect();
             redundant_spans.sort();
             redundant_spans.dedup();
-            self.r.lint_buffer.buffer_lint_with_diagnostic(
+            self.lint_buffer.buffer_lint_with_diagnostic(
                 UNUSED_IMPORTS,
                 id,
                 import.span,
@@ -1159,22 +1147,22 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
         }
     }
 
-    fn resolve_glob_import(&mut self, import: &'b Import<'b>) {
+    fn resolve_glob_import(&mut self, import: &'a Import<'a>) {
         // This function is only called for glob imports.
         let ImportKind::Glob { id, is_prelude, .. } = import.kind else { unreachable!() };
 
         let ModuleOrUniformRoot::Module(module) = import.imported_module.get().unwrap() else {
-            self.r.session.span_err(import.span, "cannot glob-import all possible crates");
+            self.tcx.sess.span_err(import.span, "cannot glob-import all possible crates");
             return;
         };
 
         if module.is_trait() {
-            self.r.session.span_err(import.span, "items in traits are not importable");
+            self.tcx.sess.span_err(import.span, "items in traits are not importable");
             return;
         } else if ptr::eq(module, import.parent_scope.module) {
             return;
         } else if is_prelude {
-            self.r.prelude = Some(module);
+            self.prelude = Some(module);
             return;
         }
 
@@ -1184,7 +1172,6 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
         // Ensure that `resolutions` isn't borrowed during `try_define`,
         // since it might get updated via a glob cycle.
         let bindings = self
-            .r
             .resolutions(module)
             .borrow()
             .iter()
@@ -1194,30 +1181,30 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
             .collect::<Vec<_>>();
         for (mut key, binding) in bindings {
             let scope = match key.ident.span.reverse_glob_adjust(module.expansion, import.span) {
-                Some(Some(def)) => self.r.expn_def_scope(def),
+                Some(Some(def)) => self.expn_def_scope(def),
                 Some(None) => import.parent_scope.module,
                 None => continue,
             };
-            if self.r.is_accessible_from(binding.vis, scope) {
-                let imported_binding = self.r.import(binding, import);
-                let _ = self.r.try_define(import.parent_scope.module, key, imported_binding);
+            if self.is_accessible_from(binding.vis, scope) {
+                let imported_binding = self.import(binding, import);
+                let _ = self.try_define(import.parent_scope.module, key, imported_binding);
             }
         }
 
         // Record the destination of this import
-        self.r.record_partial_res(id, PartialRes::new(module.res().unwrap()));
+        self.record_partial_res(id, PartialRes::new(module.res().unwrap()));
     }
 
     // Miscellaneous post-processing, including recording re-exports,
     // reporting conflicts, and reporting unresolved imports.
-    fn finalize_resolutions_in(&mut self, module: Module<'b>) {
+    fn finalize_resolutions_in(&mut self, module: Module<'a>) {
         // Since import resolution is finished, globs will not define any more names.
         *module.globs.borrow_mut() = Vec::new();
 
         if let Some(def_id) = module.opt_def_id() {
             let mut reexports = Vec::new();
 
-            module.for_each_child(self.r, |this, ident, _, binding| {
+            module.for_each_child(self, |this, ident, _, binding| {
                 if let Some(res) = this.is_reexport(binding) {
                     reexports.push(ModChild {
                         ident,
@@ -1232,7 +1219,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
             if !reexports.is_empty() {
                 // Call to `expect_local` should be fine because current
                 // code is only called for local modules.
-                self.r.reexport_map.insert(def_id.expect_local(), reexports);
+                self.reexport_map.insert(def_id.expect_local(), reexports);
             }
         }
     }
diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs
index 83932c089..7df17376b 100644
--- a/compiler/rustc_resolve/src/late.rs
+++ b/compiler/rustc_resolve/src/late.rs
@@ -8,7 +8,7 @@
 
 use RibKind::*;
 
-use crate::{path_names_to_string, BindingError, Finalize, LexicalScopeBinding};
+use crate::{path_names_to_string, rustdoc, BindingError, Finalize, LexicalScopeBinding};
 use crate::{Module, ModuleOrUniformRoot, NameBinding, ParentScope, PathResult};
 use crate::{ResolutionError, Resolver, Segment, UseError};
 
@@ -21,15 +21,16 @@ use rustc_hir::def::Namespace::{self, *};
 use rustc_hir::def::{self, CtorKind, DefKind, LifetimeRes, PartialRes, PerNS};
 use rustc_hir::def_id::{DefId, LocalDefId, CRATE_DEF_ID, LOCAL_CRATE};
 use rustc_hir::{BindingAnnotation, PrimTy, TraitCandidate};
-use rustc_middle::middle::resolve_lifetime::Set1;
+use rustc_middle::middle::resolve_bound_vars::Set1;
 use rustc_middle::ty::DefIdTree;
 use rustc_middle::{bug, span_bug};
+use rustc_session::config::{CrateType, ResolveDocLinks};
 use rustc_session::lint;
+use rustc_span::source_map::{respan, Spanned};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
-use rustc_span::{BytePos, Span};
+use rustc_span::{BytePos, Span, SyntaxContext};
 use smallvec::{smallvec, SmallVec};
 
-use rustc_span::source_map::{respan, Spanned};
 use std::assert_matches::debug_assert_matches;
 use std::borrow::Cow;
 use std::collections::{hash_map::Entry, BTreeSet};
@@ -55,7 +56,7 @@ struct BindingInfo {
 }
 
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
-pub enum PatternSource {
+pub(crate) enum PatternSource {
     Match,
     Let,
     For,
@@ -69,7 +70,7 @@ enum IsRepeatExpr {
 }
 
 impl PatternSource {
-    pub fn descr(self) -> &'static str {
+    fn descr(self) -> &'static str {
         match self {
             PatternSource::Match => "match binding",
             PatternSource::Let => "let binding",
@@ -493,6 +494,30 @@ impl<'a> PathSource<'a> {
     }
 }
 
+/// At this point for most items we can answer whether that item is exported or not,
+/// but some items like impls require type information to determine exported-ness, so we make a
+/// conservative estimate for them (e.g. based on nominal visibility).
+#[derive(Clone, Copy)]
+enum MaybeExported<'a> {
+    Ok(NodeId),
+    Impl(Option<DefId>),
+    ImplItem(Result<DefId, &'a Visibility>),
+}
+
+impl MaybeExported<'_> {
+    fn eval(self, r: &Resolver<'_, '_>) -> bool {
+        let def_id = match self {
+            MaybeExported::Ok(node_id) => Some(r.local_def_id(node_id)),
+            MaybeExported::Impl(Some(trait_def_id)) | MaybeExported::ImplItem(Ok(trait_def_id)) => {
+                trait_def_id.as_local()
+            }
+            MaybeExported::Impl(None) => return true,
+            MaybeExported::ImplItem(Err(vis)) => return vis.kind.is_pub(),
+        };
+        def_id.map_or(true, |def_id| r.effective_visibilities.is_exported(def_id))
+    }
+}
+
 #[derive(Default)]
 struct DiagnosticMetadata<'ast> {
     /// The current trait's associated items' ident, used for diagnostic suggestions.
@@ -559,8 +584,8 @@ struct DiagnosticMetadata<'ast> {
     current_elision_failures: Vec<MissingLifetime>,
 }
 
-struct LateResolutionVisitor<'a, 'b, 'ast> {
-    r: &'b mut Resolver<'a>,
+struct LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
+    r: &'b mut Resolver<'a, 'tcx>,
 
     /// The module that represents the current item scope.
     parent_scope: ParentScope<'a>,
@@ -603,7 +628,7 @@ struct LateResolutionVisitor<'a, 'b, 'ast> {
 }
 
 /// Walks the whole crate in DFS order, visiting each item, resolving names as it goes.
-impl<'a: 'ast, 'ast> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast> {
+impl<'a: 'ast, 'ast, 'tcx> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
     fn visit_attribute(&mut self, _: &'ast Attribute) {
         // We do not want to resolve expressions that appear in attributes,
         // as they do not correspond to actual code.
@@ -620,7 +645,9 @@ impl<'a: 'ast, 'ast> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast> {
         self.resolve_arm(arm);
     }
     fn visit_block(&mut self, block: &'ast Block) {
+        let old_macro_rules = self.parent_scope.macro_rules;
         self.resolve_block(block);
+        self.parent_scope.macro_rules = old_macro_rules;
     }
     fn visit_anon_const(&mut self, constant: &'ast AnonConst) {
         // We deal with repeat expressions explicitly in `resolve_expr`.
@@ -655,7 +682,7 @@ impl<'a: 'ast, 'ast> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast> {
                 // Elided lifetime in reference: we resolve as if there was some lifetime `'_` with
                 // NodeId `ty.id`.
                 // This span will be used in case of elision failure.
-                let span = self.r.session.source_map().start_point(ty.span);
+                let span = self.r.tcx.sess.source_map().start_point(ty.span);
                 self.resolve_elided_lifetime(ty.id, span);
                 visit::walk_ty(self, ty);
             }
@@ -771,6 +798,7 @@ impl<'a: 'ast, 'ast> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast> {
         );
     }
     fn visit_foreign_item(&mut self, foreign_item: &'ast ForeignItem) {
+        self.resolve_doc_links(&foreign_item.attrs, MaybeExported::Ok(foreign_item.id));
         match foreign_item.kind {
             ForeignItemKind::TyAlias(box TyAlias { ref generics, .. }) => {
                 self.with_generic_param_rib(
@@ -1159,10 +1187,20 @@ impl<'a: 'ast, 'ast> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast> {
             })
         });
     }
+
+    fn visit_variant(&mut self, v: &'ast Variant) {
+        self.resolve_doc_links(&v.attrs, MaybeExported::Ok(v.id));
+        visit::walk_variant(self, v)
+    }
+
+    fn visit_field_def(&mut self, f: &'ast FieldDef) {
+        self.resolve_doc_links(&f.attrs, MaybeExported::Ok(f.id));
+        visit::walk_field_def(self, f)
+    }
 }
 
-impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
-    fn new(resolver: &'b mut Resolver<'a>) -> LateResolutionVisitor<'a, 'b, 'ast> {
+impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
+    fn new(resolver: &'b mut Resolver<'a, 'tcx>) -> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
         // During late resolution we only track the module component of the parent scope,
         // although it may be useful to track other components as well for diagnostics.
         let graph_root = resolver.graph_root;
@@ -1533,7 +1571,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
                         ("`'_` cannot be used here", "`'_` is a reserved lifetime name")
                     };
                     let mut diag = rustc_errors::struct_span_err!(
-                        self.r.session,
+                        self.r.tcx.sess,
                         lifetime.ident.span,
                         E0637,
                         "{}",
@@ -1710,7 +1748,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
                     //     impl Foo for std::cell::Ref<u32> // note lack of '_
                     //     async fn foo(_: std::cell::Ref<u32>) { ... }
                     LifetimeRibKind::AnonymousCreateParameter { report_in_path: true, .. } => {
-                        let sess = self.r.session;
+                        let sess = self.r.tcx.sess;
                         let mut err = rustc_errors::struct_span_err!(
                             sess,
                             path_span,
@@ -1725,7 +1763,6 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
                             !segment.has_generic_args,
                             elided_lifetime_span,
                         );
-                        err.note("assuming a `'static` lifetime...");
                         err.emit();
                         should_lint = false;
 
@@ -1992,13 +2029,13 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
 
     /// List all the lifetimes that appear in the provided type.
     fn find_lifetime_for_self(&self, ty: &'ast Ty) -> Set1<LifetimeRes> {
-        struct SelfVisitor<'r, 'a> {
-            r: &'r Resolver<'a>,
+        struct SelfVisitor<'r, 'a, 'tcx> {
+            r: &'r Resolver<'a, 'tcx>,
             impl_self: Option<Res>,
             lifetime: Set1<LifetimeRes>,
         }
 
-        impl SelfVisitor<'_, '_> {
+        impl SelfVisitor<'_, '_, '_> {
             // Look for `self: &'a Self` - also desugared from `&'a self`,
             // and if that matches, use it for elision and return early.
             fn is_self_ty(&self, ty: &Ty) -> bool {
@@ -2016,7 +2053,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
             }
         }
 
-        impl<'a> Visitor<'a> for SelfVisitor<'_, '_> {
+        impl<'a> Visitor<'a> for SelfVisitor<'_, '_, '_> {
             fn visit_ty(&mut self, ty: &'a Ty) {
                 trace!("SelfVisitor considering ty={:?}", ty);
                 if let TyKind::Ref(lt, ref mt) = ty.kind && self.is_self_ty(&mt.ty) {
@@ -2145,7 +2182,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
         let segments = &use_tree.prefix.segments;
         if !segments.is_empty() {
             let ident = segments[0].ident;
-            if ident.is_path_segment_keyword() || ident.span.rust_2015() {
+            if ident.is_path_segment_keyword() || ident.span.is_rust_2015() {
                 return;
             }
 
@@ -2157,7 +2194,8 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
                 let what = if ns == TypeNS { "type parameters" } else { "local variables" };
                 if this.should_report_errs() {
                     this.r
-                        .session
+                        .tcx
+                        .sess
                         .span_err(ident.span, &format!("imports cannot refer to {}", what));
                 }
             };
@@ -2185,6 +2223,12 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
     }
 
     fn resolve_item(&mut self, item: &'ast Item) {
+        let mod_inner_docs =
+            matches!(item.kind, ItemKind::Mod(..)) && rustdoc::inner_docs(&item.attrs);
+        if !mod_inner_docs && !matches!(item.kind, ItemKind::Impl(..)) {
+            self.resolve_doc_links(&item.attrs, MaybeExported::Ok(item.id));
+        }
+
         let name = item.ident.name;
         debug!("(resolving item) resolving {} ({:?})", name, item.kind);
 
@@ -2229,7 +2273,14 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
                 ..
             }) => {
                 self.diagnostic_metadata.current_impl_items = Some(impl_items);
-                self.resolve_implementation(generics, of_trait, &self_ty, item.id, impl_items);
+                self.resolve_implementation(
+                    &item.attrs,
+                    generics,
+                    of_trait,
+                    &self_ty,
+                    item.id,
+                    impl_items,
+                );
                 self.diagnostic_metadata.current_impl_items = None;
             }
 
@@ -2274,9 +2325,20 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
                 );
             }
 
-            ItemKind::Mod(..) | ItemKind::ForeignMod(_) => {
+            ItemKind::Mod(..) => {
                 self.with_scope(item.id, |this| {
+                    if mod_inner_docs {
+                        this.resolve_doc_links(&item.attrs, MaybeExported::Ok(item.id));
+                    }
+                    let old_macro_rules = this.parent_scope.macro_rules;
                     visit::walk_item(this, item);
+                    // Maintain macro_rules scopes in the same way as during early resolution
+                    // for diagnostics and doc links.
+                    if item.attrs.iter().all(|attr| {
+                        !attr.has_name(sym::macro_use) && !attr.has_name(sym::macro_escape)
+                    }) {
+                        this.parent_scope.macro_rules = old_macro_rules;
+                    }
                 });
             }
 
@@ -2309,14 +2371,21 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
                 self.future_proof_import(use_tree);
             }
 
-            ItemKind::ExternCrate(..) | ItemKind::MacroDef(..) => {
-                // do nothing, these are just around to be encoded
+            ItemKind::MacroDef(ref macro_def) => {
+                // Maintain macro_rules scopes in the same way as during early resolution
+                // for diagnostics and doc links.
+                if macro_def.macro_rules {
+                    let def_id = self.r.local_def_id(item.id);
+                    self.parent_scope.macro_rules = self.r.macro_rules_scopes[&def_id];
+                }
             }
 
-            ItemKind::GlobalAsm(_) => {
+            ItemKind::ForeignMod(_) | ItemKind::GlobalAsm(_) => {
                 visit::walk_item(self, item);
             }
 
+            ItemKind::ExternCrate(..) => {}
+
             ItemKind::MacCall(_) => panic!("unexpanded macro in resolve!"),
         }
     }
@@ -2370,7 +2439,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
             if let GenericParamKind::Lifetime = param.kind
                 && let Some(&original) = seen_lifetimes.get(&ident)
             {
-                diagnostics::signal_lifetime_shadowing(self.r.session, original, param.ident);
+                diagnostics::signal_lifetime_shadowing(self.r.tcx.sess, original, param.ident);
                 // Record lifetime res, so lowering knows there is something fishy.
                 self.record_lifetime_param(param.id, LifetimeRes::Error);
                 continue;
@@ -2394,7 +2463,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
 
             if param.ident.name == kw::UnderscoreLifetime {
                 rustc_errors::struct_span_err!(
-                    self.r.session,
+                    self.r.tcx.sess,
                     param.ident.span,
                     E0637,
                     "`'_` cannot be used here"
@@ -2408,7 +2477,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
 
             if param.ident.name == kw::StaticLifetime {
                 rustc_errors::struct_span_err!(
-                    self.r.session,
+                    self.r.tcx.sess,
                     param.ident.span,
                     E0262,
                     "invalid lifetime parameter name: `{}`",
@@ -2437,7 +2506,13 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
 
             let res = match kind {
                 ItemRibKind(..) | AssocItemRibKind => Res::Def(def_kind, def_id.to_def_id()),
-                NormalRibKind => Res::Err,
+                NormalRibKind => {
+                    if self.r.tcx.sess.features_untracked().non_lifetime_binders {
+                        Res::Def(def_kind, def_id.to_def_id())
+                    } else {
+                        Res::Err
+                    }
+                }
                 _ => span_bug!(param.ident.span, "Unexpected rib kind {:?}", kind),
             };
             self.r.record_partial_res(param.id, PartialRes::new(res));
@@ -2544,6 +2619,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
             };
 
         for item in trait_items {
+            self.resolve_doc_links(&item.attrs, MaybeExported::Ok(item.id));
             match &item.kind {
                 AssocItemKind::Const(_, ty, default) => {
                     self.visit_ty(ty);
@@ -2631,6 +2707,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
 
     fn resolve_implementation(
         &mut self,
+        attrs: &[ast::Attribute],
         generics: &'ast Generics,
         opt_trait_reference: &'ast Option<TraitRef>,
         self_type: &'ast Ty,
@@ -2661,6 +2738,8 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
                                 opt_trait_reference.as_ref(),
                                 self_type,
                                 |this, trait_id| {
+                                    this.resolve_doc_links(attrs, MaybeExported::Impl(trait_id));
+
                                     let item_def_id = this.r.local_def_id(item_id);
 
                                     // Register the trait definitions from here.
@@ -2694,7 +2773,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
                                                 debug!("resolve_implementation with_self_rib_ns(ValueNS, ...)");
                                                 let mut seen_trait_items = Default::default();
                                                 for item in impl_items {
-                                                    this.resolve_impl_item(&**item, &mut seen_trait_items);
+                                                    this.resolve_impl_item(&**item, &mut seen_trait_items, trait_id);
                                                 }
                                             });
                                         });
@@ -2712,8 +2791,10 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
         &mut self,
         item: &'ast AssocItem,
         seen_trait_items: &mut FxHashMap<DefId, Span>,
+        trait_id: Option<DefId>,
     ) {
         use crate::ResolutionError::*;
+        self.resolve_doc_links(&item.attrs, MaybeExported::ImplItem(trait_id.ok_or(&item.vis)));
         match &item.kind {
             AssocItemKind::Const(_, ty, default) => {
                 debug!("resolve_implementation AssocItemKind::Const");
@@ -3304,7 +3385,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
             Res::SelfCtor(_) => {
                 // We resolve `Self` in pattern position as an ident sometimes during recovery,
                 // so delay a bug instead of ICEing.
-                self.r.session.delay_span_bug(
+                self.r.tcx.sess.delay_span_bug(
                     ident.span,
                     "unexpected `SelfCtor` in pattern, expected identifier"
                 );
@@ -3584,7 +3665,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
     #[inline]
     /// If we're actually rustdoc then avoid giving a name resolution error for `cfg()` items.
     fn should_report_errs(&self) -> bool {
-        !(self.r.session.opts.actually_rustdoc && self.in_func_body)
+        !(self.r.tcx.sess.opts.actually_rustdoc && self.in_func_body)
     }
 
     // Resolve in alternative namespaces if resolution in the primary namespace fails.
@@ -3749,7 +3830,7 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
             }
 
             if let Ok((_, orig_span)) = self.resolve_label(label.ident) {
-                diagnostics::signal_label_shadowing(self.r.session, orig_span, label.ident)
+                diagnostics::signal_label_shadowing(self.r.tcx.sess, orig_span, label.ident)
             }
 
             self.with_label_rib(NormalRibKind, |this| {
@@ -4055,9 +4136,9 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
     fn record_candidate_traits_for_expr_if_necessary(&mut self, expr: &'ast Expr) {
         match expr.kind {
             ExprKind::Field(_, ident) => {
-                // FIXME(#6890): Even though you can't treat a method like a
-                // field, we need to add any trait methods we find that match
-                // the field name so that we can do some nice error reporting
+                // #6890: Even though you can't treat a method like a field,
+                // we need to add any trait methods we find that match the
+                // field name so that we can do some nice error reporting
                 // later on in typeck.
                 let traits = self.traits_in_scope(ident, ValueNS);
                 self.r.trait_map.insert(expr.id, traits);
@@ -4116,15 +4197,116 @@ impl<'a: 'ast, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
             self.r.extra_lifetime_params_map.insert(async_node_id, extra_lifetime_params);
         }
     }
+
+    fn resolve_and_cache_rustdoc_path(&mut self, path_str: &str, ns: Namespace) -> bool {
+        // FIXME: This caching may be incorrect in case of multiple `macro_rules`
+        // items with the same name in the same module.
+        // Also hygiene is not considered.
+        let mut doc_link_resolutions = std::mem::take(&mut self.r.doc_link_resolutions);
+        let res = doc_link_resolutions
+            .entry(self.parent_scope.module.nearest_parent_mod().expect_local())
+            .or_default()
+            .entry((Symbol::intern(path_str), ns))
+            .or_insert_with_key(|(path, ns)| {
+                let res = self.r.resolve_rustdoc_path(path.as_str(), *ns, self.parent_scope);
+                if let Some(res) = res
+                    && let Some(def_id) = res.opt_def_id()
+                    && !def_id.is_local()
+                    && self.r.tcx.sess.crate_types().contains(&CrateType::ProcMacro)
+                    && matches!(self.r.tcx.sess.opts.resolve_doc_links, ResolveDocLinks::ExportedMetadata) {
+                    // Encoding foreign def ids in proc macro crate metadata will ICE.
+                    return None;
+                }
+                res
+            })
+            .is_some();
+        self.r.doc_link_resolutions = doc_link_resolutions;
+        res
+    }
+
+    fn resolve_doc_links(&mut self, attrs: &[Attribute], maybe_exported: MaybeExported<'_>) {
+        match self.r.tcx.sess.opts.resolve_doc_links {
+            ResolveDocLinks::None => return,
+            ResolveDocLinks::ExportedMetadata
+                if !self.r.tcx.sess.crate_types().iter().copied().any(CrateType::has_metadata)
+                    || !maybe_exported.eval(self.r) =>
+            {
+                return;
+            }
+            ResolveDocLinks::Exported if !maybe_exported.eval(self.r) => {
+                return;
+            }
+            ResolveDocLinks::ExportedMetadata
+            | ResolveDocLinks::Exported
+            | ResolveDocLinks::All => {}
+        }
+
+        if !attrs.iter().any(|attr| attr.may_have_doc_links()) {
+            return;
+        }
+
+        let mut need_traits_in_scope = false;
+        for path_str in rustdoc::attrs_to_preprocessed_links(attrs) {
+            // Resolve all namespaces due to no disambiguator or for diagnostics.
+            let mut any_resolved = false;
+            let mut need_assoc = false;
+            for ns in [TypeNS, ValueNS, MacroNS] {
+                if self.resolve_and_cache_rustdoc_path(&path_str, ns) {
+                    any_resolved = true;
+                } else if ns != MacroNS {
+                    need_assoc = true;
+                }
+            }
+
+            // Resolve all prefixes for type-relative resolution or for diagnostics.
+            if need_assoc || !any_resolved {
+                let mut path = &path_str[..];
+                while let Some(idx) = path.rfind("::") {
+                    path = &path[..idx];
+                    need_traits_in_scope = true;
+                    for ns in [TypeNS, ValueNS, MacroNS] {
+                        self.resolve_and_cache_rustdoc_path(path, ns);
+                    }
+                }
+            }
+        }
+
+        if need_traits_in_scope {
+            // FIXME: hygiene is not considered.
+            let mut doc_link_traits_in_scope = std::mem::take(&mut self.r.doc_link_traits_in_scope);
+            doc_link_traits_in_scope
+                .entry(self.parent_scope.module.nearest_parent_mod().expect_local())
+                .or_insert_with(|| {
+                    self.r
+                        .traits_in_scope(None, &self.parent_scope, SyntaxContext::root(), None)
+                        .into_iter()
+                        .filter_map(|tr| {
+                            if !tr.def_id.is_local()
+                                && self.r.tcx.sess.crate_types().contains(&CrateType::ProcMacro)
+                                && matches!(
+                                    self.r.tcx.sess.opts.resolve_doc_links,
+                                    ResolveDocLinks::ExportedMetadata
+                                )
+                            {
+                                // Encoding foreign def ids in proc macro crate metadata will ICE.
+                                return None;
+                            }
+                            Some(tr.def_id)
+                        })
+                        .collect()
+                });
+            self.r.doc_link_traits_in_scope = doc_link_traits_in_scope;
+        }
+    }
 }
 
-struct LifetimeCountVisitor<'a, 'b> {
-    r: &'b mut Resolver<'a>,
+struct LifetimeCountVisitor<'a, 'b, 'tcx> {
+    r: &'b mut Resolver<'a, 'tcx>,
 }
 
 /// Walks the whole crate in DFS order, visiting each item, counting the declared number of
 /// lifetime generic parameters.
-impl<'ast> Visitor<'ast> for LifetimeCountVisitor<'_, '_> {
+impl<'ast> Visitor<'ast> for LifetimeCountVisitor<'_, '_, '_> {
     fn visit_item(&mut self, item: &'ast Item) {
         match &item.kind {
             ItemKind::TyAlias(box TyAlias { ref generics, .. })
@@ -4158,10 +4340,11 @@ impl<'ast> Visitor<'ast> for LifetimeCountVisitor<'_, '_> {
     }
 }
 
-impl<'a> Resolver<'a> {
+impl<'a, 'tcx> Resolver<'a, 'tcx> {
     pub(crate) fn late_resolve_crate(&mut self, krate: &Crate) {
         visit::walk_crate(&mut LifetimeCountVisitor { r: self }, krate);
         let mut late_resolution_visitor = LateResolutionVisitor::new(self);
+        late_resolution_visitor.resolve_doc_links(&krate.attrs, MaybeExported::Ok(CRATE_NODE_ID));
         visit::walk_crate(&mut late_resolution_visitor, krate);
         for (id, span) in late_resolution_visitor.diagnostic_metadata.unused_labels.iter() {
             self.lint_buffer.buffer_lint(lint::builtin::UNUSED_LABELS, *id, *span, "unused label");
diff --git a/compiler/rustc_resolve/src/late/diagnostics.rs b/compiler/rustc_resolve/src/late/diagnostics.rs
index 6d448433e..b8ddc4552 100644
--- a/compiler/rustc_resolve/src/late/diagnostics.rs
+++ b/compiler/rustc_resolve/src/late/diagnostics.rs
@@ -25,9 +25,9 @@ use rustc_middle::ty::DefIdTree;
 use rustc_session::lint;
 use rustc_session::parse::feature_err;
 use rustc_session::Session;
+use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::edition::Edition;
 use rustc_span::hygiene::MacroKind;
-use rustc_span::lev_distance::find_best_match_for_name;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{BytePos, Span};
 
@@ -166,11 +166,11 @@ impl TypoCandidate {
     }
 }
 
-impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
+impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
     fn def_span(&self, def_id: DefId) -> Option<Span> {
         match def_id.krate {
             LOCAL_CRATE => self.r.opt_span(def_id),
-            _ => Some(self.r.cstore().get_span_untracked(def_id, self.r.session)),
+            _ => Some(self.r.cstore().get_span_untracked(def_id, self.r.tcx.sess)),
         }
     }
 
@@ -200,7 +200,8 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                     Res::Def(DefKind::Fn, _) => {
                         // Verify whether this is a fn call or an Fn used as a type.
                         self.r
-                            .session
+                            .tcx
+                            .sess
                             .source_map()
                             .span_to_snippet(span)
                             .map(|snippet| snippet.ends_with(')'))
@@ -227,20 +228,27 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                     && let Some(FnCtxt::Assoc(_)) = fn_kind.ctxt()
                     && let Some(items) = self.diagnostic_metadata.current_impl_items
                     && let Some(item) = items.iter().find(|i| {
-                        if let AssocItemKind::Fn(_) = &i.kind && i.ident.name == item_str.name
+                        if let AssocItemKind::Fn(..) | AssocItemKind::Const(..) = &i.kind
+                            && i.ident.name == item_str.name
                         {
                             debug!(?item_str.name);
                             return true
                         }
                         false
                     })
-                    && let AssocItemKind::Fn(fn_) = &item.kind
                 {
-                    debug!(?fn_);
-                    let self_sugg = if fn_.sig.decl.has_self() { "self." } else { "Self::" };
+                    let self_sugg = match &item.kind {
+                        AssocItemKind::Fn(fn_) if fn_.sig.decl.has_self() => "self.",
+                        _ => "Self::",
+                    };
+
                     Some((
                         item_span.shrink_to_lo(),
-                        "consider using the associated function",
+                        match &item.kind {
+                            AssocItemKind::Fn(..) => "consider using the associated function",
+                            AssocItemKind::Const(..) => "consider using the associated constant",
+                            _ => unreachable!("item kind was filtered above"),
+                        },
                         self_sugg.to_string()
                     ))
                 } else {
@@ -248,7 +256,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                 };
                 (String::new(), "this scope".to_string(), suggestion)
             } else if path.len() == 2 && path[0].ident.name == kw::PathRoot {
-                if self.r.session.edition() > Edition::Edition2015 {
+                if self.r.tcx.sess.edition() > Edition::Edition2015 {
                     // In edition 2018 onwards, the `::foo` syntax may only pull from the extern prelude
                     // which overrides all other expectations of item type
                     expected = "crate";
@@ -311,12 +319,12 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
         span: Span,
         source: PathSource<'_>,
         res: Option<Res>,
-    ) -> (DiagnosticBuilder<'a, ErrorGuaranteed>, Vec<ImportSuggestion>) {
+    ) -> (DiagnosticBuilder<'tcx, ErrorGuaranteed>, Vec<ImportSuggestion>) {
         debug!(?res, ?source);
         let base_error = self.make_base_error(path, span, source, res);
         let code = source.error_code(res.is_some());
         let mut err =
-            self.r.session.struct_span_err_with_code(base_error.span, &base_error.msg, code);
+            self.r.tcx.sess.struct_span_err_with_code(base_error.span, &base_error.msg, code);
 
         self.suggest_swapping_misplaced_self_ty_and_trait(&mut err, source, res, base_error.span);
 
@@ -425,7 +433,8 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                     } else {
                         (
                             self.r
-                                .session
+                                .tcx
+                                .sess
                                 .source_map()
                                 .span_through_char(*fn_span, '(')
                                 .shrink_to_hi(),
@@ -498,7 +507,8 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                 {
                     if self
                         .r
-                        .session
+                        .tcx
+                        .sess
                         .parse_sess
                         .type_ascription_path_suggestions
                         .borrow()
@@ -535,7 +545,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
             }
         }
 
-        // Try Levenshtein algorithm.
+        // Try finding a suitable replacement.
         let typo_sugg =
             self.lookup_typo_candidate(path, source.namespace(), is_expected).to_opt_suggestion();
         if path.len() == 1 && self.self_type_is_available() {
@@ -589,7 +599,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
             if let Some((call_span, args_span)) = self.call_has_self_arg(source) {
                 let mut args_snippet = String::new();
                 if let Some(args_span) = args_span {
-                    if let Ok(snippet) = self.r.session.source_map().span_to_snippet(args_span) {
+                    if let Ok(snippet) = self.r.tcx.sess.source_map().span_to_snippet(args_span) {
                         args_snippet = snippet;
                     }
                 }
@@ -725,7 +735,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
         let ident_span = path.last().map_or(span, |ident| ident.ident.span);
         let typo_sugg = self.lookup_typo_candidate(path, source.namespace(), is_expected);
         let is_in_same_file = &|sp1, sp2| {
-            let source_map = self.r.session.source_map();
+            let source_map = self.r.tcx.sess.source_map();
             let file1 = source_map.span_to_filename(sp1);
             let file2 = source_map.span_to_filename(sp2);
             file1 == file2
@@ -763,7 +773,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                 _ => {}
             }
 
-            // If the trait has a single item (which wasn't matched by Levenshtein), suggest it
+            // If the trait has a single item (which wasn't matched by the algorithm), suggest it
             let suggestion = self.get_single_associated_item(&path, &source, is_expected);
             if !self.r.add_typo_suggestion(err, suggestion, ident_span) {
                 fallback = !self.let_binding_suggestion(err, ident_span);
@@ -868,7 +878,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
         );
         let is_assoc_fn = self.self_type_is_available();
         if let Some((fn_kind, span)) = &self.diagnostic_metadata.current_function {
-            // The current function has a `self' parameter, but we were unable to resolve
+            // The current function has a `self` parameter, but we were unable to resolve
             // a reference to `self`. This can only happen if the `self` identifier we
             // are resolving came from a different hygiene context.
             if fn_kind.decl().inputs.get(0).map_or(false, |p| p.is_self()) {
@@ -888,7 +898,8 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                                 .map_or(*span, |ident| span.with_lo(ident.span.hi()));
                             (
                                 self.r
-                                    .session
+                                    .tcx
+                                    .sess
                                     .source_map()
                                     .span_through_char(span, '(')
                                     .shrink_to_hi(),
@@ -942,9 +953,9 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
             && let PathSource::Trait(_) = source
             && let Some(Res::Def(DefKind::Struct | DefKind::Enum | DefKind::Union, _)) = res
             && let Ok(self_ty_str) =
-                self.r.session.source_map().span_to_snippet(self_ty.span)
+                self.r.tcx.sess.source_map().span_to_snippet(self_ty.span)
             && let Ok(trait_ref_str) =
-                self.r.session.source_map().span_to_snippet(trait_ref.path.span)
+                self.r.tcx.sess.source_map().span_to_snippet(trait_ref.path.span)
         {
                 err.multipart_suggestion(
                     "`impl` items mention the trait being implemented first and the type it is being implemented for second",
@@ -1088,7 +1099,8 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                             format!(
                                 "{}: {}<{} = {}>",
                                 self.r
-                                    .session
+                                    .tcx
+                                    .sess
                                     .source_map()
                                     .span_to_snippet(ty.span) // Account for `<&'a T as Foo>::Bar`.
                                     .unwrap_or_else(|_| constrain_ident.to_string()),
@@ -1157,7 +1169,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
         // parser issue where a struct literal is being used on an expression
         // where a brace being opened means a block is being started. Look
         // ahead for the next text to see if `span` is followed by a `{`.
-        let sm = self.r.session.source_map();
+        let sm = self.r.tcx.sess.source_map();
         let sp = sm.span_look_ahead(span, None, Some(50));
         let followed_by_brace = matches!(sm.span_to_snippet(sp), Ok(ref snippet) if snippet == "{");
         // In case this could be a struct literal that needs to be surrounded
@@ -1205,7 +1217,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                 true
             } else if kind == DefKind::Struct
             && let Some(lhs_source_span) = lhs_span.find_ancestor_inside(expr.span)
-            && let Ok(snippet) = self.r.session.source_map().span_to_snippet(lhs_source_span)
+            && let Ok(snippet) = self.r.tcx.sess.source_map().span_to_snippet(lhs_source_span)
             {
                 // The LHS is a type that originates from a macro call.
                 // We have to add angle brackets around it.
@@ -1336,7 +1348,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                     "!",
                     Applicability::MaybeIncorrect,
                 );
-                if path_str == "try" && span.rust_2015() {
+                if path_str == "try" && span.is_rust_2015() {
                     err.note("if you want the `try` keyword, you need Rust 2018 or later");
                 }
             }
@@ -1345,11 +1357,11 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
             }
             (Res::Def(DefKind::TyAlias, def_id), PathSource::Trait(_)) => {
                 err.span_label(span, "type aliases cannot be used as traits");
-                if self.r.session.is_nightly_build() {
+                if self.r.tcx.sess.is_nightly_build() {
                     let msg = "you might have meant to use `#![feature(trait_alias)]` instead of a \
                                `type` alias";
                     if let Some(span) = self.def_span(def_id) {
-                        if let Ok(snip) = self.r.session.source_map().span_to_snippet(span) {
+                        if let Ok(snip) = self.r.tcx.sess.source_map().span_to_snippet(span) {
                             // The span contains a type alias so we should be able to
                             // replace `type` with `trait`.
                             let snip = snip.replacen("type", "trait", 1);
@@ -1380,7 +1392,8 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                     .last()
                     .map(|sp| {
                         self.r
-                            .session
+                            .tcx
+                            .sess
                             .parse_sess
                             .type_ascription_path_suggestions
                             .borrow()
@@ -1687,17 +1700,14 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                             let extern_prelude = self.r.extern_prelude.clone();
                             names.extend(extern_prelude.iter().flat_map(|(ident, _)| {
                                 self.r
-                                    .crate_loader()
-                                    .maybe_process_path_extern(ident.name)
+                                    .crate_loader(|c| c.maybe_process_path_extern(ident.name))
                                     .and_then(|crate_id| {
                                         let crate_mod =
                                             Res::Def(DefKind::Mod, crate_id.as_def_id());
 
-                                        if filter_fn(crate_mod) {
-                                            Some(TypoSuggestion::typo_from_ident(*ident, crate_mod))
-                                        } else {
-                                            None
-                                        }
+                                        filter_fn(crate_mod).then(|| {
+                                            TypoSuggestion::typo_from_ident(*ident, crate_mod)
+                                        })
                                     })
                             }));
 
@@ -1769,12 +1779,12 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
 
     /// Only used in a specific case of type ascription suggestions
     fn get_colon_suggestion_span(&self, start: Span) -> Span {
-        let sm = self.r.session.source_map();
+        let sm = self.r.tcx.sess.source_map();
         start.to(sm.next_point(start))
     }
 
     fn type_ascription_suggestion(&self, err: &mut Diagnostic, base_span: Span) -> bool {
-        let sm = self.r.session.source_map();
+        let sm = self.r.tcx.sess.source_map();
         let base_snippet = sm.span_to_snippet(base_span);
         if let Some(&sp) = self.diagnostic_metadata.current_type_ascription.last() {
             if let Ok(snippet) = sm.span_to_snippet(sp) {
@@ -1804,7 +1814,8 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                             show_label = false;
                             if !self
                                 .r
-                                .session
+                                .tcx
+                                .sess
                                 .parse_sess
                                 .type_ascription_path_suggestions
                                 .borrow_mut()
@@ -2237,19 +2248,23 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                 }
                 None => {
                     debug!(?param.ident, ?param.ident.span);
-
                     let deletion_span = deletion_span();
-                    self.r.lint_buffer.buffer_lint_with_diagnostic(
-                        lint::builtin::UNUSED_LIFETIMES,
-                        param.id,
-                        param.ident.span,
-                        &format!("lifetime parameter `{}` never used", param.ident),
-                        lint::BuiltinLintDiagnostics::SingleUseLifetime {
-                            param_span: param.ident.span,
-                            use_span: None,
-                            deletion_span,
-                        },
-                    );
+                    // the give lifetime originates from expanded code so we won't be able to remove it #104432
+                    let lifetime_only_in_expanded_code =
+                        deletion_span.map(|sp| sp.in_derive_expansion()).unwrap_or(true);
+                    if !lifetime_only_in_expanded_code {
+                        self.r.lint_buffer.buffer_lint_with_diagnostic(
+                            lint::builtin::UNUSED_LIFETIMES,
+                            param.id,
+                            param.ident.span,
+                            &format!("lifetime parameter `{}` never used", param.ident),
+                            lint::BuiltinLintDiagnostics::SingleUseLifetime {
+                                param_span: param.ident.span,
+                                use_span: None,
+                                deletion_span,
+                            },
+                        );
+                    }
                 }
             }
         }
@@ -2263,7 +2278,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
         debug_assert_ne!(lifetime_ref.ident.name, kw::UnderscoreLifetime);
         let mut err = if let Some(outer) = outer_lifetime_ref {
             let mut err = struct_span_err!(
-                self.r.session,
+                self.r.tcx.sess,
                 lifetime_ref.ident.span,
                 E0401,
                 "can't use generic parameters from outer item",
@@ -2273,7 +2288,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
             err
         } else {
             let mut err = struct_span_err!(
-                self.r.session,
+                self.r.tcx.sess,
                 lifetime_ref.ident.span,
                 E0261,
                 "use of undeclared lifetime name `{}`",
@@ -2331,8 +2346,13 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
                         );
                         (span, sugg)
                     } else {
-                        let span =
-                            self.r.session.source_map().span_through_char(span, '<').shrink_to_hi();
+                        let span = self
+                            .r
+                            .tcx
+                            .sess
+                            .source_map()
+                            .span_through_char(span, '<')
+                            .shrink_to_hi();
                         let sugg = format!("{}, ", name.unwrap_or("'a"));
                         (span, sugg)
                     };
@@ -2366,7 +2386,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
 
     pub(crate) fn emit_non_static_lt_in_const_generic_error(&self, lifetime_ref: &ast::Lifetime) {
         struct_span_err!(
-            self.r.session,
+            self.r.tcx.sess,
             lifetime_ref.ident.span,
             E0771,
             "use of non-static lifetime `{}` in const generic",
@@ -2386,10 +2406,10 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
         &self,
         lifetime_ref: &ast::Lifetime,
     ) {
-        let feature_active = self.r.session.features_untracked().generic_const_exprs;
+        let feature_active = self.r.tcx.sess.features_untracked().generic_const_exprs;
         if !feature_active {
             feature_err(
-                &self.r.session.parse_sess,
+                &self.r.tcx.sess.parse_sess,
                 sym::generic_const_exprs,
                 lifetime_ref.ident.span,
                 "a non-static lifetime is not allowed in a `const`",
@@ -2407,7 +2427,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
         let spans: Vec<_> = lifetime_refs.iter().map(|lt| lt.span).collect();
 
         let mut err = struct_span_err!(
-            self.r.session,
+            self.r.tcx.sess,
             spans,
             E0106,
             "missing lifetime specifier{}",
@@ -2615,7 +2635,7 @@ impl<'a: 'ast, 'ast> LateResolutionVisitor<'a, '_, 'ast> {
 }
 
 /// Report lifetime/lifetime shadowing as an error.
-pub fn signal_lifetime_shadowing(sess: &Session, orig: Ident, shadower: Ident) {
+pub(super) fn signal_lifetime_shadowing(sess: &Session, orig: Ident, shadower: Ident) {
     let mut err = struct_span_err!(
         sess,
         shadower.span,
@@ -2630,7 +2650,7 @@ pub fn signal_lifetime_shadowing(sess: &Session, orig: Ident, shadower: Ident) {
 
 /// Shadowing involving a label is only a warning for historical reasons.
 //FIXME: make this a proper lint.
-pub fn signal_label_shadowing(sess: &Session, orig: Span, shadower: Ident) {
+pub(super) fn signal_label_shadowing(sess: &Session, orig: Span, shadower: Ident) {
     let name = shadower.name;
     let shadower = shadower.span;
     let mut err = sess.struct_span_warn(
diff --git a/compiler/rustc_resolve/src/lib.rs b/compiler/rustc_resolve/src/lib.rs
index 1b181b714..1fdfb1a53 100644
--- a/compiler/rustc_resolve/src/lib.rs
+++ b/compiler/rustc_resolve/src/lib.rs
@@ -21,34 +21,34 @@
 #[macro_use]
 extern crate tracing;
 
-pub use rustc_hir::def::{Namespace, PerNS};
-
 use rustc_arena::{DroplessArena, TypedArena};
 use rustc_ast::node_id::NodeMap;
 use rustc_ast::{self as ast, NodeId, CRATE_NODE_ID};
 use rustc_ast::{AngleBracketedArg, Crate, Expr, ExprKind, GenericArg, GenericArgs, LitKind, Path};
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap, FxIndexSet};
 use rustc_data_structures::intern::Interned;
-use rustc_data_structures::sync::{Lrc, RwLock};
-use rustc_errors::{Applicability, DiagnosticBuilder, ErrorGuaranteed};
+use rustc_data_structures::sync::{Lrc, MappedReadGuard};
+use rustc_errors::{
+    Applicability, DiagnosticBuilder, DiagnosticMessage, ErrorGuaranteed, SubdiagnosticMessage,
+};
 use rustc_expand::base::{DeriveResolutions, SyntaxExtension, SyntaxExtensionKind};
-use rustc_hir::def::Namespace::*;
-use rustc_hir::def::{self, CtorOf, DefKind, LifetimeRes, PartialRes};
+use rustc_hir::def::Namespace::{self, *};
+use rustc_hir::def::{self, CtorOf, DefKind, DocLinkResMap, LifetimeRes, PartialRes, PerNS};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId};
 use rustc_hir::def_id::{CRATE_DEF_ID, LOCAL_CRATE};
-use rustc_hir::definitions::{DefPathData, Definitions};
+use rustc_hir::definitions::DefPathData;
 use rustc_hir::TraitCandidate;
 use rustc_index::vec::IndexVec;
+use rustc_macros::fluent_messages;
 use rustc_metadata::creader::{CStore, CrateLoader};
 use rustc_middle::metadata::ModChild;
 use rustc_middle::middle::privacy::EffectiveVisibilities;
 use rustc_middle::span_bug;
-use rustc_middle::ty::{self, DefIdTree, MainDefinition, RegisteredTools};
+use rustc_middle::ty::{self, DefIdTree, MainDefinition, RegisteredTools, TyCtxt};
 use rustc_middle::ty::{ResolverGlobalCtxt, ResolverOutputs};
 use rustc_query_system::ich::StableHashingContext;
-use rustc_session::cstore::{CrateStore, MetadataLoaderDyn, Untracked};
+use rustc_session::cstore::CrateStore;
 use rustc_session::lint::LintBuffer;
-use rustc_session::Session;
 use rustc_span::hygiene::{ExpnId, LocalExpnId, MacroKind, SyntaxContext, Transparency};
 use rustc_span::source_map::Spanned;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
@@ -60,7 +60,7 @@ use std::collections::BTreeSet;
 use std::{fmt, ptr};
 
 use diagnostics::{ImportSuggestion, LabelSuggestion, Suggestion};
-use imports::{Import, ImportKind, ImportResolver, NameResolution};
+use imports::{Import, ImportKind, NameResolution};
 use late::{HasGenericParams, PathSource, PatternSource};
 use macros::{MacroRulesBinding, MacroRulesScope, MacroRulesScopeRef};
 
@@ -78,6 +78,9 @@ mod ident;
 mod imports;
 mod late;
 mod macros;
+pub mod rustdoc;
+
+fluent_messages! { "../locales/en-US.ftl" }
 
 enum Weak {
     Yes,
@@ -85,7 +88,7 @@ enum Weak {
 }
 
 #[derive(Copy, Clone, PartialEq, Debug)]
-pub enum Determinacy {
+enum Determinacy {
     Determined,
     Undetermined,
 }
@@ -138,17 +141,17 @@ enum ScopeSet<'a> {
 /// This struct is currently used only for early resolution (imports and macros),
 /// but not for late resolution yet.
 #[derive(Clone, Copy, Debug)]
-pub struct ParentScope<'a> {
-    pub module: Module<'a>,
+struct ParentScope<'a> {
+    module: Module<'a>,
     expansion: LocalExpnId,
-    pub macro_rules: MacroRulesScopeRef<'a>,
+    macro_rules: MacroRulesScopeRef<'a>,
     derives: &'a [ast::Path],
 }
 
 impl<'a> ParentScope<'a> {
     /// Creates a parent scope with the passed argument used as the module scope component,
     /// and other scope components set to default empty values.
-    pub fn module(module: Module<'a>, resolver: &Resolver<'a>) -> ParentScope<'a> {
+    fn module(module: Module<'a>, resolver: &Resolver<'a, '_>) -> ParentScope<'a> {
         ParentScope {
             module,
             expansion: LocalExpnId::ROOT,
@@ -256,7 +259,7 @@ enum VisResolutionError<'a> {
 /// A minimal representation of a path segment. We use this in resolve because we synthesize 'path
 /// segments' which don't have the rest of an AST or HIR `PathSegment`.
 #[derive(Clone, Copy, Debug)]
-pub struct Segment {
+struct Segment {
     ident: Ident,
     id: Option<NodeId>,
     /// Signals whether this `PathSegment` has generic arguments. Used to avoid providing
@@ -379,7 +382,7 @@ impl ModuleOrUniformRoot<'_> {
     }
 }
 
-#[derive(Clone, Debug)]
+#[derive(Debug)]
 enum PathResult<'a> {
     Module(ModuleOrUniformRoot<'a>),
     NonModule(PartialRes),
@@ -434,7 +437,7 @@ enum ModuleKind {
 
 impl ModuleKind {
     /// Get name of the module.
-    pub fn name(&self) -> Option<Symbol> {
+    fn name(&self) -> Option<Symbol> {
         match self {
             ModuleKind::Block => None,
             ModuleKind::Def(.., name) => Some(*name),
@@ -470,7 +473,7 @@ type Resolutions<'a> = RefCell<FxIndexMap<BindingKey, &'a RefCell<NameResolution
 /// * curly-braced block with statements
 ///
 /// You can use [`ModuleData::kind`] to determine the kind of module this is.
-pub struct ModuleData<'a> {
+struct ModuleData<'a> {
     /// The direct parent module (it may not be a `mod`, however).
     parent: Option<Module<'a>>,
     /// What kind of module this is, because this may not be a `mod`.
@@ -529,9 +532,9 @@ impl<'a> ModuleData<'a> {
         }
     }
 
-    fn for_each_child<R, F>(&'a self, resolver: &mut R, mut f: F)
+    fn for_each_child<'tcx, R, F>(&'a self, resolver: &mut R, mut f: F)
     where
-        R: AsMut<Resolver<'a>>,
+        R: AsMut<Resolver<'a, 'tcx>>,
         F: FnMut(&mut R, Ident, Namespace, &'a NameBinding<'a>),
     {
         for (key, name_resolution) in resolver.as_mut().resolutions(self).borrow().iter() {
@@ -542,9 +545,9 @@ impl<'a> ModuleData<'a> {
     }
 
     /// This modifies `self` in place. The traits will be stored in `self.traits`.
-    fn ensure_traits<R>(&'a self, resolver: &mut R)
+    fn ensure_traits<'tcx, R>(&'a self, resolver: &mut R)
     where
-        R: AsMut<Resolver<'a>>,
+        R: AsMut<Resolver<'a, 'tcx>>,
     {
         let mut traits = self.traits.borrow_mut();
         if traits.is_none() {
@@ -569,7 +572,7 @@ impl<'a> ModuleData<'a> {
     }
 
     // Public for rustdoc.
-    pub fn def_id(&self) -> DefId {
+    fn def_id(&self) -> DefId {
         self.opt_def_id().expect("`ModuleData::def_id` is called on a block module")
     }
 
@@ -627,7 +630,7 @@ impl<'a> fmt::Debug for ModuleData<'a> {
 
 /// Records a possibly-private value, type, or module definition.
 #[derive(Clone, Debug)]
-pub struct NameBinding<'a> {
+struct NameBinding<'a> {
     kind: NameBindingKind<'a>,
     ambiguity: Option<(&'a NameBinding<'a>, AmbiguityKind)>,
     expansion: LocalExpnId,
@@ -635,7 +638,7 @@ pub struct NameBinding<'a> {
     vis: ty::Visibility<DefId>,
 }
 
-pub trait ToNameBinding<'a> {
+trait ToNameBinding<'a> {
     fn to_name_binding(self, arenas: &'a ResolverArenas<'a>) -> &'a NameBinding<'a>;
 }
 
@@ -839,9 +842,9 @@ impl<'a> NameBinding<'a> {
 }
 
 #[derive(Default, Clone)]
-pub struct ExternPreludeEntry<'a> {
+struct ExternPreludeEntry<'a> {
     extern_crate_item: Option<&'a NameBinding<'a>>,
-    pub introduced_by_item: bool,
+    introduced_by_item: bool,
 }
 
 /// Used for better errors for E0773
@@ -865,8 +868,8 @@ struct MacroData {
 /// The main resolver class.
 ///
 /// This is the visitor that walks the whole crate.
-pub struct Resolver<'a> {
-    session: &'a Session,
+pub struct Resolver<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
 
     /// Item with a given `LocalDefId` was defined during macro expansion with ID `ExpnId`.
     expn_that_defined: FxHashMap<LocalDefId, ExpnId>,
@@ -943,23 +946,19 @@ pub struct Resolver<'a> {
     has_pub_restricted: bool,
     used_imports: FxHashSet<NodeId>,
     maybe_unused_trait_imports: FxIndexSet<LocalDefId>,
-    maybe_unused_extern_crates: Vec<(LocalDefId, Span)>,
 
     /// Privacy errors are delayed until the end in order to deduplicate them.
     privacy_errors: Vec<PrivacyError<'a>>,
     /// Ambiguity errors are delayed for deduplication.
     ambiguity_errors: Vec<AmbiguityError<'a>>,
     /// `use` injections are delayed for better placement and deduplication.
-    use_injections: Vec<UseError<'a>>,
+    use_injections: Vec<UseError<'tcx>>,
     /// Crate-local macro expanded `macro_export` referred to by a module-relative path.
     macro_expanded_macro_export_errors: BTreeSet<(Span, Span)>,
 
     arenas: &'a ResolverArenas<'a>,
     dummy_binding: &'a NameBinding<'a>,
 
-    local_crate_name: Symbol,
-    metadata_loader: Box<MetadataLoaderDyn>,
-    untracked: Untracked,
     used_extern_options: FxHashSet<Symbol>,
     macro_names: FxHashSet<Ident>,
     builtin_macros: FxHashMap<Symbol, BuiltinMacroState>,
@@ -1046,6 +1045,9 @@ pub struct Resolver<'a> {
     lifetime_elision_allowed: FxHashSet<NodeId>,
 
     effective_visibilities: EffectiveVisibilities,
+    doc_link_resolutions: FxHashMap<LocalDefId, DocLinkResMap>,
+    doc_link_traits_in_scope: FxHashMap<LocalDefId, Vec<DefId>>,
+    all_macro_rules: FxHashMap<Symbol, Res>,
 }
 
 /// Nothing really interesting here; it just provides memory for the rest of the crate.
@@ -1109,42 +1111,25 @@ impl<'a> ResolverArenas<'a> {
     }
 }
 
-impl<'a> AsMut<Resolver<'a>> for Resolver<'a> {
-    fn as_mut(&mut self) -> &mut Resolver<'a> {
+impl<'a, 'tcx> AsMut<Resolver<'a, 'tcx>> for Resolver<'a, 'tcx> {
+    fn as_mut(&mut self) -> &mut Resolver<'a, 'tcx> {
         self
     }
 }
 
-/// A minimal subset of resolver that can implemenent `DefIdTree`, sometimes
-/// required to satisfy borrow checker by avoiding borrowing the whole resolver.
-#[derive(Clone, Copy)]
-struct ResolverTree<'a>(&'a Untracked);
-
-impl DefIdTree for ResolverTree<'_> {
-    #[inline]
-    fn opt_parent(self, id: DefId) -> Option<DefId> {
-        let ResolverTree(Untracked { definitions, cstore, .. }) = self;
-        match id.as_local() {
-            Some(id) => definitions.read().def_key(id).parent,
-            None => cstore.as_any().downcast_ref::<CStore>().unwrap().def_key(id).parent,
-        }
-        .map(|index| DefId { index, ..id })
-    }
-}
-
-impl<'a, 'b> DefIdTree for &'a Resolver<'b> {
+impl<'a, 'b, 'tcx> DefIdTree for &'a Resolver<'b, 'tcx> {
     #[inline]
     fn opt_parent(self, id: DefId) -> Option<DefId> {
-        ResolverTree(&self.untracked).opt_parent(id)
+        self.tcx.opt_parent(id)
     }
 }
 
-impl<'a> Resolver<'a> {
+impl<'tcx> Resolver<'_, 'tcx> {
     fn opt_local_def_id(&self, node: NodeId) -> Option<LocalDefId> {
         self.node_id_to_def_id.get(&node).copied()
     }
 
-    pub fn local_def_id(&self, node: NodeId) -> LocalDefId {
+    fn local_def_id(&self, node: NodeId) -> LocalDefId {
         self.opt_local_def_id(node).unwrap_or_else(|| panic!("no entry for node id: `{:?}`", node))
     }
 
@@ -1162,10 +1147,11 @@ impl<'a> Resolver<'a> {
             "adding a def'n for node-id {:?} and data {:?} but a previous def'n exists: {:?}",
             node_id,
             data,
-            self.untracked.definitions.read().def_key(self.node_id_to_def_id[&node_id]),
+            self.tcx.definitions_untracked().def_key(self.node_id_to_def_id[&node_id]),
         );
 
-        let def_id = self.untracked.definitions.write().create_def(parent, data);
+        // FIXME: remove `def_span` body, pass in the right spans here and call `tcx.at().create_def()`
+        let def_id = self.tcx.untracked().definitions.write().create_def(parent, data);
 
         // Create the definition.
         if expn_id != ExpnId::root() {
@@ -1174,7 +1160,7 @@ impl<'a> Resolver<'a> {
 
         // A relative span's parent must be an absolute span.
         debug_assert_eq!(span.data_untracked().parent, None);
-        let _id = self.untracked.source_span.push(span);
+        let _id = self.tcx.untracked().source_span.push(span);
         debug_assert_eq!(_id, def_id);
 
         // Some things for which we allocate `LocalDefId`s don't correspond to
@@ -1193,23 +1179,21 @@ impl<'a> Resolver<'a> {
         if let Some(def_id) = def_id.as_local() {
             self.item_generics_num_lifetimes[&def_id]
         } else {
-            self.cstore().item_generics_num_lifetimes(def_id, self.session)
+            self.cstore().item_generics_num_lifetimes(def_id, self.tcx.sess)
         }
     }
 
-    pub fn sess(&self) -> &'a Session {
-        self.session
+    pub fn tcx(&self) -> TyCtxt<'tcx> {
+        self.tcx
     }
 }
 
-impl<'a> Resolver<'a> {
+impl<'a, 'tcx> Resolver<'a, 'tcx> {
     pub fn new(
-        session: &'a Session,
+        tcx: TyCtxt<'tcx>,
         krate: &Crate,
-        crate_name: Symbol,
-        metadata_loader: Box<MetadataLoaderDyn>,
         arenas: &'a ResolverArenas<'a>,
-    ) -> Resolver<'a> {
+    ) -> Resolver<'a, 'tcx> {
         let root_def_id = CRATE_DEF_ID.to_def_id();
         let mut module_map = FxHashMap::default();
         let graph_root = arenas.new_module(
@@ -1217,7 +1201,7 @@ impl<'a> Resolver<'a> {
             ModuleKind::Def(DefKind::Mod, root_def_id, kw::Empty),
             ExpnId::root(),
             krate.spans.inner_span,
-            session.contains_name(&krate.attrs, sym::no_implicit_prelude),
+            tcx.sess.contains_name(&krate.attrs, sym::no_implicit_prelude),
             &mut module_map,
         );
         let empty_module = arenas.new_module(
@@ -1229,8 +1213,6 @@ impl<'a> Resolver<'a> {
             &mut FxHashMap::default(),
         );
 
-        let definitions = Definitions::new(session.local_stable_crate_id());
-
         let mut visibilities = FxHashMap::default();
         visibilities.insert(CRATE_DEF_ID, ty::Visibility::Public);
 
@@ -1242,11 +1224,8 @@ impl<'a> Resolver<'a> {
         let mut invocation_parents = FxHashMap::default();
         invocation_parents.insert(LocalExpnId::ROOT, (CRATE_DEF_ID, ImplTraitContext::Existential));
 
-        let mut source_span = IndexVec::default();
-        let _id = source_span.push(krate.spans.inner_span);
-        debug_assert_eq!(_id, CRATE_DEF_ID);
-
-        let mut extern_prelude: FxHashMap<Ident, ExternPreludeEntry<'_>> = session
+        let mut extern_prelude: FxHashMap<Ident, ExternPreludeEntry<'_>> = tcx
+            .sess
             .opts
             .externs
             .iter()
@@ -1254,19 +1233,19 @@ impl<'a> Resolver<'a> {
             .map(|(name, _)| (Ident::from_str(name), Default::default()))
             .collect();
 
-        if !session.contains_name(&krate.attrs, sym::no_core) {
+        if !tcx.sess.contains_name(&krate.attrs, sym::no_core) {
             extern_prelude.insert(Ident::with_dummy_span(sym::core), Default::default());
-            if !session.contains_name(&krate.attrs, sym::no_std) {
+            if !tcx.sess.contains_name(&krate.attrs, sym::no_std) {
                 extern_prelude.insert(Ident::with_dummy_span(sym::std), Default::default());
             }
         }
 
-        let registered_tools = macros::registered_tools(session, &krate.attrs);
+        let registered_tools = macros::registered_tools(tcx.sess, &krate.attrs);
 
-        let features = session.features_untracked();
+        let features = tcx.sess.features_untracked();
 
         let mut resolver = Resolver {
-            session,
+            tcx,
 
             expn_that_defined: Default::default(),
 
@@ -1304,7 +1283,6 @@ impl<'a> Resolver<'a> {
             has_pub_restricted: false,
             used_imports: FxHashSet::default(),
             maybe_unused_trait_imports: Default::default(),
-            maybe_unused_extern_crates: Vec::new(),
 
             privacy_errors: Vec::new(),
             ambiguity_errors: Vec::new(),
@@ -1320,23 +1298,16 @@ impl<'a> Resolver<'a> {
                 vis: ty::Visibility::Public,
             }),
 
-            metadata_loader,
-            local_crate_name: crate_name,
             used_extern_options: Default::default(),
-            untracked: Untracked {
-                cstore: Box::new(CStore::new(session)),
-                source_span,
-                definitions: RwLock::new(definitions),
-            },
             macro_names: FxHashSet::default(),
             builtin_macros: Default::default(),
             builtin_macro_kinds: Default::default(),
             registered_tools,
             macro_use_prelude: FxHashMap::default(),
             macro_map: FxHashMap::default(),
-            dummy_ext_bang: Lrc::new(SyntaxExtension::dummy_bang(session.edition())),
-            dummy_ext_derive: Lrc::new(SyntaxExtension::dummy_derive(session.edition())),
-            non_macro_attr: Lrc::new(SyntaxExtension::non_macro_attr(session.edition())),
+            dummy_ext_bang: Lrc::new(SyntaxExtension::dummy_bang(tcx.sess.edition())),
+            dummy_ext_derive: Lrc::new(SyntaxExtension::dummy_derive(tcx.sess.edition())),
+            non_macro_attr: Lrc::new(SyntaxExtension::non_macro_attr(tcx.sess.edition())),
             invocation_parent_scopes: Default::default(),
             output_macro_rules_scopes: Default::default(),
             macro_rules_scopes: Default::default(),
@@ -1374,6 +1345,9 @@ impl<'a> Resolver<'a> {
             confused_type_with_std_module: Default::default(),
             lifetime_elision_allowed: Default::default(),
             effective_visibilities: Default::default(),
+            doc_link_resolutions: Default::default(),
+            doc_link_traits_in_scope: Default::default(),
+            all_macro_rules: Default::default(),
         };
 
         let root_parent_scope = ParentScope::module(graph_root, &resolver);
@@ -1394,14 +1368,14 @@ impl<'a> Resolver<'a> {
         self.arenas.new_module(parent, kind, expn_id, span, no_implicit_prelude, module_map)
     }
 
-    pub fn next_node_id(&mut self) -> NodeId {
+    fn next_node_id(&mut self) -> NodeId {
         let start = self.next_node_id;
         let next = start.as_u32().checked_add(1).expect("input too large; ran out of NodeIds");
         self.next_node_id = ast::NodeId::from_u32(next);
         start
     }
 
-    pub fn next_node_ids(&mut self, count: usize) -> std::ops::Range<NodeId> {
+    fn next_node_ids(&mut self, count: usize) -> std::ops::Range<NodeId> {
         let start = self.next_node_id;
         let end = start.as_usize().checked_add(count).expect("input too large; ran out of NodeIds");
         self.next_node_id = ast::NodeId::from_usize(end);
@@ -1424,12 +1398,10 @@ impl<'a> Resolver<'a> {
         let extern_crate_map = self.extern_crate_map;
         let reexport_map = self.reexport_map;
         let maybe_unused_trait_imports = self.maybe_unused_trait_imports;
-        let maybe_unused_extern_crates = self.maybe_unused_extern_crates;
         let glob_map = self.glob_map;
         let main_def = self.main_def;
         let confused_type_with_std_module = self.confused_type_with_std_module;
         let effective_visibilities = self.effective_visibilities;
-        let untracked = self.untracked;
         let global_ctxt = ResolverGlobalCtxt {
             expn_that_defined,
             visibilities,
@@ -1439,17 +1411,14 @@ impl<'a> Resolver<'a> {
             reexport_map,
             glob_map,
             maybe_unused_trait_imports,
-            maybe_unused_extern_crates,
-            extern_prelude: self
-                .extern_prelude
-                .iter()
-                .map(|(ident, entry)| (ident.name, entry.introduced_by_item))
-                .collect(),
             main_def,
             trait_impls: self.trait_impls,
             proc_macros,
             confused_type_with_std_module,
             registered_tools: self.registered_tools,
+            doc_link_resolutions: self.doc_link_resolutions,
+            doc_link_traits_in_scope: self.doc_link_traits_in_scope,
+            all_macro_rules: self.all_macro_rules,
         };
         let ast_lowering = ty::ResolverAstLowering {
             legacy_const_generic_args: self.legacy_const_generic_args,
@@ -1465,70 +1434,21 @@ impl<'a> Resolver<'a> {
             builtin_macro_kinds: self.builtin_macro_kinds,
             lifetime_elision_allowed: self.lifetime_elision_allowed,
         };
-        ResolverOutputs { global_ctxt, ast_lowering, untracked }
-    }
-
-    pub fn clone_outputs(&self) -> ResolverOutputs {
-        let proc_macros = self.proc_macros.iter().map(|id| self.local_def_id(*id)).collect();
-        let definitions = self.untracked.definitions.clone();
-        let cstore = Box::new(self.cstore().clone());
-        let untracked =
-            Untracked { cstore, source_span: self.untracked.source_span.clone(), definitions };
-        let global_ctxt = ResolverGlobalCtxt {
-            expn_that_defined: self.expn_that_defined.clone(),
-            visibilities: self.visibilities.clone(),
-            has_pub_restricted: self.has_pub_restricted,
-            extern_crate_map: self.extern_crate_map.clone(),
-            reexport_map: self.reexport_map.clone(),
-            glob_map: self.glob_map.clone(),
-            maybe_unused_trait_imports: self.maybe_unused_trait_imports.clone(),
-            maybe_unused_extern_crates: self.maybe_unused_extern_crates.clone(),
-            extern_prelude: self
-                .extern_prelude
-                .iter()
-                .map(|(ident, entry)| (ident.name, entry.introduced_by_item))
-                .collect(),
-            main_def: self.main_def,
-            trait_impls: self.trait_impls.clone(),
-            proc_macros,
-            confused_type_with_std_module: self.confused_type_with_std_module.clone(),
-            registered_tools: self.registered_tools.clone(),
-            effective_visibilities: self.effective_visibilities.clone(),
-        };
-        let ast_lowering = ty::ResolverAstLowering {
-            legacy_const_generic_args: self.legacy_const_generic_args.clone(),
-            partial_res_map: self.partial_res_map.clone(),
-            import_res_map: self.import_res_map.clone(),
-            label_res_map: self.label_res_map.clone(),
-            lifetimes_res_map: self.lifetimes_res_map.clone(),
-            extra_lifetime_params_map: self.extra_lifetime_params_map.clone(),
-            next_node_id: self.next_node_id,
-            node_id_to_def_id: self.node_id_to_def_id.clone(),
-            def_id_to_node_id: self.def_id_to_node_id.clone(),
-            trait_map: self.trait_map.clone(),
-            builtin_macro_kinds: self.builtin_macro_kinds.clone(),
-            lifetime_elision_allowed: self.lifetime_elision_allowed.clone(),
-        };
-        ResolverOutputs { global_ctxt, ast_lowering, untracked }
+        ResolverOutputs { global_ctxt, ast_lowering }
     }
 
     fn create_stable_hashing_context(&self) -> StableHashingContext<'_> {
-        StableHashingContext::new(self.session, &self.untracked)
+        StableHashingContext::new(self.tcx.sess, self.tcx.untracked())
     }
 
-    pub fn crate_loader(&mut self) -> CrateLoader<'_> {
-        CrateLoader::new(
-            &self.session,
-            &*self.metadata_loader,
-            self.local_crate_name,
-            &mut *self.untracked.cstore.untracked_as_any().downcast_mut().unwrap(),
-            self.untracked.definitions.read(),
-            &mut self.used_extern_options,
-        )
+    fn crate_loader<T>(&mut self, f: impl FnOnce(&mut CrateLoader<'_, '_>) -> T) -> T {
+        let mut cstore = self.tcx.untracked().cstore.write();
+        let cstore = cstore.untracked_as_any().downcast_mut().unwrap();
+        f(&mut CrateLoader::new(self.tcx, &mut *cstore, &mut self.used_extern_options))
     }
 
-    pub fn cstore(&self) -> &CStore {
-        self.untracked.cstore.as_any().downcast_ref().unwrap()
+    fn cstore(&self) -> MappedReadGuard<'_, CStore> {
+        CStore::from_tcx(self.tcx)
     }
 
     fn dummy_ext(&self, macro_kind: MacroKind) -> Lrc<SyntaxExtension> {
@@ -1561,21 +1481,26 @@ impl<'a> Resolver<'a> {
 
     /// Entry point to crate resolution.
     pub fn resolve_crate(&mut self, krate: &Crate) {
-        self.session.time("resolve_crate", || {
-            self.session.time("finalize_imports", || ImportResolver { r: self }.finalize_imports());
-            self.session.time("compute_effective_visibilities", || {
+        self.tcx.sess.time("resolve_crate", || {
+            self.tcx.sess.time("finalize_imports", || self.finalize_imports());
+            self.tcx.sess.time("compute_effective_visibilities", || {
                 EffectiveVisibilitiesVisitor::compute_effective_visibilities(self, krate)
             });
-            self.session.time("finalize_macro_resolutions", || self.finalize_macro_resolutions());
-            self.session.time("late_resolve_crate", || self.late_resolve_crate(krate));
-            self.session.time("resolve_main", || self.resolve_main());
-            self.session.time("resolve_check_unused", || self.check_unused(krate));
-            self.session.time("resolve_report_errors", || self.report_errors(krate));
-            self.session.time("resolve_postprocess", || self.crate_loader().postprocess(krate));
+            self.tcx.sess.time("finalize_macro_resolutions", || self.finalize_macro_resolutions());
+            self.tcx.sess.time("late_resolve_crate", || self.late_resolve_crate(krate));
+            self.tcx.sess.time("resolve_main", || self.resolve_main());
+            self.tcx.sess.time("resolve_check_unused", || self.check_unused(krate));
+            self.tcx.sess.time("resolve_report_errors", || self.report_errors(krate));
+            self.tcx
+                .sess
+                .time("resolve_postprocess", || self.crate_loader(|c| c.postprocess(krate)));
         });
+
+        // Make sure we don't mutate the cstore from here on.
+        self.tcx.untracked().cstore.leak();
     }
 
-    pub fn traits_in_scope(
+    fn traits_in_scope(
         &mut self,
         current_trait: Option<Module<'a>>,
         parent_scope: &ParentScope<'a>,
@@ -1911,10 +1836,10 @@ impl<'a> Resolver<'a> {
             } else {
                 let crate_id = if finalize {
                     let Some(crate_id) =
-                        self.crate_loader().process_path_extern(ident.name, ident.span) else { return Some(self.dummy_binding); };
+                        self.crate_loader(|c| c.process_path_extern(ident.name, ident.span)) else { return Some(self.dummy_binding); };
                     crate_id
                 } else {
-                    self.crate_loader().maybe_process_path_extern(ident.name)?
+                    self.crate_loader(|c| c.maybe_process_path_extern(ident.name))?
                 };
                 let crate_root = self.expect_module(crate_id.as_def_id());
                 let vis = ty::Visibility::<LocalDefId>::Public;
@@ -1927,7 +1852,7 @@ impl<'a> Resolver<'a> {
     /// isn't something that can be returned because it can't be made to live that long,
     /// and also it's a private type. Fortunately rustdoc doesn't need to know the error,
     /// just that an error occurred.
-    pub fn resolve_rustdoc_path(
+    fn resolve_rustdoc_path(
         &mut self,
         path_str: &str,
         ns: Namespace,
@@ -1959,41 +1884,17 @@ impl<'a> Resolver<'a> {
         }
     }
 
-    /// For rustdoc.
-    /// For local modules returns only reexports, for external modules returns all children.
-    pub fn module_children_or_reexports(&self, def_id: DefId) -> Vec<ModChild> {
-        if let Some(def_id) = def_id.as_local() {
-            self.reexport_map.get(&def_id).cloned().unwrap_or_default()
-        } else {
-            self.cstore().module_children_untracked(def_id, self.session).collect()
-        }
-    }
-
-    /// For rustdoc.
-    pub fn macro_rules_scope(&self, def_id: LocalDefId) -> (MacroRulesScopeRef<'a>, Res) {
-        let scope = *self.macro_rules_scopes.get(&def_id).expect("not a `macro_rules` item");
-        match scope.get() {
-            MacroRulesScope::Binding(mb) => (scope, mb.binding.res()),
-            _ => unreachable!(),
-        }
-    }
-
-    /// For rustdoc.
-    pub fn get_partial_res(&self, node_id: NodeId) -> Option<PartialRes> {
-        self.partial_res_map.get(&node_id).copied()
-    }
-
     /// Retrieves the span of the given `DefId` if `DefId` is in the local crate.
     #[inline]
-    pub fn opt_span(&self, def_id: DefId) -> Option<Span> {
-        def_id.as_local().map(|def_id| self.untracked.source_span[def_id])
+    fn opt_span(&self, def_id: DefId) -> Option<Span> {
+        def_id.as_local().map(|def_id| self.tcx.source_span(def_id))
     }
 
     /// Retrieves the name of the given `DefId`.
     #[inline]
-    pub fn opt_name(&self, def_id: DefId) -> Option<Symbol> {
+    fn opt_name(&self, def_id: DefId) -> Option<Symbol> {
         let def_key = match def_id.as_local() {
-            Some(def_id) => self.untracked.definitions.read().def_key(def_id),
+            Some(def_id) => self.tcx.definitions_untracked().def_key(def_id),
             None => self.cstore().def_key(def_id),
         };
         def_key.get_opt_name()
@@ -2002,7 +1903,7 @@ impl<'a> Resolver<'a> {
     /// Checks if an expression refers to a function marked with
     /// `#[rustc_legacy_const_generics]` and returns the argument index list
     /// from the attribute.
-    pub fn legacy_const_generic_args(&mut self, expr: &Expr) -> Option<Vec<usize>> {
+    fn legacy_const_generic_args(&mut self, expr: &Expr) -> Option<Vec<usize>> {
         if let ExprKind::Path(None, path) = &expr.kind {
             // Don't perform legacy const generics rewriting if the path already
             // has generic arguments.
@@ -2025,7 +1926,7 @@ impl<'a> Resolver<'a> {
 
                 let attr = self
                     .cstore()
-                    .item_attrs_untracked(def_id, self.session)
+                    .item_attrs_untracked(def_id, self.tcx.sess)
                     .find(|a| a.has_name(sym::rustc_legacy_const_generics))?;
                 let mut ret = Vec::new();
                 for meta in attr.meta_item_list()? {
diff --git a/compiler/rustc_resolve/src/macros.rs b/compiler/rustc_resolve/src/macros.rs
index b5b1602c5..b38c11e8b 100644
--- a/compiler/rustc_resolve/src/macros.rs
+++ b/compiler/rustc_resolve/src/macros.rs
@@ -1,7 +1,6 @@
 //! A bunch of methods and structures more or less related to resolving macros and
 //! interface provided by `Resolver` to macro expander.
 
-use crate::imports::ImportResolver;
 use crate::Namespace::*;
 use crate::{BuiltinMacroState, Determinacy};
 use crate::{DeriveData, Finalize, ParentScope, ResolutionError, Resolver, ScopeSet};
@@ -39,7 +38,7 @@ type Res = def::Res<NodeId>;
 /// Binding produced by a `macro_rules` item.
 /// Not modularized, can shadow previous `macro_rules` bindings, etc.
 #[derive(Debug)]
-pub struct MacroRulesBinding<'a> {
+pub(crate) struct MacroRulesBinding<'a> {
     pub(crate) binding: &'a NameBinding<'a>,
     /// `macro_rules` scope into which the `macro_rules` item was planted.
     pub(crate) parent_macro_rules_scope: MacroRulesScopeRef<'a>,
@@ -52,7 +51,7 @@ pub struct MacroRulesBinding<'a> {
 /// Some macro invocations need to introduce `macro_rules` scopes too because they
 /// can potentially expand into macro definitions.
 #[derive(Copy, Clone, Debug)]
-pub enum MacroRulesScope<'a> {
+pub(crate) enum MacroRulesScope<'a> {
     /// Empty "root" scope at the crate start containing no names.
     Empty,
     /// The scope introduced by a `macro_rules!` macro definition.
@@ -160,7 +159,7 @@ fn soft_custom_inner_attributes_gate(path: &ast::Path, invoc: &Invocation) -> bo
     false
 }
 
-impl<'a> ResolverExpand for Resolver<'a> {
+impl<'a, 'tcx> ResolverExpand for Resolver<'a, 'tcx> {
     fn next_node_id(&mut self) -> NodeId {
         self.next_node_id()
     }
@@ -195,7 +194,8 @@ impl<'a> ResolverExpand for Resolver<'a> {
 
     fn register_builtin_macro(&mut self, name: Symbol, ext: SyntaxExtensionKind) {
         if self.builtin_macros.insert(name, BuiltinMacroState::NotYetSeen(ext)).is_some() {
-            self.session
+            self.tcx
+                .sess
                 .diagnostic()
                 .bug(&format!("built-in macro `{}` was already registered", name));
         }
@@ -216,7 +216,7 @@ impl<'a> ResolverExpand for Resolver<'a> {
             ExpnData::allow_unstable(
                 ExpnKind::AstPass(pass),
                 call_site,
-                self.session.edition(),
+                self.tcx.sess.edition(),
                 features.into(),
                 None,
                 parent_module,
@@ -232,7 +232,7 @@ impl<'a> ResolverExpand for Resolver<'a> {
     }
 
     fn resolve_imports(&mut self) {
-        ImportResolver { r: self }.resolve_imports()
+        self.resolve_imports()
     }
 
     fn resolve_macro_invocation(
@@ -430,7 +430,7 @@ impl<'a> ResolverExpand for Resolver<'a> {
                 PathResult::NonModule(..) |
                 // HACK(Urgau): This shouldn't be necessary
                 PathResult::Failed { is_error_from_last_segment: false, .. } => {
-                    self.session
+                    self.tcx.sess
                         .struct_span_err(span, "not sure whether the path is accessible or not")
                         .note("the type may have associated items, but we are currently not checking them")
                         .emit();
@@ -455,7 +455,7 @@ impl<'a> ResolverExpand for Resolver<'a> {
     }
 
     fn get_proc_macro_quoted_span(&self, krate: CrateNum, id: usize) -> Span {
-        self.cstore().get_proc_macro_quoted_span_untracked(krate, id, self.session)
+        self.cstore().get_proc_macro_quoted_span_untracked(krate, id, self.tcx.sess)
     }
 
     fn declare_proc_macro(&mut self, id: NodeId) {
@@ -467,7 +467,7 @@ impl<'a> ResolverExpand for Resolver<'a> {
     }
 }
 
-impl<'a> Resolver<'a> {
+impl<'a, 'tcx> Resolver<'a, 'tcx> {
     /// Resolve macro path with error reporting and recovery.
     /// Uses dummy syntax extensions for unresolved macros or macros with unexpected resolutions
     /// for better error recovery.
@@ -493,10 +493,10 @@ impl<'a> Resolver<'a> {
         // Report errors for the resolved macro.
         for segment in &path.segments {
             if let Some(args) = &segment.args {
-                self.session.span_err(args.span(), "generic arguments in macro path");
+                self.tcx.sess.span_err(args.span(), "generic arguments in macro path");
             }
             if kind == MacroKind::Attr && segment.ident.as_str().starts_with("rustc") {
-                self.session.span_err(
+                self.tcx.sess.span_err(
                     segment.ident.span,
                     "attributes starting with `rustc` are reserved for use by the `rustc` compiler",
                 );
@@ -508,7 +508,7 @@ impl<'a> Resolver<'a> {
                 if let Some(def_id) = def_id.as_local() {
                     self.unused_macros.remove(&def_id);
                     if self.proc_macro_stubs.contains(&def_id) {
-                        self.session.span_err(
+                        self.tcx.sess.span_err(
                             path.span,
                             "can't use a procedural macro from the same crate that defines it",
                         );
@@ -540,7 +540,8 @@ impl<'a> Resolver<'a> {
         if let Some((article, expected)) = unexpected_res {
             let path_str = pprust::path_to_string(path);
             let msg = format!("expected {}, found {} `{}`", expected, res.descr(), path_str);
-            self.session
+            self.tcx
+                .sess
                 .struct_span_err(path.span, &msg)
                 .span_label(path.span, format!("not {} {}", article, expected))
                 .emit();
@@ -550,7 +551,7 @@ impl<'a> Resolver<'a> {
         // We are trying to avoid reporting this error if other related errors were reported.
         if res != Res::Err
             && inner_attr
-            && !self.session.features_untracked().custom_inner_attributes
+            && !self.tcx.sess.features_untracked().custom_inner_attributes
         {
             let msg = match res {
                 Res::Def(..) => "inner macro attributes are unstable",
@@ -558,17 +559,22 @@ impl<'a> Resolver<'a> {
                 _ => unreachable!(),
             };
             if soft_custom_inner_attributes_gate {
-                self.session.parse_sess.buffer_lint(SOFT_UNSTABLE, path.span, node_id, msg);
+                self.tcx.sess.parse_sess.buffer_lint(SOFT_UNSTABLE, path.span, node_id, msg);
             } else {
-                feature_err(&self.session.parse_sess, sym::custom_inner_attributes, path.span, msg)
-                    .emit();
+                feature_err(
+                    &self.tcx.sess.parse_sess,
+                    sym::custom_inner_attributes,
+                    path.span,
+                    msg,
+                )
+                .emit();
             }
         }
 
         Ok((ext, res))
     }
 
-    pub fn resolve_macro_path(
+    pub(crate) fn resolve_macro_path(
         &mut self,
         path: &ast::Path,
         kind: Option<MacroKind>,
@@ -655,7 +661,7 @@ impl<'a> Resolver<'a> {
                     // Make sure compilation does not succeed if preferred macro resolution
                     // has changed after the macro had been expanded. In theory all such
                     // situations should be reported as errors, so this is a bug.
-                    this.session.delay_span_bug(span, "inconsistent resolution for a macro");
+                    this.tcx.sess.delay_span_bug(span, "inconsistent resolution for a macro");
                 }
             } else {
                 // It's possible that the macro was unresolved (indeterminate) and silently
@@ -672,7 +678,7 @@ impl<'a> Resolver<'a> {
                         Segment::names_to_string(path)
                     );
                     let msg_note = "import resolution is stuck, try simplifying macro imports";
-                    this.session.struct_span_err(span, &msg).note(msg_note).emit();
+                    this.tcx.sess.struct_span_err(span, &msg).note(msg_note).emit();
                 }
             }
         };
@@ -699,7 +705,7 @@ impl<'a> Resolver<'a> {
                         // try to suggest if it's not a macro, maybe a function
                         if let PathResult::NonModule(partial_res) = self.maybe_resolve_path(&path, Some(ValueNS), &parent_scope)
                             && partial_res.unresolved_segments() == 0 {
-                            let sm = self.session.source_map();
+                            let sm = self.tcx.sess.source_map();
                             let exclamation_span = sm.next_point(span);
                             suggestion = Some((
                                 vec![(exclamation_span, "".to_string())],
@@ -762,7 +768,7 @@ impl<'a> Resolver<'a> {
                 Err(..) => {
                     let expected = kind.descr_expected();
                     let msg = format!("cannot find {} `{}` in this scope", expected, ident);
-                    let mut err = self.session.struct_span_err(ident.span, &msg);
+                    let mut err = self.tcx.sess.struct_span_err(ident.span, &msg);
                     self.unresolved_macro_suggestions(&mut err, kind, &parent_scope, ident);
                     err.emit();
                 }
@@ -804,7 +810,7 @@ impl<'a> Resolver<'a> {
                     let soft_handler =
                         |lint, span, msg: &_| lint_buffer.buffer_lint(lint, node_id, span, msg);
                     stability::report_unstable(
-                        self.session,
+                        self.tcx.sess,
                         feature,
                         reason.to_opt_reason(),
                         issue,
@@ -840,7 +846,7 @@ impl<'a> Resolver<'a> {
             if kind != NonMacroAttrKind::Tool && binding.map_or(true, |b| b.is_import()) {
                 let msg =
                     format!("cannot use {} {} through an import", kind.article(), kind.descr());
-                let mut err = self.session.struct_span_err(span, &msg);
+                let mut err = self.tcx.sess.struct_span_err(span, &msg);
                 if let Some(binding) = binding {
                     err.span_note(binding.span, &format!("the {} imported here", kind.descr()));
                 }
@@ -855,7 +861,7 @@ impl<'a> Resolver<'a> {
         if ident.name == sym::cfg || ident.name == sym::cfg_attr {
             let macro_kind = self.get_macro(res).map(|macro_data| macro_data.ext.macro_kind());
             if macro_kind.is_some() && sub_namespace_match(macro_kind, Some(MacroKind::Attr)) {
-                self.session.span_err(
+                self.tcx.sess.span_err(
                     ident.span,
                     &format!("name `{}` is reserved in attribute namespace", ident),
                 );
@@ -871,12 +877,7 @@ impl<'a> Resolver<'a> {
         item: &ast::Item,
         edition: Edition,
     ) -> (SyntaxExtension, Vec<(usize, Span)>) {
-        let (mut result, mut rule_spans) = compile_declarative_macro(
-            &self.session,
-            self.session.features_untracked(),
-            item,
-            edition,
-        );
+        let (mut result, mut rule_spans) = compile_declarative_macro(self.tcx.sess, item, edition);
 
         if let Some(builtin_name) = result.builtin_name {
             // The macro was marked with `#[rustc_builtin_macro]`.
@@ -895,7 +896,7 @@ impl<'a> Resolver<'a> {
                     }
                     BuiltinMacroState::AlreadySeen(span) => {
                         struct_span_err!(
-                            self.session,
+                            self.tcx.sess,
                             item.span,
                             E0773,
                             "attempted to define built-in macro more than once"
@@ -906,7 +907,7 @@ impl<'a> Resolver<'a> {
                 }
             } else {
                 let msg = format!("cannot find a built-in macro with name `{}`", item.ident);
-                self.session.span_err(item.span, &msg);
+                self.tcx.sess.span_err(item.span, &msg);
             }
         }
 
diff --git a/compiler/rustc_resolve/src/rustdoc.rs b/compiler/rustc_resolve/src/rustdoc.rs
new file mode 100644
index 000000000..b8853c174
--- /dev/null
+++ b/compiler/rustc_resolve/src/rustdoc.rs
@@ -0,0 +1,388 @@
+use pulldown_cmark::{BrokenLink, Event, LinkType, Options, Parser, Tag};
+use rustc_ast as ast;
+use rustc_ast::util::comments::beautify_doc_string;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_span::def_id::DefId;
+use rustc_span::symbol::{kw, Symbol};
+use rustc_span::Span;
+use std::{cmp, mem};
+
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+pub enum DocFragmentKind {
+    /// A doc fragment created from a `///` or `//!` doc comment.
+    SugaredDoc,
+    /// A doc fragment created from a "raw" `#[doc=""]` attribute.
+    RawDoc,
+}
+
+/// A portion of documentation, extracted from a `#[doc]` attribute.
+///
+/// Each variant contains the line number within the complete doc-comment where the fragment
+/// starts, as well as the Span where the corresponding doc comment or attribute is located.
+///
+/// Included files are kept separate from inline doc comments so that proper line-number
+/// information can be given when a doctest fails. Sugared doc comments and "raw" doc comments are
+/// kept separate because of issue #42760.
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub struct DocFragment {
+    pub span: Span,
+    /// The module this doc-comment came from.
+    ///
+    /// This allows distinguishing between the original documentation and a pub re-export.
+    /// If it is `None`, the item was not re-exported.
+    pub parent_module: Option<DefId>,
+    pub doc: Symbol,
+    pub kind: DocFragmentKind,
+    pub indent: usize,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub enum MalformedGenerics {
+    /// This link has unbalanced angle brackets.
+    ///
+    /// For example, `Vec<T` should trigger this, as should `Vec<T>>`.
+    UnbalancedAngleBrackets,
+    /// The generics are not attached to a type.
+    ///
+    /// For example, `<T>` should trigger this.
+    ///
+    /// This is detected by checking if the path is empty after the generics are stripped.
+    MissingType,
+    /// The link uses fully-qualified syntax, which is currently unsupported.
+    ///
+    /// For example, `<Vec as IntoIterator>::into_iter` should trigger this.
+    ///
+    /// This is detected by checking if ` as ` (the keyword `as` with spaces around it) is inside
+    /// angle brackets.
+    HasFullyQualifiedSyntax,
+    /// The link has an invalid path separator.
+    ///
+    /// For example, `Vec:<T>:new()` should trigger this. Note that `Vec:new()` will **not**
+    /// trigger this because it has no generics and thus [`strip_generics_from_path`] will not be
+    /// called.
+    ///
+    /// Note that this will also **not** be triggered if the invalid path separator is inside angle
+    /// brackets because rustdoc mostly ignores what's inside angle brackets (except for
+    /// [`HasFullyQualifiedSyntax`](MalformedGenerics::HasFullyQualifiedSyntax)).
+    ///
+    /// This is detected by checking if there is a colon followed by a non-colon in the link.
+    InvalidPathSeparator,
+    /// The link has too many angle brackets.
+    ///
+    /// For example, `Vec<<T>>` should trigger this.
+    TooManyAngleBrackets,
+    /// The link has empty angle brackets.
+    ///
+    /// For example, `Vec<>` should trigger this.
+    EmptyAngleBrackets,
+}
+
+/// Removes excess indentation on comments in order for the Markdown
+/// to be parsed correctly. This is necessary because the convention for
+/// writing documentation is to provide a space between the /// or //! marker
+/// and the doc text, but Markdown is whitespace-sensitive. For example,
+/// a block of text with four-space indentation is parsed as a code block,
+/// so if we didn't unindent comments, these list items
+///
+/// /// A list:
+/// ///
+/// ///    - Foo
+/// ///    - Bar
+///
+/// would be parsed as if they were in a code block, which is likely not what the user intended.
+pub fn unindent_doc_fragments(docs: &mut [DocFragment]) {
+    // `add` is used in case the most common sugared doc syntax is used ("/// "). The other
+    // fragments kind's lines are never starting with a whitespace unless they are using some
+    // markdown formatting requiring it. Therefore, if the doc block have a mix between the two,
+    // we need to take into account the fact that the minimum indent minus one (to take this
+    // whitespace into account).
+    //
+    // For example:
+    //
+    // /// hello!
+    // #[doc = "another"]
+    //
+    // In this case, you want "hello! another" and not "hello!  another".
+    let add = if docs.windows(2).any(|arr| arr[0].kind != arr[1].kind)
+        && docs.iter().any(|d| d.kind == DocFragmentKind::SugaredDoc)
+    {
+        // In case we have a mix of sugared doc comments and "raw" ones, we want the sugared one to
+        // "decide" how much the minimum indent will be.
+        1
+    } else {
+        0
+    };
+
+    // `min_indent` is used to know how much whitespaces from the start of each lines must be
+    // removed. Example:
+    //
+    // ///     hello!
+    // #[doc = "another"]
+    //
+    // In here, the `min_indent` is 1 (because non-sugared fragment are always counted with minimum
+    // 1 whitespace), meaning that "hello!" will be considered a codeblock because it starts with 4
+    // (5 - 1) whitespaces.
+    let Some(min_indent) = docs
+        .iter()
+        .map(|fragment| {
+            fragment.doc.as_str().lines().fold(usize::MAX, |min_indent, line| {
+                if line.chars().all(|c| c.is_whitespace()) {
+                    min_indent
+                } else {
+                    // Compare against either space or tab, ignoring whether they are
+                    // mixed or not.
+                    let whitespace = line.chars().take_while(|c| *c == ' ' || *c == '\t').count();
+                    cmp::min(min_indent, whitespace)
+                        + if fragment.kind == DocFragmentKind::SugaredDoc { 0 } else { add }
+                }
+            })
+        })
+        .min()
+    else {
+        return;
+    };
+
+    for fragment in docs {
+        if fragment.doc == kw::Empty {
+            continue;
+        }
+
+        let min_indent = if fragment.kind != DocFragmentKind::SugaredDoc && min_indent > 0 {
+            min_indent - add
+        } else {
+            min_indent
+        };
+
+        fragment.indent = min_indent;
+    }
+}
+
+/// The goal of this function is to apply the `DocFragment` transformation that is required when
+/// transforming into the final Markdown, which is applying the computed indent to each line in
+/// each doc fragment (a `DocFragment` can contain multiple lines in case of `#[doc = ""]`).
+///
+/// Note: remove the trailing newline where appropriate
+pub fn add_doc_fragment(out: &mut String, frag: &DocFragment) {
+    let s = frag.doc.as_str();
+    let mut iter = s.lines();
+    if s.is_empty() {
+        out.push('\n');
+        return;
+    }
+    while let Some(line) = iter.next() {
+        if line.chars().any(|c| !c.is_whitespace()) {
+            assert!(line.len() >= frag.indent);
+            out.push_str(&line[frag.indent..]);
+        } else {
+            out.push_str(line);
+        }
+        out.push('\n');
+    }
+}
+
+pub fn attrs_to_doc_fragments<'a>(
+    attrs: impl Iterator<Item = (&'a ast::Attribute, Option<DefId>)>,
+    doc_only: bool,
+) -> (Vec<DocFragment>, ast::AttrVec) {
+    let mut doc_fragments = Vec::new();
+    let mut other_attrs = ast::AttrVec::new();
+    for (attr, parent_module) in attrs {
+        if let Some((doc_str, comment_kind)) = attr.doc_str_and_comment_kind() {
+            let doc = beautify_doc_string(doc_str, comment_kind);
+            let kind = if attr.is_doc_comment() {
+                DocFragmentKind::SugaredDoc
+            } else {
+                DocFragmentKind::RawDoc
+            };
+            let fragment = DocFragment { span: attr.span, doc, kind, parent_module, indent: 0 };
+            doc_fragments.push(fragment);
+        } else if !doc_only {
+            other_attrs.push(attr.clone());
+        }
+    }
+
+    unindent_doc_fragments(&mut doc_fragments);
+
+    (doc_fragments, other_attrs)
+}
+
+/// Return the doc-comments on this item, grouped by the module they came from.
+/// The module can be different if this is a re-export with added documentation.
+///
+/// The last newline is not trimmed so the produced strings are reusable between
+/// early and late doc link resolution regardless of their position.
+pub fn prepare_to_doc_link_resolution(
+    doc_fragments: &[DocFragment],
+) -> FxHashMap<Option<DefId>, String> {
+    let mut res = FxHashMap::default();
+    for fragment in doc_fragments {
+        let out_str = res.entry(fragment.parent_module).or_default();
+        add_doc_fragment(out_str, fragment);
+    }
+    res
+}
+
+/// Options for rendering Markdown in the main body of documentation.
+pub fn main_body_opts() -> Options {
+    Options::ENABLE_TABLES
+        | Options::ENABLE_FOOTNOTES
+        | Options::ENABLE_STRIKETHROUGH
+        | Options::ENABLE_TASKLISTS
+        | Options::ENABLE_SMART_PUNCTUATION
+}
+
+fn strip_generics_from_path_segment(segment: Vec<char>) -> Result<String, MalformedGenerics> {
+    let mut stripped_segment = String::new();
+    let mut param_depth = 0;
+
+    let mut latest_generics_chunk = String::new();
+
+    for c in segment {
+        if c == '<' {
+            param_depth += 1;
+            latest_generics_chunk.clear();
+        } else if c == '>' {
+            param_depth -= 1;
+            if latest_generics_chunk.contains(" as ") {
+                // The segment tries to use fully-qualified syntax, which is currently unsupported.
+                // Give a helpful error message instead of completely ignoring the angle brackets.
+                return Err(MalformedGenerics::HasFullyQualifiedSyntax);
+            }
+        } else {
+            if param_depth == 0 {
+                stripped_segment.push(c);
+            } else {
+                latest_generics_chunk.push(c);
+            }
+        }
+    }
+
+    if param_depth == 0 {
+        Ok(stripped_segment)
+    } else {
+        // The segment has unbalanced angle brackets, e.g. `Vec<T` or `Vec<T>>`
+        Err(MalformedGenerics::UnbalancedAngleBrackets)
+    }
+}
+
+pub fn strip_generics_from_path(path_str: &str) -> Result<Box<str>, MalformedGenerics> {
+    if !path_str.contains(['<', '>']) {
+        return Ok(path_str.into());
+    }
+    let mut stripped_segments = vec![];
+    let mut path = path_str.chars().peekable();
+    let mut segment = Vec::new();
+
+    while let Some(chr) = path.next() {
+        match chr {
+            ':' => {
+                if path.next_if_eq(&':').is_some() {
+                    let stripped_segment =
+                        strip_generics_from_path_segment(mem::take(&mut segment))?;
+                    if !stripped_segment.is_empty() {
+                        stripped_segments.push(stripped_segment);
+                    }
+                } else {
+                    return Err(MalformedGenerics::InvalidPathSeparator);
+                }
+            }
+            '<' => {
+                segment.push(chr);
+
+                match path.next() {
+                    Some('<') => {
+                        return Err(MalformedGenerics::TooManyAngleBrackets);
+                    }
+                    Some('>') => {
+                        return Err(MalformedGenerics::EmptyAngleBrackets);
+                    }
+                    Some(chr) => {
+                        segment.push(chr);
+
+                        while let Some(chr) = path.next_if(|c| *c != '>') {
+                            segment.push(chr);
+                        }
+                    }
+                    None => break,
+                }
+            }
+            _ => segment.push(chr),
+        }
+        trace!("raw segment: {:?}", segment);
+    }
+
+    if !segment.is_empty() {
+        let stripped_segment = strip_generics_from_path_segment(segment)?;
+        if !stripped_segment.is_empty() {
+            stripped_segments.push(stripped_segment);
+        }
+    }
+
+    debug!("path_str: {:?}\nstripped segments: {:?}", path_str, &stripped_segments);
+
+    let stripped_path = stripped_segments.join("::");
+
+    if !stripped_path.is_empty() {
+        Ok(stripped_path.into())
+    } else {
+        Err(MalformedGenerics::MissingType)
+    }
+}
+
+/// Returns whether the first doc-comment is an inner attribute.
+///
+//// If there are no doc-comments, return true.
+/// FIXME(#78591): Support both inner and outer attributes on the same item.
+pub fn inner_docs(attrs: &[ast::Attribute]) -> bool {
+    attrs.iter().find(|a| a.doc_str().is_some()).map_or(true, |a| a.style == ast::AttrStyle::Inner)
+}
+
+/// Simplified version of the corresponding function in rustdoc.
+/// If the rustdoc version returns a successful result, this function must return the same result.
+/// Otherwise this function may return anything.
+fn preprocess_link(link: &str) -> Box<str> {
+    let link = link.replace('`', "");
+    let link = link.split('#').next().unwrap();
+    let link = link.trim();
+    let link = link.rsplit('@').next().unwrap();
+    let link = link.strip_suffix("()").unwrap_or(link);
+    let link = link.strip_suffix("{}").unwrap_or(link);
+    let link = link.strip_suffix("[]").unwrap_or(link);
+    let link = if link != "!" { link.strip_suffix('!').unwrap_or(link) } else { link };
+    strip_generics_from_path(link).unwrap_or_else(|_| link.into())
+}
+
+/// Keep inline and reference links `[]`,
+/// but skip autolinks `<>` which we never consider to be intra-doc links.
+pub fn may_be_doc_link(link_type: LinkType) -> bool {
+    match link_type {
+        LinkType::Inline
+        | LinkType::Reference
+        | LinkType::ReferenceUnknown
+        | LinkType::Collapsed
+        | LinkType::CollapsedUnknown
+        | LinkType::Shortcut
+        | LinkType::ShortcutUnknown => true,
+        LinkType::Autolink | LinkType::Email => false,
+    }
+}
+
+/// Simplified version of `preprocessed_markdown_links` from rustdoc.
+/// Must return at least the same links as it, but may add some more links on top of that.
+pub(crate) fn attrs_to_preprocessed_links(attrs: &[ast::Attribute]) -> Vec<Box<str>> {
+    let (doc_fragments, _) = attrs_to_doc_fragments(attrs.iter().map(|attr| (attr, None)), true);
+    let doc = prepare_to_doc_link_resolution(&doc_fragments).into_values().next().unwrap();
+
+    Parser::new_with_broken_link_callback(
+        &doc,
+        main_body_opts(),
+        Some(&mut |link: BrokenLink<'_>| Some((link.reference, "".into()))),
+    )
+    .filter_map(|event| match event {
+        Event::Start(Tag::Link(link_type, dest, _)) if may_be_doc_link(link_type) => {
+            Some(preprocess_link(&dest))
+        }
+        _ => None,
+    })
+    .collect()
+}
diff --git a/compiler/rustc_save_analysis/Cargo.toml b/compiler/rustc_save_analysis/Cargo.toml
deleted file mode 100644
index 181e27f33..000000000
--- a/compiler/rustc_save_analysis/Cargo.toml
+++ /dev/null
@@ -1,21 +0,0 @@
-[package]
-name = "rustc_save_analysis"
-version = "0.0.0"
-edition = "2021"
-
-[dependencies]
-tracing = "0.1"
-rustc_middle = { path = "../rustc_middle" }
-rustc_ast = { path = "../rustc_ast" }
-rustc_ast_pretty = { path = "../rustc_ast_pretty" }
-rustc_data_structures = { path = "../rustc_data_structures" }
-rustc_errors = { path = "../rustc_errors" }
-rustc_hir = { path = "../rustc_hir" }
-rustc_hir_pretty = { path = "../rustc_hir_pretty" }
-rustc_lexer = { path = "../rustc_lexer" }
-rustc_macros = { path = "../rustc_macros" }
-serde_json = "1"
-rustc_session = { path = "../rustc_session" }
-rustc_span = { path = "../rustc_span" }
-rls-data = "0.19"
-rls-span = "0.5"
diff --git a/compiler/rustc_save_analysis/src/dump_visitor.rs b/compiler/rustc_save_analysis/src/dump_visitor.rs
deleted file mode 100644
index a5f09de1c..000000000
--- a/compiler/rustc_save_analysis/src/dump_visitor.rs
+++ /dev/null
@@ -1,1471 +0,0 @@
-//! Write the output of rustc's analysis to an implementor of Dump.
-//!
-//! Dumping the analysis is implemented by walking the AST and getting a bunch of
-//! info out from all over the place. We use `DefId`s to identify objects. The
-//! tricky part is getting syntactic (span, source text) and semantic (reference
-//! `DefId`s) information for parts of expressions which the compiler has discarded.
-//! E.g., in a path `foo::bar::baz`, the compiler only keeps a span for the whole
-//! path and a reference to `baz`, but we want spans and references for all three
-//! idents.
-//!
-//! SpanUtils is used to manipulate spans. In particular, to extract sub-spans
-//! from spans (e.g., the span for `bar` from the above example path).
-//! DumpVisitor walks the AST and processes it, and Dumper is used for
-//! recording the output.
-
-use rustc_ast as ast;
-use rustc_ast::walk_list;
-use rustc_data_structures::fx::FxHashSet;
-use rustc_hir as hir;
-use rustc_hir::def::{DefKind as HirDefKind, Res};
-use rustc_hir::def_id::{DefId, LocalDefId, CRATE_DEF_ID};
-use rustc_hir::intravisit::{self, Visitor};
-use rustc_hir_pretty::{bounds_to_string, fn_to_string, generic_params_to_string, ty_to_string};
-use rustc_middle::hir::nested_filter;
-use rustc_middle::span_bug;
-use rustc_middle::ty::{self, DefIdTree, TyCtxt};
-use rustc_session::config::Input;
-use rustc_span::symbol::Ident;
-use rustc_span::*;
-
-use std::env;
-use std::path::Path;
-
-use crate::dumper::{Access, Dumper};
-use crate::sig;
-use crate::span_utils::SpanUtils;
-use crate::{
-    escape, generated_code, id_from_def_id, id_from_hir_id, lower_attributes, PathCollector,
-    SaveContext,
-};
-
-use rls_data::{
-    CompilationOptions, CratePreludeData, Def, DefKind, GlobalCrateId, Import, ImportKind, Ref,
-    RefKind, Relation, RelationKind, SpanData,
-};
-
-#[rustfmt::skip] // https://github.com/rust-lang/rustfmt/issues/5213
-macro_rules! down_cast_data {
-    ($id:ident, $kind:ident, $sp:expr) => {
-        let super::Data::$kind($id) = $id else {
-            span_bug!($sp, "unexpected data kind: {:?}", $id);
-        };
-    };
-}
-
-macro_rules! access_from {
-    ($save_ctxt:expr, $id:expr) => {
-        Access {
-            public: $save_ctxt.tcx.visibility($id).is_public(),
-            reachable: $save_ctxt.effective_visibilities.is_reachable($id),
-        }
-    };
-}
-
-pub struct DumpVisitor<'tcx> {
-    pub save_ctxt: SaveContext<'tcx>,
-    tcx: TyCtxt<'tcx>,
-    dumper: Dumper,
-
-    span: SpanUtils<'tcx>,
-    // Set of macro definition (callee) spans, and the set
-    // of macro use (callsite) spans. We store these to ensure
-    // we only write one macro def per unique macro definition, and
-    // one macro use per unique callsite span.
-    // mac_defs: FxHashSet<Span>,
-    // macro_calls: FxHashSet<Span>,
-}
-
-impl<'tcx> DumpVisitor<'tcx> {
-    pub fn new(save_ctxt: SaveContext<'tcx>) -> DumpVisitor<'tcx> {
-        let span_utils = SpanUtils::new(&save_ctxt.tcx.sess);
-        let dumper = Dumper::new(save_ctxt.config.clone());
-        DumpVisitor { tcx: save_ctxt.tcx, save_ctxt, dumper, span: span_utils }
-    }
-
-    pub fn analysis(&self) -> &rls_data::Analysis {
-        self.dumper.analysis()
-    }
-
-    fn nest_typeck_results<F>(&mut self, item_def_id: LocalDefId, f: F)
-    where
-        F: FnOnce(&mut Self),
-    {
-        let typeck_results = if self.tcx.has_typeck_results(item_def_id) {
-            Some(self.tcx.typeck(item_def_id))
-        } else {
-            None
-        };
-
-        let old_maybe_typeck_results = self.save_ctxt.maybe_typeck_results;
-        self.save_ctxt.maybe_typeck_results = typeck_results;
-        f(self);
-        self.save_ctxt.maybe_typeck_results = old_maybe_typeck_results;
-    }
-
-    fn span_from_span(&self, span: Span) -> SpanData {
-        self.save_ctxt.span_from_span(span)
-    }
-
-    fn lookup_def_id(&self, ref_id: hir::HirId) -> Option<DefId> {
-        self.save_ctxt.lookup_def_id(ref_id)
-    }
-
-    pub fn dump_crate_info(&mut self, name: Symbol) {
-        let crate_root = self.tcx.sess.local_crate_source_file().map(|source_file| {
-            match source_file.file_name() {
-                Some(_) => source_file.parent().unwrap().display(),
-                None => source_file.display(),
-            }
-            .to_string()
-        });
-
-        let data = CratePreludeData {
-            crate_id: GlobalCrateId {
-                name: name.to_string(),
-                disambiguator: (self.tcx.sess.local_stable_crate_id().to_u64(), 0),
-            },
-            crate_root: crate_root.unwrap_or_else(|| "<no source>".to_owned()),
-            external_crates: self.save_ctxt.get_external_crates(),
-            span: self.span_from_span(self.tcx.def_span(CRATE_DEF_ID)),
-        };
-
-        self.dumper.crate_prelude(data);
-    }
-
-    pub fn dump_compilation_options(&mut self, input: &Input, crate_name: Symbol) {
-        // Apply possible `remap-path-prefix` remapping to the input source file
-        // (and don't include remapping args anymore)
-        let (program, arguments) = {
-            let remap_arg_indices = {
-                let mut indices = FxHashSet::default();
-                // Args are guaranteed to be valid UTF-8 (checked early)
-                for (i, e) in env::args().enumerate() {
-                    if e.starts_with("--remap-path-prefix=") {
-                        indices.insert(i);
-                    } else if e == "--remap-path-prefix" {
-                        indices.insert(i);
-                        indices.insert(i + 1);
-                    }
-                }
-                indices
-            };
-
-            let mut args = env::args()
-                .enumerate()
-                .filter(|(i, _)| !remap_arg_indices.contains(i))
-                .map(|(_, arg)| match input {
-                    Input::File(ref path) if path == Path::new(&arg) => self
-                        .tcx
-                        .sess
-                        .local_crate_source_file()
-                        .as_ref()
-                        .unwrap()
-                        .to_string_lossy()
-                        .into(),
-                    _ => arg,
-                });
-
-            (args.next().unwrap(), args.collect())
-        };
-
-        let data = CompilationOptions {
-            directory: self.tcx.sess.opts.working_dir.remapped_path_if_available().into(),
-            program,
-            arguments,
-            output: self.save_ctxt.compilation_output(crate_name),
-        };
-
-        self.dumper.compilation_opts(data);
-    }
-
-    fn write_segments(&mut self, segments: impl IntoIterator<Item = &'tcx hir::PathSegment<'tcx>>) {
-        for seg in segments {
-            if let Some(data) = self.save_ctxt.get_path_segment_data(seg) {
-                self.dumper.dump_ref(data);
-            }
-        }
-    }
-
-    fn write_sub_paths<R>(&mut self, path: &'tcx hir::Path<'tcx, R>) {
-        self.write_segments(path.segments)
-    }
-
-    // As write_sub_paths, but does not process the last ident in the path (assuming it
-    // will be processed elsewhere). See note on write_sub_paths about global.
-    fn write_sub_paths_truncated<R>(&mut self, path: &'tcx hir::Path<'tcx, R>) {
-        if let [segments @ .., _] = path.segments {
-            self.write_segments(segments)
-        }
-    }
-
-    fn process_formals(&mut self, formals: &'tcx [hir::Param<'tcx>], qualname: &str) {
-        for arg in formals {
-            self.visit_pat(&arg.pat);
-            let mut collector = PathCollector::new(self.tcx);
-            collector.visit_pat(&arg.pat);
-
-            for (hir_id, ident, ..) in collector.collected_idents {
-                let typ = match self.save_ctxt.typeck_results().node_type_opt(hir_id) {
-                    Some(s) => s.to_string(),
-                    None => continue,
-                };
-                if !self.span.filter_generated(ident.span) {
-                    let id = id_from_hir_id(hir_id, &self.save_ctxt);
-                    let span = self.span_from_span(ident.span);
-
-                    self.dumper.dump_def(
-                        &Access { public: false, reachable: false },
-                        Def {
-                            kind: DefKind::Local,
-                            id,
-                            span,
-                            name: ident.to_string(),
-                            qualname: format!("{}::{}", qualname, ident),
-                            value: typ,
-                            parent: None,
-                            children: vec![],
-                            decl_id: None,
-                            docs: String::new(),
-                            sig: None,
-                            attributes: vec![],
-                        },
-                    );
-                }
-            }
-        }
-    }
-
-    fn process_method(
-        &mut self,
-        sig: &'tcx hir::FnSig<'tcx>,
-        body: Option<hir::BodyId>,
-        def_id: LocalDefId,
-        ident: Ident,
-        generics: &'tcx hir::Generics<'tcx>,
-        span: Span,
-    ) {
-        debug!("process_method: {:?}:{}", def_id, ident);
-
-        let map = self.tcx.hir();
-        let hir_id = map.local_def_id_to_hir_id(def_id);
-        self.nest_typeck_results(def_id, |v| {
-            if let Some(mut method_data) = v.save_ctxt.get_method_data(hir_id, ident, span) {
-                if let Some(body) = body {
-                    v.process_formals(map.body(body).params, &method_data.qualname);
-                }
-                v.process_generic_params(&generics, &method_data.qualname, hir_id);
-
-                method_data.value =
-                    fn_to_string(sig.decl, sig.header, Some(ident.name), generics, &[], None);
-                method_data.sig = sig::method_signature(hir_id, ident, generics, sig, &v.save_ctxt);
-
-                v.dumper.dump_def(&access_from!(v.save_ctxt, def_id), method_data);
-            }
-
-            // walk arg and return types
-            for arg in sig.decl.inputs {
-                v.visit_ty(arg);
-            }
-
-            if let hir::FnRetTy::Return(ref ret_ty) = sig.decl.output {
-                v.visit_ty(ret_ty)
-            }
-
-            // walk the fn body
-            if let Some(body) = body {
-                v.visit_expr(&map.body(body).value);
-            }
-        });
-    }
-
-    fn process_struct_field_def(
-        &mut self,
-        field: &'tcx hir::FieldDef<'tcx>,
-        parent_id: hir::HirId,
-    ) {
-        let field_data = self.save_ctxt.get_field_data(field, parent_id);
-        if let Some(field_data) = field_data {
-            self.dumper.dump_def(
-                &access_from!(self.save_ctxt, self.tcx.hir().local_def_id(field.hir_id)),
-                field_data,
-            );
-        }
-    }
-
-    // Dump generic params bindings, then visit_generics
-    fn process_generic_params(
-        &mut self,
-        generics: &'tcx hir::Generics<'tcx>,
-        prefix: &str,
-        id: hir::HirId,
-    ) {
-        for param in generics.params {
-            match param.kind {
-                hir::GenericParamKind::Lifetime { .. } => {}
-                hir::GenericParamKind::Type { .. } => {
-                    let param_ss = param.name.ident().span;
-                    let name = escape(self.span.snippet(param_ss));
-                    // Append $id to name to make sure each one is unique.
-                    let qualname = format!("{}::{}${}", prefix, name, id);
-                    if !self.span.filter_generated(param_ss) {
-                        let id = id_from_hir_id(param.hir_id, &self.save_ctxt);
-                        let span = self.span_from_span(param_ss);
-
-                        self.dumper.dump_def(
-                            &Access { public: false, reachable: false },
-                            Def {
-                                kind: DefKind::Type,
-                                id,
-                                span,
-                                name,
-                                qualname,
-                                value: String::new(),
-                                parent: None,
-                                children: vec![],
-                                decl_id: None,
-                                docs: String::new(),
-                                sig: None,
-                                attributes: vec![],
-                            },
-                        );
-                    }
-                }
-                hir::GenericParamKind::Const { .. } => {}
-            }
-        }
-
-        self.visit_generics(generics)
-    }
-
-    fn process_fn(
-        &mut self,
-        item: &'tcx hir::Item<'tcx>,
-        decl: &'tcx hir::FnDecl<'tcx>,
-        _header: &'tcx hir::FnHeader,
-        ty_params: &'tcx hir::Generics<'tcx>,
-        body: hir::BodyId,
-    ) {
-        let map = self.tcx.hir();
-        self.nest_typeck_results(item.owner_id.def_id, |v| {
-            let body = map.body(body);
-            if let Some(fn_data) = v.save_ctxt.get_item_data(item) {
-                down_cast_data!(fn_data, DefData, item.span);
-                v.process_formals(body.params, &fn_data.qualname);
-                v.process_generic_params(ty_params, &fn_data.qualname, item.hir_id());
-
-                v.dumper.dump_def(&access_from!(v.save_ctxt, item.owner_id.def_id), fn_data);
-            }
-
-            for arg in decl.inputs {
-                v.visit_ty(arg)
-            }
-
-            if let hir::FnRetTy::Return(ref ret_ty) = decl.output {
-                v.visit_ty(ret_ty)
-            }
-
-            v.visit_expr(&body.value);
-        });
-    }
-
-    fn process_static_or_const_item(
-        &mut self,
-        item: &'tcx hir::Item<'tcx>,
-        typ: &'tcx hir::Ty<'tcx>,
-        expr: &'tcx hir::Expr<'tcx>,
-    ) {
-        self.nest_typeck_results(item.owner_id.def_id, |v| {
-            if let Some(var_data) = v.save_ctxt.get_item_data(item) {
-                down_cast_data!(var_data, DefData, item.span);
-                v.dumper.dump_def(&access_from!(v.save_ctxt, item.owner_id.def_id), var_data);
-            }
-            v.visit_ty(&typ);
-            v.visit_expr(expr);
-        });
-    }
-
-    fn process_assoc_const(
-        &mut self,
-        def_id: LocalDefId,
-        ident: Ident,
-        typ: &'tcx hir::Ty<'tcx>,
-        expr: Option<&'tcx hir::Expr<'tcx>>,
-        parent_id: DefId,
-        attrs: &'tcx [ast::Attribute],
-    ) {
-        let qualname = format!("::{}", self.tcx.def_path_str(def_id.to_def_id()));
-
-        if !self.span.filter_generated(ident.span) {
-            let hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id);
-            let sig = sig::assoc_const_signature(hir_id, ident.name, typ, expr, &self.save_ctxt);
-            let span = self.span_from_span(ident.span);
-
-            self.dumper.dump_def(
-                &access_from!(self.save_ctxt, def_id),
-                Def {
-                    kind: DefKind::Const,
-                    id: id_from_hir_id(hir_id, &self.save_ctxt),
-                    span,
-                    name: ident.name.to_string(),
-                    qualname,
-                    value: ty_to_string(&typ),
-                    parent: Some(id_from_def_id(parent_id)),
-                    children: vec![],
-                    decl_id: None,
-                    docs: self.save_ctxt.docs_for_attrs(attrs),
-                    sig,
-                    attributes: lower_attributes(attrs.to_owned(), &self.save_ctxt),
-                },
-            );
-        }
-
-        // walk type and init value
-        self.nest_typeck_results(def_id, |v| {
-            v.visit_ty(typ);
-            if let Some(expr) = expr {
-                v.visit_expr(expr);
-            }
-        });
-    }
-
-    // FIXME tuple structs should generate tuple-specific data.
-    fn process_struct(
-        &mut self,
-        item: &'tcx hir::Item<'tcx>,
-        def: &'tcx hir::VariantData<'tcx>,
-        ty_params: &'tcx hir::Generics<'tcx>,
-    ) {
-        debug!("process_struct {:?} {:?}", item, item.span);
-        let name = item.ident.to_string();
-        let qualname = format!("::{}", self.tcx.def_path_str(item.owner_id.to_def_id()));
-
-        let kind = match item.kind {
-            hir::ItemKind::Struct(_, _) => DefKind::Struct,
-            hir::ItemKind::Union(_, _) => DefKind::Union,
-            _ => unreachable!(),
-        };
-
-        let (value, fields) = match item.kind {
-            hir::ItemKind::Struct(hir::VariantData::Struct(ref fields, ..), ..)
-            | hir::ItemKind::Union(hir::VariantData::Struct(ref fields, ..), ..) => {
-                let include_priv_fields = !self.save_ctxt.config.pub_only;
-                let fields_str = fields
-                    .iter()
-                    .filter_map(|f| {
-                        if include_priv_fields {
-                            return Some(f.ident.to_string());
-                        }
-                        let def_id = self.save_ctxt.tcx.hir().local_def_id(f.hir_id);
-                        if self.save_ctxt.tcx.visibility(def_id).is_public() {
-                            Some(f.ident.to_string())
-                        } else {
-                            None
-                        }
-                    })
-                    .collect::<Vec<_>>()
-                    .join(", ");
-                let value = format!("{} {{ {} }}", name, fields_str);
-                (value, fields.iter().map(|f| id_from_hir_id(f.hir_id, &self.save_ctxt)).collect())
-            }
-            _ => (String::new(), vec![]),
-        };
-
-        if !self.span.filter_generated(item.ident.span) {
-            let span = self.span_from_span(item.ident.span);
-            let attrs = self.tcx.hir().attrs(item.hir_id());
-            self.dumper.dump_def(
-                &access_from!(self.save_ctxt, item.owner_id.def_id),
-                Def {
-                    kind,
-                    id: id_from_def_id(item.owner_id.to_def_id()),
-                    span,
-                    name,
-                    qualname: qualname.clone(),
-                    value,
-                    parent: None,
-                    children: fields,
-                    decl_id: None,
-                    docs: self.save_ctxt.docs_for_attrs(attrs),
-                    sig: sig::item_signature(item, &self.save_ctxt),
-                    attributes: lower_attributes(attrs.to_vec(), &self.save_ctxt),
-                },
-            );
-        }
-
-        self.nest_typeck_results(item.owner_id.def_id, |v| {
-            for field in def.fields() {
-                v.process_struct_field_def(field, item.hir_id());
-                v.visit_ty(&field.ty);
-            }
-
-            v.process_generic_params(ty_params, &qualname, item.hir_id());
-        });
-    }
-
-    fn process_enum(
-        &mut self,
-        item: &'tcx hir::Item<'tcx>,
-        enum_definition: &'tcx hir::EnumDef<'tcx>,
-        ty_params: &'tcx hir::Generics<'tcx>,
-    ) {
-        let enum_data = self.save_ctxt.get_item_data(item);
-        let Some(enum_data) = enum_data else {
-            return;
-        };
-        down_cast_data!(enum_data, DefData, item.span);
-
-        let access = access_from!(self.save_ctxt, item.owner_id.def_id);
-
-        for variant in enum_definition.variants {
-            let name = variant.ident.name.to_string();
-            let qualname = format!("{}::{}", enum_data.qualname, name);
-            let name_span = variant.ident.span;
-
-            match variant.data {
-                hir::VariantData::Struct(ref fields, ..) => {
-                    let fields_str =
-                        fields.iter().map(|f| f.ident.to_string()).collect::<Vec<_>>().join(", ");
-                    let value = format!("{}::{} {{ {} }}", enum_data.name, name, fields_str);
-                    if !self.span.filter_generated(name_span) {
-                        let span = self.span_from_span(name_span);
-                        let id = id_from_hir_id(variant.hir_id, &self.save_ctxt);
-                        let parent = Some(id_from_def_id(item.owner_id.to_def_id()));
-                        let attrs = self.tcx.hir().attrs(variant.hir_id);
-
-                        self.dumper.dump_def(
-                            &access,
-                            Def {
-                                kind: DefKind::StructVariant,
-                                id,
-                                span,
-                                name,
-                                qualname,
-                                value,
-                                parent,
-                                children: vec![],
-                                decl_id: None,
-                                docs: self.save_ctxt.docs_for_attrs(attrs),
-                                sig: sig::variant_signature(variant, &self.save_ctxt),
-                                attributes: lower_attributes(attrs.to_vec(), &self.save_ctxt),
-                            },
-                        );
-                    }
-                }
-                ref v => {
-                    let mut value = format!("{}::{}", enum_data.name, name);
-                    if let hir::VariantData::Tuple(fields, _, _) = v {
-                        value.push('(');
-                        value.push_str(
-                            &fields
-                                .iter()
-                                .map(|f| ty_to_string(&f.ty))
-                                .collect::<Vec<_>>()
-                                .join(", "),
-                        );
-                        value.push(')');
-                    }
-                    if !self.span.filter_generated(name_span) {
-                        let span = self.span_from_span(name_span);
-                        let id = id_from_hir_id(variant.hir_id, &self.save_ctxt);
-                        let parent = Some(id_from_def_id(item.owner_id.to_def_id()));
-                        let attrs = self.tcx.hir().attrs(variant.hir_id);
-
-                        self.dumper.dump_def(
-                            &access,
-                            Def {
-                                kind: DefKind::TupleVariant,
-                                id,
-                                span,
-                                name,
-                                qualname,
-                                value,
-                                parent,
-                                children: vec![],
-                                decl_id: None,
-                                docs: self.save_ctxt.docs_for_attrs(attrs),
-                                sig: sig::variant_signature(variant, &self.save_ctxt),
-                                attributes: lower_attributes(attrs.to_vec(), &self.save_ctxt),
-                            },
-                        );
-                    }
-                }
-            }
-
-            for field in variant.data.fields() {
-                self.process_struct_field_def(field, variant.hir_id);
-                self.visit_ty(field.ty);
-            }
-        }
-        self.process_generic_params(ty_params, &enum_data.qualname, item.hir_id());
-        self.dumper.dump_def(&access, enum_data);
-    }
-
-    fn process_impl(&mut self, item: &'tcx hir::Item<'tcx>, impl_: &'tcx hir::Impl<'tcx>) {
-        if let Some(impl_data) = self.save_ctxt.get_item_data(item) {
-            if !self.span.filter_generated(item.span) {
-                if let super::Data::RelationData(rel, imp) = impl_data {
-                    self.dumper.dump_relation(rel);
-                    self.dumper.dump_impl(imp);
-                } else {
-                    span_bug!(item.span, "unexpected data kind: {:?}", impl_data);
-                }
-            }
-        }
-
-        let map = self.tcx.hir();
-        self.nest_typeck_results(item.owner_id.def_id, |v| {
-            v.visit_ty(&impl_.self_ty);
-            if let Some(trait_ref) = &impl_.of_trait {
-                v.process_path(trait_ref.hir_ref_id, &hir::QPath::Resolved(None, &trait_ref.path));
-            }
-            v.process_generic_params(&impl_.generics, "", item.hir_id());
-            for impl_item in impl_.items {
-                v.process_impl_item(map.impl_item(impl_item.id), item.owner_id.to_def_id());
-            }
-        });
-    }
-
-    fn process_trait(
-        &mut self,
-        item: &'tcx hir::Item<'tcx>,
-        generics: &'tcx hir::Generics<'tcx>,
-        trait_refs: hir::GenericBounds<'tcx>,
-        methods: &'tcx [hir::TraitItemRef],
-    ) {
-        let name = item.ident.to_string();
-        let qualname = format!("::{}", self.tcx.def_path_str(item.owner_id.to_def_id()));
-        let mut val = name.clone();
-        if !generics.params.is_empty() {
-            val.push_str(&generic_params_to_string(generics.params));
-        }
-        if !trait_refs.is_empty() {
-            val.push_str(": ");
-            val.push_str(&bounds_to_string(trait_refs));
-        }
-        if !self.span.filter_generated(item.ident.span) {
-            let id = id_from_def_id(item.owner_id.to_def_id());
-            let span = self.span_from_span(item.ident.span);
-            let children =
-                methods.iter().map(|i| id_from_def_id(i.id.owner_id.to_def_id())).collect();
-            let attrs = self.tcx.hir().attrs(item.hir_id());
-            self.dumper.dump_def(
-                &access_from!(self.save_ctxt, item.owner_id.def_id),
-                Def {
-                    kind: DefKind::Trait,
-                    id,
-                    span,
-                    name,
-                    qualname: qualname.clone(),
-                    value: val,
-                    parent: None,
-                    children,
-                    decl_id: None,
-                    docs: self.save_ctxt.docs_for_attrs(attrs),
-                    sig: sig::item_signature(item, &self.save_ctxt),
-                    attributes: lower_attributes(attrs.to_vec(), &self.save_ctxt),
-                },
-            );
-        }
-
-        // supertraits
-        for super_bound in trait_refs.iter() {
-            let (def_id, sub_span) = match *super_bound {
-                hir::GenericBound::Trait(ref trait_ref, _) => (
-                    self.lookup_def_id(trait_ref.trait_ref.hir_ref_id),
-                    trait_ref.trait_ref.path.segments.last().unwrap().ident.span,
-                ),
-                hir::GenericBound::LangItemTrait(lang_item, span, _, _) => {
-                    (Some(self.tcx.require_lang_item(lang_item, Some(span))), span)
-                }
-                hir::GenericBound::Outlives(..) => continue,
-            };
-
-            if let Some(id) = def_id {
-                if !self.span.filter_generated(sub_span) {
-                    let span = self.span_from_span(sub_span);
-                    self.dumper.dump_ref(Ref {
-                        kind: RefKind::Type,
-                        span: span.clone(),
-                        ref_id: id_from_def_id(id),
-                    });
-
-                    self.dumper.dump_relation(Relation {
-                        kind: RelationKind::SuperTrait,
-                        span,
-                        from: id_from_def_id(id),
-                        to: id_from_def_id(item.owner_id.to_def_id()),
-                    });
-                }
-            }
-        }
-
-        // walk generics and methods
-        self.process_generic_params(generics, &qualname, item.hir_id());
-        for method in methods {
-            let map = self.tcx.hir();
-            self.process_trait_item(map.trait_item(method.id), item.owner_id.to_def_id())
-        }
-    }
-
-    // `item` is the module in question, represented as an( item.
-    fn process_mod(&mut self, item: &'tcx hir::Item<'tcx>) {
-        if let Some(mod_data) = self.save_ctxt.get_item_data(item) {
-            down_cast_data!(mod_data, DefData, item.span);
-            self.dumper.dump_def(&access_from!(self.save_ctxt, item.owner_id.def_id), mod_data);
-        }
-    }
-
-    fn dump_path_ref(&mut self, id: hir::HirId, path: &hir::QPath<'tcx>) {
-        let path_data = self.save_ctxt.get_path_data(id, path);
-        if let Some(path_data) = path_data {
-            self.dumper.dump_ref(path_data);
-        }
-    }
-
-    fn dump_path_segment_ref(&mut self, id: hir::HirId, segment: &hir::PathSegment<'tcx>) {
-        let segment_data = self.save_ctxt.get_path_segment_data_with_id(segment, id);
-        if let Some(segment_data) = segment_data {
-            self.dumper.dump_ref(segment_data);
-        }
-    }
-
-    fn process_path(&mut self, id: hir::HirId, path: &hir::QPath<'tcx>) {
-        if self.span.filter_generated(path.span()) {
-            return;
-        }
-        self.dump_path_ref(id, path);
-
-        // Type arguments
-        let segments = match path {
-            hir::QPath::Resolved(ty, path) => {
-                if let Some(ty) = ty {
-                    self.visit_ty(ty);
-                }
-                path.segments
-            }
-            hir::QPath::TypeRelative(ty, segment) => {
-                self.visit_ty(ty);
-                std::slice::from_ref(*segment)
-            }
-            hir::QPath::LangItem(..) => return,
-        };
-        for seg in segments {
-            if let Some(ref generic_args) = seg.args {
-                for arg in generic_args.args {
-                    if let hir::GenericArg::Type(ref ty) = arg {
-                        self.visit_ty(ty);
-                    }
-                }
-            }
-        }
-
-        if let hir::QPath::Resolved(_, path) = path {
-            self.write_sub_paths_truncated(path);
-        }
-    }
-
-    fn process_struct_lit(
-        &mut self,
-        ex: &'tcx hir::Expr<'tcx>,
-        path: &'tcx hir::QPath<'tcx>,
-        fields: &'tcx [hir::ExprField<'tcx>],
-        variant: &'tcx ty::VariantDef,
-        rest: Option<&'tcx hir::Expr<'tcx>>,
-    ) {
-        if let Some(_ex_res_data) = self.save_ctxt.get_expr_data(ex) {
-            if let hir::QPath::Resolved(_, path) = path {
-                self.write_sub_paths_truncated(path);
-            }
-            // For MyEnum::MyVariant, get_expr_data gives us MyEnum, not MyVariant.
-            // For recording the span's ref id, we want MyVariant.
-            if !generated_code(ex.span) {
-                let sub_span = path.last_segment_span();
-                let span = self.save_ctxt.span_from_span(sub_span);
-                let reff =
-                    Ref { kind: RefKind::Type, span, ref_id: id_from_def_id(variant.def_id) };
-                self.dumper.dump_ref(reff);
-            }
-
-            for field in fields {
-                if let Some(field_data) = self.save_ctxt.get_field_ref_data(field, variant) {
-                    self.dumper.dump_ref(field_data);
-                }
-
-                self.visit_expr(&field.expr)
-            }
-        }
-
-        if let Some(base) = rest {
-            self.visit_expr(&base);
-        }
-    }
-
-    fn process_method_call(
-        &mut self,
-        ex: &'tcx hir::Expr<'tcx>,
-        seg: &'tcx hir::PathSegment<'tcx>,
-        receiver: &'tcx hir::Expr<'tcx>,
-        args: &'tcx [hir::Expr<'tcx>],
-    ) {
-        debug!("process_method_call {:?} {:?}", ex, ex.span);
-        if let Some(mcd) = self.save_ctxt.get_expr_data(ex) {
-            down_cast_data!(mcd, RefData, ex.span);
-            if !generated_code(ex.span) {
-                self.dumper.dump_ref(mcd);
-            }
-        }
-
-        // Explicit types in the turbo-fish.
-        if let Some(generic_args) = seg.args {
-            for arg in generic_args.args {
-                if let hir::GenericArg::Type(ty) = arg {
-                    self.visit_ty(&ty)
-                };
-            }
-        }
-
-        // walk receiver and args
-        self.visit_expr(receiver);
-        walk_list!(self, visit_expr, args);
-    }
-
-    fn process_pat(&mut self, p: &'tcx hir::Pat<'tcx>) {
-        match p.kind {
-            hir::PatKind::Struct(ref _path, fields, _) => {
-                // FIXME do something with _path?
-                let adt = match self.save_ctxt.typeck_results().node_type_opt(p.hir_id) {
-                    Some(ty) if ty.ty_adt_def().is_some() => ty.ty_adt_def().unwrap(),
-                    _ => {
-                        intravisit::walk_pat(self, p);
-                        return;
-                    }
-                };
-                let variant = adt.variant_of_res(self.save_ctxt.get_path_res(p.hir_id));
-
-                for field in fields {
-                    if let Some(index) = self.tcx.find_field_index(field.ident, variant) {
-                        if !self.span.filter_generated(field.ident.span) {
-                            let span = self.span_from_span(field.ident.span);
-                            self.dumper.dump_ref(Ref {
-                                kind: RefKind::Variable,
-                                span,
-                                ref_id: id_from_def_id(variant.fields[index].did),
-                            });
-                        }
-                    }
-                    self.visit_pat(&field.pat);
-                }
-            }
-            _ => intravisit::walk_pat(self, p),
-        }
-    }
-
-    fn process_var_decl(&mut self, pat: &'tcx hir::Pat<'tcx>) {
-        // The pattern could declare multiple new vars,
-        // we must walk the pattern and collect them all.
-        let mut collector = PathCollector::new(self.tcx);
-        collector.visit_pat(&pat);
-        self.visit_pat(&pat);
-
-        // Process collected paths.
-        for (id, ident, _) in collector.collected_idents {
-            let res = self.save_ctxt.get_path_res(id);
-            match res {
-                Res::Local(hir_id) => {
-                    let typ = self
-                        .save_ctxt
-                        .typeck_results()
-                        .node_type_opt(hir_id)
-                        .map(|t| t.to_string())
-                        .unwrap_or_default();
-
-                    // Rust uses the id of the pattern for var lookups, so we'll use it too.
-                    if !self.span.filter_generated(ident.span) {
-                        let qualname = format!("{}${}", ident, hir_id);
-                        let id = id_from_hir_id(hir_id, &self.save_ctxt);
-                        let span = self.span_from_span(ident.span);
-
-                        self.dumper.dump_def(
-                            &Access { public: false, reachable: false },
-                            Def {
-                                kind: DefKind::Local,
-                                id,
-                                span,
-                                name: ident.to_string(),
-                                qualname,
-                                value: typ,
-                                parent: None,
-                                children: vec![],
-                                decl_id: None,
-                                docs: String::new(),
-                                sig: None,
-                                attributes: vec![],
-                            },
-                        );
-                    }
-                }
-                Res::Def(
-                    HirDefKind::Ctor(..)
-                    | HirDefKind::Const
-                    | HirDefKind::AssocConst
-                    | HirDefKind::Struct
-                    | HirDefKind::Variant
-                    | HirDefKind::TyAlias
-                    | HirDefKind::AssocTy,
-                    _,
-                )
-                | Res::SelfTyParam { .. }
-                | Res::SelfTyAlias { .. } => {
-                    self.dump_path_segment_ref(
-                        id,
-                        &hir::PathSegment::new(ident, hir::HirId::INVALID, Res::Err),
-                    );
-                }
-                def => {
-                    error!("unexpected definition kind when processing collected idents: {:?}", def)
-                }
-            }
-        }
-
-        for (id, ref path) in collector.collected_paths {
-            self.process_path(id, path);
-        }
-    }
-
-    /// Extracts macro use and definition information from the AST node defined
-    /// by the given NodeId, using the expansion information from the node's
-    /// span.
-    ///
-    /// If the span is not macro-generated, do nothing, else use callee and
-    /// callsite spans to record macro definition and use data, using the
-    /// mac_uses and mac_defs sets to prevent multiples.
-    fn process_macro_use(&mut self, _span: Span) {
-        // FIXME if we're not dumping the defs (see below), there is no point
-        // dumping refs either.
-        // let source_span = span.source_callsite();
-        // if !self.macro_calls.insert(source_span) {
-        //     return;
-        // }
-
-        // let data = match self.save_ctxt.get_macro_use_data(span) {
-        //     None => return,
-        //     Some(data) => data,
-        // };
-
-        // self.dumper.macro_use(data);
-
-        // FIXME write the macro def
-        // let mut hasher = DefaultHasher::new();
-        // data.callee_span.hash(&mut hasher);
-        // let hash = hasher.finish();
-        // let qualname = format!("{}::{}", data.name, hash);
-        // Don't write macro definition for imported macros
-        // if !self.mac_defs.contains(&data.callee_span)
-        //     && !data.imported {
-        //     self.mac_defs.insert(data.callee_span);
-        //     if let Some(sub_span) = self.span.span_for_macro_def_name(data.callee_span) {
-        //         self.dumper.macro_data(MacroData {
-        //             span: sub_span,
-        //             name: data.name.clone(),
-        //             qualname: qualname.clone(),
-        //             // FIXME where do macro docs come from?
-        //             docs: String::new(),
-        //         }.lower(self.tcx));
-        //     }
-        // }
-    }
-
-    fn process_trait_item(&mut self, trait_item: &'tcx hir::TraitItem<'tcx>, trait_id: DefId) {
-        self.process_macro_use(trait_item.span);
-        match trait_item.kind {
-            hir::TraitItemKind::Const(ref ty, body) => {
-                let body = body.map(|b| self.tcx.hir().body(b).value);
-                let attrs = self.tcx.hir().attrs(trait_item.hir_id());
-                self.process_assoc_const(
-                    trait_item.owner_id.def_id,
-                    trait_item.ident,
-                    &ty,
-                    body,
-                    trait_id,
-                    attrs,
-                );
-            }
-            hir::TraitItemKind::Fn(ref sig, ref trait_fn) => {
-                let body =
-                    if let hir::TraitFn::Provided(body) = trait_fn { Some(*body) } else { None };
-                self.process_method(
-                    sig,
-                    body,
-                    trait_item.owner_id.def_id,
-                    trait_item.ident,
-                    &trait_item.generics,
-                    trait_item.span,
-                );
-            }
-            hir::TraitItemKind::Type(ref bounds, ref default_ty) => {
-                // FIXME do something with _bounds (for type refs)
-                let name = trait_item.ident.name.to_string();
-                let qualname =
-                    format!("::{}", self.tcx.def_path_str(trait_item.owner_id.to_def_id()));
-
-                if !self.span.filter_generated(trait_item.ident.span) {
-                    let span = self.span_from_span(trait_item.ident.span);
-                    let id = id_from_def_id(trait_item.owner_id.to_def_id());
-                    let attrs = self.tcx.hir().attrs(trait_item.hir_id());
-
-                    self.dumper.dump_def(
-                        &Access { public: true, reachable: true },
-                        Def {
-                            kind: DefKind::Type,
-                            id,
-                            span,
-                            name,
-                            qualname,
-                            value: self.span.snippet(trait_item.span),
-                            parent: Some(id_from_def_id(trait_id)),
-                            children: vec![],
-                            decl_id: None,
-                            docs: self.save_ctxt.docs_for_attrs(attrs),
-                            sig: sig::assoc_type_signature(
-                                trait_item.hir_id(),
-                                trait_item.ident,
-                                Some(bounds),
-                                default_ty.as_deref(),
-                                &self.save_ctxt,
-                            ),
-                            attributes: lower_attributes(attrs.to_vec(), &self.save_ctxt),
-                        },
-                    );
-                }
-
-                if let Some(default_ty) = default_ty {
-                    self.visit_ty(default_ty)
-                }
-            }
-        }
-    }
-
-    fn process_impl_item(&mut self, impl_item: &'tcx hir::ImplItem<'tcx>, impl_id: DefId) {
-        self.process_macro_use(impl_item.span);
-        match impl_item.kind {
-            hir::ImplItemKind::Const(ref ty, body) => {
-                let body = self.tcx.hir().body(body);
-                let attrs = self.tcx.hir().attrs(impl_item.hir_id());
-                self.process_assoc_const(
-                    impl_item.owner_id.def_id,
-                    impl_item.ident,
-                    &ty,
-                    Some(&body.value),
-                    impl_id,
-                    attrs,
-                );
-            }
-            hir::ImplItemKind::Fn(ref sig, body) => {
-                self.process_method(
-                    sig,
-                    Some(body),
-                    impl_item.owner_id.def_id,
-                    impl_item.ident,
-                    &impl_item.generics,
-                    impl_item.span,
-                );
-            }
-            hir::ImplItemKind::Type(ref ty) => {
-                // FIXME: uses of the assoc type should ideally point to this
-                // 'def' and the name here should be a ref to the def in the
-                // trait.
-                self.visit_ty(ty)
-            }
-        }
-    }
-
-    pub(crate) fn process_crate(&mut self) {
-        let id = hir::CRATE_HIR_ID;
-        let qualname =
-            format!("::{}", self.tcx.def_path_str(self.tcx.hir().local_def_id(id).to_def_id()));
-
-        let sm = self.tcx.sess.source_map();
-        let krate_mod = self.tcx.hir().root_module();
-        let filename = sm.span_to_filename(krate_mod.spans.inner_span);
-        let data_id = id_from_hir_id(id, &self.save_ctxt);
-        let children =
-            krate_mod.item_ids.iter().map(|i| id_from_def_id(i.owner_id.to_def_id())).collect();
-        let span = self.span_from_span(krate_mod.spans.inner_span);
-        let attrs = self.tcx.hir().attrs(id);
-
-        self.dumper.dump_def(
-            &Access { public: true, reachable: true },
-            Def {
-                kind: DefKind::Mod,
-                id: data_id,
-                name: String::new(),
-                qualname,
-                span,
-                value: filename.prefer_remapped().to_string(),
-                children,
-                parent: None,
-                decl_id: None,
-                docs: self.save_ctxt.docs_for_attrs(attrs),
-                sig: None,
-                attributes: lower_attributes(attrs.to_owned(), &self.save_ctxt),
-            },
-        );
-        self.tcx.hir().walk_toplevel_module(self);
-    }
-
-    fn process_bounds(&mut self, bounds: hir::GenericBounds<'tcx>) {
-        for bound in bounds {
-            if let hir::GenericBound::Trait(ref trait_ref, _) = *bound {
-                self.process_path(
-                    trait_ref.trait_ref.hir_ref_id,
-                    &hir::QPath::Resolved(None, &trait_ref.trait_ref.path),
-                )
-            }
-        }
-    }
-}
-
-impl<'tcx> Visitor<'tcx> for DumpVisitor<'tcx> {
-    type NestedFilter = nested_filter::All;
-
-    fn nested_visit_map(&mut self) -> Self::Map {
-        self.tcx.hir()
-    }
-
-    fn visit_item(&mut self, item: &'tcx hir::Item<'tcx>) {
-        self.process_macro_use(item.span);
-        match item.kind {
-            hir::ItemKind::Use(path, hir::UseKind::Single) => {
-                let sub_span = path.segments.last().unwrap().ident.span;
-                if !self.span.filter_generated(sub_span) {
-                    let access = access_from!(self.save_ctxt, item.owner_id.def_id);
-                    let ref_id = self.lookup_def_id(item.hir_id()).map(id_from_def_id);
-                    let span = self.span_from_span(sub_span);
-                    let parent = self.save_ctxt.tcx.local_parent(item.owner_id.def_id);
-                    self.dumper.import(
-                        &access,
-                        Import {
-                            kind: ImportKind::Use,
-                            ref_id,
-                            span,
-                            alias_span: None,
-                            name: item.ident.to_string(),
-                            value: String::new(),
-                            parent: Some(id_from_def_id(parent.to_def_id())),
-                        },
-                    );
-                    self.write_sub_paths_truncated(&path);
-                }
-            }
-            hir::ItemKind::Use(path, hir::UseKind::Glob) => {
-                // Make a comma-separated list of names of imported modules.
-                let names = self.tcx.names_imported_by_glob_use(item.owner_id.def_id);
-                let names: Vec<_> = names.iter().map(|n| n.to_string()).collect();
-
-                // Otherwise it's a span with wrong macro expansion info, which
-                // we don't want to track anyway, since it's probably macro-internal `use`
-                if let Some(sub_span) = self.span.sub_span_of_star(item.span) {
-                    if !self.span.filter_generated(item.span) {
-                        let access = access_from!(self.save_ctxt, item.owner_id.def_id);
-                        let span = self.span_from_span(sub_span);
-                        let parent = self.save_ctxt.tcx.local_parent(item.owner_id.def_id);
-                        self.dumper.import(
-                            &access,
-                            Import {
-                                kind: ImportKind::GlobUse,
-                                ref_id: None,
-                                span,
-                                alias_span: None,
-                                name: "*".to_owned(),
-                                value: names.join(", "),
-                                parent: Some(id_from_def_id(parent.to_def_id())),
-                            },
-                        );
-                        self.write_sub_paths(&path);
-                    }
-                }
-            }
-            hir::ItemKind::ExternCrate(_) => {
-                let name_span = item.ident.span;
-                if !self.span.filter_generated(name_span) {
-                    let span = self.span_from_span(name_span);
-                    let parent = self.save_ctxt.tcx.local_parent(item.owner_id.def_id);
-                    self.dumper.import(
-                        &Access { public: false, reachable: false },
-                        Import {
-                            kind: ImportKind::ExternCrate,
-                            ref_id: None,
-                            span,
-                            alias_span: None,
-                            name: item.ident.to_string(),
-                            value: String::new(),
-                            parent: Some(id_from_def_id(parent.to_def_id())),
-                        },
-                    );
-                }
-            }
-            hir::ItemKind::Fn(ref sig, ref ty_params, body) => {
-                self.process_fn(item, sig.decl, &sig.header, ty_params, body)
-            }
-            hir::ItemKind::Static(ref typ, _, body) => {
-                let body = self.tcx.hir().body(body);
-                self.process_static_or_const_item(item, typ, &body.value)
-            }
-            hir::ItemKind::Const(ref typ, body) => {
-                let body = self.tcx.hir().body(body);
-                self.process_static_or_const_item(item, typ, &body.value)
-            }
-            hir::ItemKind::Struct(ref def, ref ty_params)
-            | hir::ItemKind::Union(ref def, ref ty_params) => {
-                self.process_struct(item, def, ty_params)
-            }
-            hir::ItemKind::Enum(ref def, ref ty_params) => self.process_enum(item, def, ty_params),
-            hir::ItemKind::Impl(ref impl_) => self.process_impl(item, impl_),
-            hir::ItemKind::Trait(_, _, ref generics, ref trait_refs, methods) => {
-                self.process_trait(item, generics, trait_refs, methods)
-            }
-            hir::ItemKind::Mod(ref m) => {
-                self.process_mod(item);
-                intravisit::walk_mod(self, m, item.hir_id());
-            }
-            hir::ItemKind::TyAlias(ty, ref generics) => {
-                let qualname = format!("::{}", self.tcx.def_path_str(item.owner_id.to_def_id()));
-                let value = ty_to_string(&ty);
-                if !self.span.filter_generated(item.ident.span) {
-                    let span = self.span_from_span(item.ident.span);
-                    let id = id_from_def_id(item.owner_id.to_def_id());
-                    let attrs = self.tcx.hir().attrs(item.hir_id());
-
-                    self.dumper.dump_def(
-                        &access_from!(self.save_ctxt, item.owner_id.def_id),
-                        Def {
-                            kind: DefKind::Type,
-                            id,
-                            span,
-                            name: item.ident.to_string(),
-                            qualname: qualname.clone(),
-                            value,
-                            parent: None,
-                            children: vec![],
-                            decl_id: None,
-                            docs: self.save_ctxt.docs_for_attrs(attrs),
-                            sig: sig::item_signature(item, &self.save_ctxt),
-                            attributes: lower_attributes(attrs.to_vec(), &self.save_ctxt),
-                        },
-                    );
-                }
-
-                self.visit_ty(ty);
-                self.process_generic_params(generics, &qualname, item.hir_id());
-            }
-            _ => intravisit::walk_item(self, item),
-        }
-    }
-
-    fn visit_generics(&mut self, generics: &'tcx hir::Generics<'tcx>) {
-        for param in generics.params {
-            match param.kind {
-                hir::GenericParamKind::Lifetime { .. } => {}
-                hir::GenericParamKind::Type { ref default, .. } => {
-                    if let Some(ref ty) = default {
-                        self.visit_ty(ty);
-                    }
-                }
-                hir::GenericParamKind::Const { ref ty, ref default } => {
-                    self.visit_ty(ty);
-                    if let Some(default) = default {
-                        self.visit_anon_const(default);
-                    }
-                }
-            }
-        }
-        for pred in generics.predicates {
-            if let hir::WherePredicate::BoundPredicate(ref wbp) = *pred {
-                self.process_bounds(wbp.bounds);
-                self.visit_ty(wbp.bounded_ty);
-            }
-        }
-    }
-
-    fn visit_ty(&mut self, t: &'tcx hir::Ty<'tcx>) {
-        self.process_macro_use(t.span);
-        match t.kind {
-            hir::TyKind::Path(ref path) => {
-                if generated_code(t.span) {
-                    return;
-                }
-
-                if let Some(id) = self.lookup_def_id(t.hir_id) {
-                    let sub_span = path.last_segment_span();
-                    let span = self.span_from_span(sub_span);
-                    self.dumper.dump_ref(Ref {
-                        kind: RefKind::Type,
-                        span,
-                        ref_id: id_from_def_id(id),
-                    });
-                }
-
-                if let hir::QPath::Resolved(_, path) = path {
-                    self.write_sub_paths_truncated(path);
-                }
-                intravisit::walk_qpath(self, path, t.hir_id);
-            }
-            hir::TyKind::Array(ref ty, ref length) => {
-                self.visit_ty(ty);
-                let map = self.tcx.hir();
-                match length {
-                    // FIXME(generic_arg_infer): We probably want to
-                    // output the inferred type here? :shrug:
-                    hir::ArrayLen::Infer(..) => {}
-                    hir::ArrayLen::Body(anon_const) => self
-                        .nest_typeck_results(self.tcx.hir().local_def_id(anon_const.hir_id), |v| {
-                            v.visit_expr(&map.body(anon_const.body).value)
-                        }),
-                }
-            }
-            hir::TyKind::OpaqueDef(item_id, _, _) => {
-                let item = self.tcx.hir().item(item_id);
-                self.nest_typeck_results(item_id.owner_id.def_id, |v| v.visit_item(item));
-            }
-            _ => intravisit::walk_ty(self, t),
-        }
-    }
-
-    fn visit_expr(&mut self, ex: &'tcx hir::Expr<'tcx>) {
-        debug!("visit_expr {:?}", ex.kind);
-        self.process_macro_use(ex.span);
-        match ex.kind {
-            hir::ExprKind::Struct(ref path, ref fields, ref rest) => {
-                let hir_expr = self.save_ctxt.tcx.hir().expect_expr(ex.hir_id);
-                let adt = match self.save_ctxt.typeck_results().expr_ty_opt(&hir_expr) {
-                    Some(ty) if ty.ty_adt_def().is_some() => ty.ty_adt_def().unwrap(),
-                    _ => {
-                        intravisit::walk_expr(self, ex);
-                        return;
-                    }
-                };
-                let res = self.save_ctxt.get_path_res(hir_expr.hir_id);
-                self.process_struct_lit(ex, path, fields, adt.variant_of_res(res), *rest)
-            }
-            hir::ExprKind::MethodCall(ref seg, receiver, args, _) => {
-                self.process_method_call(ex, seg, receiver, args)
-            }
-            hir::ExprKind::Field(ref sub_ex, _) => {
-                self.visit_expr(&sub_ex);
-
-                if let Some(field_data) = self.save_ctxt.get_expr_data(ex) {
-                    down_cast_data!(field_data, RefData, ex.span);
-                    if !generated_code(ex.span) {
-                        self.dumper.dump_ref(field_data);
-                    }
-                }
-            }
-            hir::ExprKind::Closure(&hir::Closure { ref fn_decl, body, .. }) => {
-                let id = format!("${}", ex.hir_id);
-
-                // walk arg and return types
-                for ty in fn_decl.inputs {
-                    self.visit_ty(ty);
-                }
-
-                if let hir::FnRetTy::Return(ref ret_ty) = fn_decl.output {
-                    self.visit_ty(ret_ty);
-                }
-
-                // walk the body
-                let map = self.tcx.hir();
-                self.nest_typeck_results(self.tcx.hir().local_def_id(ex.hir_id), |v| {
-                    let body = map.body(body);
-                    v.process_formals(body.params, &id);
-                    v.visit_expr(&body.value)
-                });
-            }
-            hir::ExprKind::Repeat(ref expr, ref length) => {
-                self.visit_expr(expr);
-                let map = self.tcx.hir();
-                match length {
-                    // FIXME(generic_arg_infer): We probably want to
-                    // output the inferred type here? :shrug:
-                    hir::ArrayLen::Infer(..) => {}
-                    hir::ArrayLen::Body(anon_const) => self
-                        .nest_typeck_results(self.tcx.hir().local_def_id(anon_const.hir_id), |v| {
-                            v.visit_expr(&map.body(anon_const.body).value)
-                        }),
-                }
-            }
-            // In particular, we take this branch for call and path expressions,
-            // where we'll index the idents involved just by continuing to walk.
-            _ => intravisit::walk_expr(self, ex),
-        }
-    }
-
-    fn visit_pat(&mut self, p: &'tcx hir::Pat<'tcx>) {
-        self.process_macro_use(p.span);
-        self.process_pat(p);
-    }
-
-    fn visit_arm(&mut self, arm: &'tcx hir::Arm<'tcx>) {
-        self.process_var_decl(&arm.pat);
-        if let Some(hir::Guard::If(expr)) = &arm.guard {
-            self.visit_expr(expr);
-        }
-        self.visit_expr(&arm.body);
-    }
-
-    fn visit_qpath(&mut self, path: &'tcx hir::QPath<'tcx>, id: hir::HirId, _: Span) {
-        self.process_path(id, path);
-    }
-
-    fn visit_stmt(&mut self, s: &'tcx hir::Stmt<'tcx>) {
-        self.process_macro_use(s.span);
-        intravisit::walk_stmt(self, s)
-    }
-
-    fn visit_local(&mut self, l: &'tcx hir::Local<'tcx>) {
-        self.process_macro_use(l.span);
-        self.process_var_decl(&l.pat);
-
-        // Just walk the initializer, the else branch and type (don't want to walk the pattern again).
-        walk_list!(self, visit_ty, &l.ty);
-        walk_list!(self, visit_expr, &l.init);
-        walk_list!(self, visit_block, l.els);
-    }
-
-    fn visit_foreign_item(&mut self, item: &'tcx hir::ForeignItem<'tcx>) {
-        let access = access_from!(self.save_ctxt, item.owner_id.def_id);
-
-        match item.kind {
-            hir::ForeignItemKind::Fn(decl, _, ref generics) => {
-                if let Some(fn_data) = self.save_ctxt.get_extern_item_data(item) {
-                    down_cast_data!(fn_data, DefData, item.span);
-
-                    self.process_generic_params(generics, &fn_data.qualname, item.hir_id());
-                    self.dumper.dump_def(&access, fn_data);
-                }
-
-                for ty in decl.inputs {
-                    self.visit_ty(ty);
-                }
-
-                if let hir::FnRetTy::Return(ref ret_ty) = decl.output {
-                    self.visit_ty(ret_ty);
-                }
-            }
-            hir::ForeignItemKind::Static(ref ty, _) => {
-                if let Some(var_data) = self.save_ctxt.get_extern_item_data(item) {
-                    down_cast_data!(var_data, DefData, item.span);
-                    self.dumper.dump_def(&access, var_data);
-                }
-
-                self.visit_ty(ty);
-            }
-            hir::ForeignItemKind::Type => {
-                if let Some(var_data) = self.save_ctxt.get_extern_item_data(item) {
-                    down_cast_data!(var_data, DefData, item.span);
-                    self.dumper.dump_def(&access, var_data);
-                }
-            }
-        }
-    }
-}
diff --git a/compiler/rustc_save_analysis/src/dumper.rs b/compiler/rustc_save_analysis/src/dumper.rs
deleted file mode 100644
index 5a2628287..000000000
--- a/compiler/rustc_save_analysis/src/dumper.rs
+++ /dev/null
@@ -1,91 +0,0 @@
-use rls_data::config::Config;
-use rls_data::{
-    self, Analysis, CompilationOptions, CratePreludeData, Def, DefKind, Impl, Import, MacroRef,
-    Ref, RefKind, Relation,
-};
-use rls_span::{Column, Row};
-
-#[derive(Debug)]
-pub struct Access {
-    pub reachable: bool,
-    pub public: bool,
-}
-
-pub struct Dumper {
-    result: Analysis,
-    config: Config,
-}
-
-impl Dumper {
-    pub fn new(config: Config) -> Dumper {
-        Dumper { config: config.clone(), result: Analysis::new(config) }
-    }
-
-    pub fn analysis(&self) -> &Analysis {
-        &self.result
-    }
-}
-
-impl Dumper {
-    pub fn crate_prelude(&mut self, data: CratePreludeData) {
-        self.result.prelude = Some(data)
-    }
-
-    pub fn compilation_opts(&mut self, data: CompilationOptions) {
-        self.result.compilation = Some(data);
-    }
-
-    pub fn _macro_use(&mut self, data: MacroRef) {
-        if self.config.pub_only || self.config.reachable_only {
-            return;
-        }
-        self.result.macro_refs.push(data);
-    }
-
-    pub fn import(&mut self, access: &Access, import: Import) {
-        if !access.public && self.config.pub_only || !access.reachable && self.config.reachable_only
-        {
-            return;
-        }
-        self.result.imports.push(import);
-    }
-
-    pub fn dump_ref(&mut self, data: Ref) {
-        if self.config.pub_only || self.config.reachable_only {
-            return;
-        }
-        self.result.refs.push(data);
-    }
-
-    pub fn dump_def(&mut self, access: &Access, mut data: Def) {
-        if !access.public && self.config.pub_only || !access.reachable && self.config.reachable_only
-        {
-            return;
-        }
-        if data.kind == DefKind::Mod && data.span.file_name.to_str().unwrap() != data.value {
-            // If the module is an out-of-line definition, then we'll make the
-            // definition the first character in the module's file and turn
-            // the declaration into a reference to it.
-            let rf = Ref { kind: RefKind::Mod, span: data.span, ref_id: data.id };
-            self.result.refs.push(rf);
-            data.span = rls_data::SpanData {
-                file_name: data.value.clone().into(),
-                byte_start: 0,
-                byte_end: 0,
-                line_start: Row::new_one_indexed(1),
-                line_end: Row::new_one_indexed(1),
-                column_start: Column::new_one_indexed(1),
-                column_end: Column::new_one_indexed(1),
-            }
-        }
-        self.result.defs.push(data);
-    }
-
-    pub fn dump_relation(&mut self, data: Relation) {
-        self.result.relations.push(data);
-    }
-
-    pub fn dump_impl(&mut self, data: Impl) {
-        self.result.impls.push(data);
-    }
-}
diff --git a/compiler/rustc_save_analysis/src/errors.rs b/compiler/rustc_save_analysis/src/errors.rs
deleted file mode 100644
index 585aac8c1..000000000
--- a/compiler/rustc_save_analysis/src/errors.rs
+++ /dev/null
@@ -1,10 +0,0 @@
-use rustc_macros::Diagnostic;
-
-use std::path::Path;
-
-#[derive(Diagnostic)]
-#[diag(save_analysis_could_not_open)]
-pub(crate) struct CouldNotOpen<'a> {
-    pub file_name: &'a Path,
-    pub err: std::io::Error,
-}
diff --git a/compiler/rustc_save_analysis/src/lib.rs b/compiler/rustc_save_analysis/src/lib.rs
deleted file mode 100644
index a8d82de02..000000000
--- a/compiler/rustc_save_analysis/src/lib.rs
+++ /dev/null
@@ -1,1072 +0,0 @@
-#![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![feature(if_let_guard)]
-#![recursion_limit = "256"]
-#![allow(rustc::potential_query_instability)]
-#![feature(never_type)]
-#![deny(rustc::untranslatable_diagnostic)]
-#![deny(rustc::diagnostic_outside_of_impl)]
-
-#[macro_use]
-extern crate tracing;
-
-mod dump_visitor;
-mod dumper;
-#[macro_use]
-mod span_utils;
-mod errors;
-mod sig;
-
-use rustc_ast as ast;
-use rustc_ast::util::comments::beautify_doc_string;
-use rustc_ast_pretty::pprust::attribute_to_string;
-use rustc_hir as hir;
-use rustc_hir::def::{DefKind as HirDefKind, Res};
-use rustc_hir::def_id::{DefId, LOCAL_CRATE};
-use rustc_hir::intravisit::{self, Visitor};
-use rustc_hir::Node;
-use rustc_hir_pretty::{enum_def_to_string, fn_to_string, ty_to_string};
-use rustc_middle::hir::nested_filter;
-use rustc_middle::middle::privacy::EffectiveVisibilities;
-use rustc_middle::ty::{self, print::with_no_trimmed_paths, DefIdTree, TyCtxt};
-use rustc_middle::{bug, span_bug};
-use rustc_session::config::{CrateType, Input, OutputType};
-use rustc_session::cstore::ExternCrate;
-use rustc_session::output::{filename_for_metadata, out_filename};
-use rustc_span::symbol::Ident;
-use rustc_span::*;
-
-use std::cell::Cell;
-use std::env;
-use std::fs::File;
-use std::io::BufWriter;
-use std::path::{Path, PathBuf};
-
-use dump_visitor::DumpVisitor;
-use span_utils::SpanUtils;
-
-use rls_data::config::Config;
-use rls_data::{
-    Analysis, Def, DefKind, ExternalCrateData, GlobalCrateId, Impl, ImplKind, MacroRef, Ref,
-    RefKind, Relation, RelationKind, SpanData,
-};
-
-pub struct SaveContext<'tcx> {
-    tcx: TyCtxt<'tcx>,
-    maybe_typeck_results: Option<&'tcx ty::TypeckResults<'tcx>>,
-    effective_visibilities: &'tcx EffectiveVisibilities,
-    span_utils: SpanUtils<'tcx>,
-    config: Config,
-    impl_counter: Cell<u32>,
-}
-
-#[derive(Debug)]
-pub enum Data {
-    RefData(Ref),
-    DefData(Def),
-    RelationData(Relation, Impl),
-}
-
-impl<'tcx> SaveContext<'tcx> {
-    /// Gets the type-checking results for the current body.
-    /// As this will ICE if called outside bodies, only call when working with
-    /// `Expr` or `Pat` nodes (they are guaranteed to be found only in bodies).
-    #[track_caller]
-    fn typeck_results(&self) -> &'tcx ty::TypeckResults<'tcx> {
-        self.maybe_typeck_results.expect("`SaveContext::typeck_results` called outside of body")
-    }
-
-    fn span_from_span(&self, span: Span) -> SpanData {
-        use rls_span::{Column, Row};
-
-        let sm = self.tcx.sess.source_map();
-        let start = sm.lookup_char_pos(span.lo());
-        let end = sm.lookup_char_pos(span.hi());
-
-        SpanData {
-            file_name: start.file.name.prefer_remapped().to_string().into(),
-            byte_start: span.lo().0,
-            byte_end: span.hi().0,
-            line_start: Row::new_one_indexed(start.line as u32),
-            line_end: Row::new_one_indexed(end.line as u32),
-            column_start: Column::new_one_indexed(start.col.0 as u32 + 1),
-            column_end: Column::new_one_indexed(end.col.0 as u32 + 1),
-        }
-    }
-
-    /// Returns path to the compilation output (e.g., libfoo-12345678.rmeta)
-    pub fn compilation_output(&self, crate_name: Symbol) -> PathBuf {
-        let sess = &self.tcx.sess;
-        // Save-analysis is emitted per whole session, not per each crate type
-        let crate_type = sess.crate_types()[0];
-        let outputs = &*self.tcx.output_filenames(());
-
-        if outputs.outputs.contains_key(&OutputType::Metadata) {
-            filename_for_metadata(sess, crate_name, outputs)
-        } else if outputs.outputs.should_codegen() {
-            out_filename(sess, crate_type, outputs, crate_name)
-        } else {
-            // Otherwise it's only a DepInfo, in which case we return early and
-            // not even reach the analysis stage.
-            unreachable!()
-        }
-    }
-
-    /// List external crates used by the current crate.
-    pub fn get_external_crates(&self) -> Vec<ExternalCrateData> {
-        let mut result = Vec::with_capacity(self.tcx.crates(()).len());
-
-        for &n in self.tcx.crates(()).iter() {
-            let Some(&ExternCrate { span, .. }) = self.tcx.extern_crate(n.as_def_id()) else {
-                debug!("skipping crate {}, no data", n);
-                continue;
-            };
-            let lo_loc = self.span_utils.sess.source_map().lookup_char_pos(span.lo());
-            result.push(ExternalCrateData {
-                // FIXME: change file_name field to PathBuf in rls-data
-                // https://github.com/nrc/rls-data/issues/7
-                file_name: self.span_utils.make_filename_string(&lo_loc.file),
-                num: n.as_u32(),
-                id: GlobalCrateId {
-                    name: self.tcx.crate_name(n).to_string(),
-                    disambiguator: (
-                        self.tcx.def_path_hash(n.as_def_id()).stable_crate_id().to_u64(),
-                        0,
-                    ),
-                },
-            });
-        }
-
-        result
-    }
-
-    pub fn get_extern_item_data(&self, item: &hir::ForeignItem<'_>) -> Option<Data> {
-        let def_id = item.owner_id.to_def_id();
-        let qualname = format!("::{}", self.tcx.def_path_str(def_id));
-        let attrs = self.tcx.hir().attrs(item.hir_id());
-        match item.kind {
-            hir::ForeignItemKind::Fn(ref decl, arg_names, ref generics) => {
-                filter!(self.span_utils, item.ident.span);
-
-                Some(Data::DefData(Def {
-                    kind: DefKind::ForeignFunction,
-                    id: id_from_def_id(def_id),
-                    span: self.span_from_span(item.ident.span),
-                    name: item.ident.to_string(),
-                    qualname,
-                    value: fn_to_string(
-                        decl,
-                        hir::FnHeader {
-                            // functions in extern block are implicitly unsafe
-                            unsafety: hir::Unsafety::Unsafe,
-                            // functions in extern block cannot be const
-                            constness: hir::Constness::NotConst,
-                            abi: self.tcx.hir().get_foreign_abi(item.hir_id()),
-                            // functions in extern block cannot be async
-                            asyncness: hir::IsAsync::NotAsync,
-                        },
-                        Some(item.ident.name),
-                        generics,
-                        arg_names,
-                        None,
-                    ),
-                    parent: None,
-                    children: vec![],
-                    decl_id: None,
-                    docs: self.docs_for_attrs(attrs),
-                    sig: sig::foreign_item_signature(item, self),
-                    attributes: lower_attributes(attrs.to_vec(), self),
-                }))
-            }
-            hir::ForeignItemKind::Static(ref ty, _) => {
-                filter!(self.span_utils, item.ident.span);
-
-                let id = id_from_def_id(def_id);
-                let span = self.span_from_span(item.ident.span);
-
-                Some(Data::DefData(Def {
-                    kind: DefKind::ForeignStatic,
-                    id,
-                    span,
-                    name: item.ident.to_string(),
-                    qualname,
-                    value: ty_to_string(ty),
-                    parent: None,
-                    children: vec![],
-                    decl_id: None,
-                    docs: self.docs_for_attrs(attrs),
-                    sig: sig::foreign_item_signature(item, self),
-                    attributes: lower_attributes(attrs.to_vec(), self),
-                }))
-            }
-            // FIXME(plietar): needs a new DefKind in rls-data
-            hir::ForeignItemKind::Type => None,
-        }
-    }
-
-    pub fn get_item_data(&self, item: &hir::Item<'_>) -> Option<Data> {
-        let def_id = item.owner_id.to_def_id();
-        let attrs = self.tcx.hir().attrs(item.hir_id());
-        match item.kind {
-            hir::ItemKind::Fn(ref sig, ref generics, _) => {
-                let qualname = format!("::{}", self.tcx.def_path_str(def_id));
-                filter!(self.span_utils, item.ident.span);
-                Some(Data::DefData(Def {
-                    kind: DefKind::Function,
-                    id: id_from_def_id(def_id),
-                    span: self.span_from_span(item.ident.span),
-                    name: item.ident.to_string(),
-                    qualname,
-                    value: fn_to_string(
-                        sig.decl,
-                        sig.header,
-                        Some(item.ident.name),
-                        generics,
-                        &[],
-                        None,
-                    ),
-                    parent: None,
-                    children: vec![],
-                    decl_id: None,
-                    docs: self.docs_for_attrs(attrs),
-                    sig: sig::item_signature(item, self),
-                    attributes: lower_attributes(attrs.to_vec(), self),
-                }))
-            }
-            hir::ItemKind::Static(ref typ, ..) => {
-                let qualname = format!("::{}", self.tcx.def_path_str(def_id));
-
-                filter!(self.span_utils, item.ident.span);
-
-                let id = id_from_def_id(def_id);
-                let span = self.span_from_span(item.ident.span);
-
-                Some(Data::DefData(Def {
-                    kind: DefKind::Static,
-                    id,
-                    span,
-                    name: item.ident.to_string(),
-                    qualname,
-                    value: ty_to_string(&typ),
-                    parent: None,
-                    children: vec![],
-                    decl_id: None,
-                    docs: self.docs_for_attrs(attrs),
-                    sig: sig::item_signature(item, self),
-                    attributes: lower_attributes(attrs.to_vec(), self),
-                }))
-            }
-            hir::ItemKind::Const(ref typ, _) => {
-                let qualname = format!("::{}", self.tcx.def_path_str(def_id));
-                filter!(self.span_utils, item.ident.span);
-
-                let id = id_from_def_id(def_id);
-                let span = self.span_from_span(item.ident.span);
-
-                Some(Data::DefData(Def {
-                    kind: DefKind::Const,
-                    id,
-                    span,
-                    name: item.ident.to_string(),
-                    qualname,
-                    value: ty_to_string(typ),
-                    parent: None,
-                    children: vec![],
-                    decl_id: None,
-                    docs: self.docs_for_attrs(attrs),
-                    sig: sig::item_signature(item, self),
-                    attributes: lower_attributes(attrs.to_vec(), self),
-                }))
-            }
-            hir::ItemKind::Mod(ref m) => {
-                let qualname = format!("::{}", self.tcx.def_path_str(def_id));
-
-                let sm = self.tcx.sess.source_map();
-                let filename = sm.span_to_filename(m.spans.inner_span);
-
-                filter!(self.span_utils, item.ident.span);
-
-                Some(Data::DefData(Def {
-                    kind: DefKind::Mod,
-                    id: id_from_def_id(def_id),
-                    name: item.ident.to_string(),
-                    qualname,
-                    span: self.span_from_span(item.ident.span),
-                    value: filename.prefer_remapped().to_string(),
-                    parent: None,
-                    children: m
-                        .item_ids
-                        .iter()
-                        .map(|i| id_from_def_id(i.owner_id.to_def_id()))
-                        .collect(),
-                    decl_id: None,
-                    docs: self.docs_for_attrs(attrs),
-                    sig: sig::item_signature(item, self),
-                    attributes: lower_attributes(attrs.to_vec(), self),
-                }))
-            }
-            hir::ItemKind::Enum(ref def, ref generics) => {
-                let name = item.ident.to_string();
-                let qualname = format!("::{}", self.tcx.def_path_str(def_id));
-                filter!(self.span_utils, item.ident.span);
-                let value =
-                    enum_def_to_string(def, generics, item.ident.name, item.span);
-                Some(Data::DefData(Def {
-                    kind: DefKind::Enum,
-                    id: id_from_def_id(def_id),
-                    span: self.span_from_span(item.ident.span),
-                    name,
-                    qualname,
-                    value,
-                    parent: None,
-                    children: def.variants.iter().map(|v| id_from_hir_id(v.hir_id, self)).collect(),
-                    decl_id: None,
-                    docs: self.docs_for_attrs(attrs),
-                    sig: sig::item_signature(item, self),
-                    attributes: lower_attributes(attrs.to_vec(), self),
-                }))
-            }
-            hir::ItemKind::Impl(hir::Impl { ref of_trait, ref self_ty, ref items, .. })
-                if let hir::TyKind::Path(hir::QPath::Resolved(_, ref path)) = self_ty.kind =>
-            {
-                // Common case impl for a struct or something basic.
-                if generated_code(path.span) {
-                    return None;
-                }
-                let sub_span = path.segments.last().unwrap().ident.span;
-                filter!(self.span_utils, sub_span);
-
-                let impl_id = self.next_impl_id();
-                let span = self.span_from_span(sub_span);
-
-                let type_data = self.lookup_def_id(self_ty.hir_id);
-                type_data.map(|type_data| {
-                    Data::RelationData(
-                        Relation {
-                            kind: RelationKind::Impl { id: impl_id },
-                            span: span.clone(),
-                            from: id_from_def_id(type_data),
-                            to: of_trait
-                                .as_ref()
-                                .and_then(|t| self.lookup_def_id(t.hir_ref_id))
-                                .map(id_from_def_id)
-                                .unwrap_or_else(null_id),
-                        },
-                        Impl {
-                            id: impl_id,
-                            kind: match *of_trait {
-                                Some(_) => ImplKind::Direct,
-                                None => ImplKind::Inherent,
-                            },
-                            span,
-                            value: String::new(),
-                            parent: None,
-                            children: items
-                                .iter()
-                                .map(|i| id_from_def_id(i.id.owner_id.to_def_id()))
-                                .collect(),
-                            docs: String::new(),
-                            sig: None,
-                            attributes: vec![],
-                        },
-                    )
-                })
-            }
-            hir::ItemKind::Impl(_) => None,
-            _ => {
-                // FIXME
-                bug!();
-            }
-        }
-    }
-
-    pub fn get_field_data(&self, field: &hir::FieldDef<'_>, scope: hir::HirId) -> Option<Def> {
-        let name = field.ident.to_string();
-        let scope_def_id = self.tcx.hir().local_def_id(scope).to_def_id();
-        let qualname = format!("::{}::{}", self.tcx.def_path_str(scope_def_id), field.ident);
-        filter!(self.span_utils, field.ident.span);
-        let field_def_id = self.tcx.hir().local_def_id(field.hir_id).to_def_id();
-        let typ = self.tcx.type_of(field_def_id).to_string();
-
-        let id = id_from_def_id(field_def_id);
-        let span = self.span_from_span(field.ident.span);
-        let attrs = self.tcx.hir().attrs(field.hir_id);
-
-        Some(Def {
-            kind: DefKind::Field,
-            id,
-            span,
-            name,
-            qualname,
-            value: typ,
-            parent: Some(id_from_def_id(scope_def_id)),
-            children: vec![],
-            decl_id: None,
-            docs: self.docs_for_attrs(attrs),
-            sig: sig::field_signature(field, self),
-            attributes: lower_attributes(attrs.to_vec(), self),
-        })
-    }
-
-    // FIXME would be nice to take a MethodItem here, but the ast provides both
-    // trait and impl flavours, so the caller must do the disassembly.
-    pub fn get_method_data(&self, hir_id: hir::HirId, ident: Ident, span: Span) -> Option<Def> {
-        // The qualname for a method is the trait name or name of the struct in an impl in
-        // which the method is declared in, followed by the method's name.
-        let def_id = self.tcx.hir().local_def_id(hir_id).to_def_id();
-        let (qualname, parent_scope, decl_id, docs, attributes) =
-            match self.tcx.impl_of_method(def_id) {
-                Some(impl_id) => match self.tcx.hir().get_if_local(impl_id) {
-                    Some(Node::Item(item)) => match item.kind {
-                        hir::ItemKind::Impl(hir::Impl { ref self_ty, .. }) => {
-                            let hir = self.tcx.hir();
-
-                            let mut qualname = String::from("<");
-                            qualname
-                                .push_str(&rustc_hir_pretty::id_to_string(&hir, self_ty.hir_id));
-
-                            let trait_id = self.tcx.trait_id_of_impl(impl_id);
-                            let mut docs = String::new();
-                            let mut attrs = vec![];
-                            if let Some(Node::ImplItem(_)) = hir.find(hir_id) {
-                                attrs = self.tcx.hir().attrs(hir_id).to_vec();
-                                docs = self.docs_for_attrs(&attrs);
-                            }
-
-                            let mut decl_id = None;
-                            if let Some(def_id) = trait_id {
-                                // A method in a trait impl.
-                                qualname.push_str(" as ");
-                                qualname.push_str(&self.tcx.def_path_str(def_id));
-
-                                decl_id = self
-                                    .tcx
-                                    .associated_items(def_id)
-                                    .filter_by_name_unhygienic(ident.name)
-                                    .next()
-                                    .map(|item| item.def_id);
-                            }
-                            qualname.push('>');
-
-                            (qualname, trait_id, decl_id, docs, attrs)
-                        }
-                        _ => {
-                            span_bug!(
-                                span,
-                                "Container {:?} for method {} not an impl?",
-                                impl_id,
-                                hir_id
-                            );
-                        }
-                    },
-                    r => {
-                        span_bug!(
-                            span,
-                            "Container {:?} for method {} is not a node item {:?}",
-                            impl_id,
-                            hir_id,
-                            r
-                        );
-                    }
-                },
-                None => match self.tcx.trait_of_item(def_id) {
-                    Some(def_id) => {
-                        let mut docs = String::new();
-                        let mut attrs = vec![];
-
-                        if let Some(Node::TraitItem(_)) = self.tcx.hir().find(hir_id) {
-                            attrs = self.tcx.hir().attrs(hir_id).to_vec();
-                            docs = self.docs_for_attrs(&attrs);
-                        }
-
-                        (
-                            format!("::{}", self.tcx.def_path_str(def_id)),
-                            Some(def_id),
-                            None,
-                            docs,
-                            attrs,
-                        )
-                    }
-                    None => {
-                        debug!("could not find container for method {} at {:?}", hir_id, span);
-                        // This is not necessarily a bug, if there was a compilation error,
-                        // the typeck results we need might not exist.
-                        return None;
-                    }
-                },
-            };
-
-        let qualname = format!("{}::{}", qualname, ident.name);
-
-        filter!(self.span_utils, ident.span);
-
-        Some(Def {
-            kind: DefKind::Method,
-            id: id_from_def_id(def_id),
-            span: self.span_from_span(ident.span),
-            name: ident.name.to_string(),
-            qualname,
-            // FIXME you get better data here by using the visitor.
-            value: String::new(),
-            parent: parent_scope.map(id_from_def_id),
-            children: vec![],
-            decl_id: decl_id.map(id_from_def_id),
-            docs,
-            sig: None,
-            attributes: lower_attributes(attributes, self),
-        })
-    }
-
-    pub fn get_expr_data(&self, expr: &hir::Expr<'_>) -> Option<Data> {
-        let ty = self.typeck_results().expr_ty_adjusted_opt(expr)?;
-        if matches!(ty.kind(), ty::Error(_)) {
-            return None;
-        }
-        match expr.kind {
-            hir::ExprKind::Field(ref sub_ex, ident) => {
-                match self.typeck_results().expr_ty_adjusted(&sub_ex).kind() {
-                    ty::Adt(def, _) if !def.is_enum() => {
-                        let variant = &def.non_enum_variant();
-                        filter!(self.span_utils, ident.span);
-                        let span = self.span_from_span(ident.span);
-                        Some(Data::RefData(Ref {
-                            kind: RefKind::Variable,
-                            span,
-                            ref_id: self
-                                .tcx
-                                .find_field_index(ident, variant)
-                                .map(|index| id_from_def_id(variant.fields[index].did))
-                                .unwrap_or_else(null_id),
-                        }))
-                    }
-                    ty::Tuple(..) => None,
-                    _ => {
-                        debug!("expected struct or union type, found {:?}", ty);
-                        None
-                    }
-                }
-            }
-            hir::ExprKind::Struct(qpath, ..) => match ty.kind() {
-                ty::Adt(def, _) => {
-                    let sub_span = qpath.last_segment_span();
-                    filter!(self.span_utils, sub_span);
-                    let span = self.span_from_span(sub_span);
-                    Some(Data::RefData(Ref {
-                        kind: RefKind::Type,
-                        span,
-                        ref_id: id_from_def_id(def.did()),
-                    }))
-                }
-                _ => {
-                    debug!("expected adt, found {:?}", ty);
-                    None
-                }
-            },
-            hir::ExprKind::MethodCall(ref seg, ..) => {
-                let Some(method_id) = self.typeck_results().type_dependent_def_id(expr.hir_id) else {
-                    debug!("could not resolve method id for {:?}", expr);
-                    return None;
-                };
-                let (def_id, decl_id) = match self.tcx.associated_item(method_id).container {
-                    ty::ImplContainer => (Some(method_id), None),
-                    ty::TraitContainer => (None, Some(method_id)),
-                };
-                let sub_span = seg.ident.span;
-                filter!(self.span_utils, sub_span);
-                let span = self.span_from_span(sub_span);
-                Some(Data::RefData(Ref {
-                    kind: RefKind::Function,
-                    span,
-                    ref_id: def_id.or(decl_id).map(id_from_def_id).unwrap_or_else(null_id),
-                }))
-            }
-            hir::ExprKind::Path(ref path) => {
-                self.get_path_data(expr.hir_id, path).map(Data::RefData)
-            }
-            _ => {
-                // FIXME
-                bug!("invalid expression: {:?}", expr);
-            }
-        }
-    }
-
-    pub fn get_path_res(&self, hir_id: hir::HirId) -> Res {
-        match self.tcx.hir().get(hir_id) {
-            Node::TraitRef(tr) => tr.path.res,
-
-            Node::Item(&hir::Item { kind: hir::ItemKind::Use(path, _), .. }) => {
-                path.res.get(0).copied().unwrap_or(Res::Err)
-            }
-            Node::PathSegment(seg) => {
-                if seg.res != Res::Err {
-                    seg.res
-                } else {
-                    let parent_node = self.tcx.hir().parent_id(hir_id);
-                    self.get_path_res(parent_node)
-                }
-            }
-
-            Node::Expr(&hir::Expr { kind: hir::ExprKind::Struct(ref qpath, ..), .. }) => {
-                self.typeck_results().qpath_res(qpath, hir_id)
-            }
-
-            Node::Expr(&hir::Expr { kind: hir::ExprKind::Path(ref qpath), .. })
-            | Node::Pat(&hir::Pat {
-                kind:
-                    hir::PatKind::Path(ref qpath)
-                    | hir::PatKind::Struct(ref qpath, ..)
-                    | hir::PatKind::TupleStruct(ref qpath, ..),
-                ..
-            })
-            | Node::Ty(&hir::Ty { kind: hir::TyKind::Path(ref qpath), .. }) => match qpath {
-                hir::QPath::Resolved(_, path) => path.res,
-                hir::QPath::TypeRelative(..) | hir::QPath::LangItem(..) => {
-                    // #75962: `self.typeck_results` may be different from the `hir_id`'s result.
-                    if self.tcx.has_typeck_results(hir_id.owner.to_def_id()) {
-                        self.tcx.typeck(hir_id.owner.def_id).qpath_res(qpath, hir_id)
-                    } else {
-                        Res::Err
-                    }
-                }
-            },
-
-            Node::Pat(&hir::Pat { kind: hir::PatKind::Binding(_, canonical_id, ..), .. }) => {
-                Res::Local(canonical_id)
-            }
-
-            _ => Res::Err,
-        }
-    }
-
-    pub fn get_path_data(&self, id: hir::HirId, path: &hir::QPath<'_>) -> Option<Ref> {
-        let segment = match path {
-            hir::QPath::Resolved(_, path) => path.segments.last(),
-            hir::QPath::TypeRelative(_, segment) => Some(*segment),
-            hir::QPath::LangItem(..) => None,
-        };
-        segment.and_then(|seg| {
-            self.get_path_segment_data(seg).or_else(|| self.get_path_segment_data_with_id(seg, id))
-        })
-    }
-
-    pub fn get_path_segment_data(&self, path_seg: &hir::PathSegment<'_>) -> Option<Ref> {
-        self.get_path_segment_data_with_id(path_seg, path_seg.hir_id)
-    }
-
-    pub fn get_path_segment_data_with_id(
-        &self,
-        path_seg: &hir::PathSegment<'_>,
-        id: hir::HirId,
-    ) -> Option<Ref> {
-        // Returns true if the path is function type sugar, e.g., `Fn(A) -> B`.
-        fn fn_type(seg: &hir::PathSegment<'_>) -> bool {
-            seg.args.map_or(false, |args| args.parenthesized)
-        }
-
-        let res = self.get_path_res(id);
-        let span = path_seg.ident.span;
-        filter!(self.span_utils, span);
-        let span = self.span_from_span(span);
-
-        match res {
-            Res::Local(id) => {
-                Some(Ref { kind: RefKind::Variable, span, ref_id: id_from_hir_id(id, self) })
-            }
-            Res::Def(HirDefKind::Trait, def_id) if fn_type(path_seg) => {
-                Some(Ref { kind: RefKind::Type, span, ref_id: id_from_def_id(def_id) })
-            }
-            Res::Def(
-                HirDefKind::Struct
-                | HirDefKind::Variant
-                | HirDefKind::Union
-                | HirDefKind::Enum
-                | HirDefKind::TyAlias
-                | HirDefKind::ForeignTy
-                | HirDefKind::TraitAlias
-                | HirDefKind::AssocTy
-                | HirDefKind::Trait
-                | HirDefKind::OpaqueTy
-                | HirDefKind::ImplTraitPlaceholder
-                | HirDefKind::TyParam,
-                def_id,
-            ) => Some(Ref { kind: RefKind::Type, span, ref_id: id_from_def_id(def_id) }),
-            Res::Def(HirDefKind::ConstParam, def_id) => {
-                Some(Ref { kind: RefKind::Variable, span, ref_id: id_from_def_id(def_id) })
-            }
-            Res::Def(HirDefKind::Ctor(..), def_id) => {
-                // This is a reference to a tuple struct or an enum variant where the def_id points
-                // to an invisible constructor function. That is not a very useful
-                // def, so adjust to point to the tuple struct or enum variant itself.
-                let parent_def_id = self.tcx.parent(def_id);
-                Some(Ref { kind: RefKind::Type, span, ref_id: id_from_def_id(parent_def_id) })
-            }
-            Res::Def(HirDefKind::Static(_) | HirDefKind::Const | HirDefKind::AssocConst, _) => {
-                Some(Ref { kind: RefKind::Variable, span, ref_id: id_from_def_id(res.def_id()) })
-            }
-            Res::Def(HirDefKind::AssocFn, decl_id) => {
-                let def_id = if decl_id.is_local() {
-                    if self.tcx.impl_defaultness(decl_id).has_value() {
-                        Some(decl_id)
-                    } else {
-                        None
-                    }
-                } else {
-                    None
-                };
-                Some(Ref {
-                    kind: RefKind::Function,
-                    span,
-                    ref_id: id_from_def_id(def_id.unwrap_or(decl_id)),
-                })
-            }
-            Res::Def(HirDefKind::Fn, def_id) => {
-                Some(Ref { kind: RefKind::Function, span, ref_id: id_from_def_id(def_id) })
-            }
-            Res::Def(HirDefKind::Mod, def_id) => {
-                Some(Ref { kind: RefKind::Mod, span, ref_id: id_from_def_id(def_id) })
-            }
-
-            Res::Def(
-                HirDefKind::Macro(..)
-                | HirDefKind::ExternCrate
-                | HirDefKind::ForeignMod
-                | HirDefKind::LifetimeParam
-                | HirDefKind::AnonConst
-                | HirDefKind::InlineConst
-                | HirDefKind::Use
-                | HirDefKind::Field
-                | HirDefKind::GlobalAsm
-                | HirDefKind::Impl
-                | HirDefKind::Closure
-                | HirDefKind::Generator,
-                _,
-            )
-            | Res::PrimTy(..)
-            | Res::SelfTyParam { .. }
-            | Res::SelfTyAlias { .. }
-            | Res::ToolMod
-            | Res::NonMacroAttr(..)
-            | Res::SelfCtor(..)
-            | Res::Err => None,
-        }
-    }
-
-    pub fn get_field_ref_data(
-        &self,
-        field_ref: &hir::ExprField<'_>,
-        variant: &ty::VariantDef,
-    ) -> Option<Ref> {
-        filter!(self.span_utils, field_ref.ident.span);
-        self.tcx.find_field_index(field_ref.ident, variant).map(|index| {
-            let span = self.span_from_span(field_ref.ident.span);
-            Ref { kind: RefKind::Variable, span, ref_id: id_from_def_id(variant.fields[index].did) }
-        })
-    }
-
-    /// Attempt to return MacroRef for any AST node.
-    ///
-    /// For a given piece of AST defined by the supplied Span and NodeId,
-    /// returns `None` if the node is not macro-generated or the span is malformed,
-    /// else uses the expansion callsite and callee to return some MacroRef.
-    ///
-    /// FIXME: [`DumpVisitor::process_macro_use`] should actually dump this data
-    #[allow(dead_code)]
-    fn get_macro_use_data(&self, span: Span) -> Option<MacroRef> {
-        if !generated_code(span) {
-            return None;
-        }
-        // Note we take care to use the source callsite/callee, to handle
-        // nested expansions and ensure we only generate data for source-visible
-        // macro uses.
-        let callsite = span.source_callsite();
-        let callsite_span = self.span_from_span(callsite);
-        let callee = span.source_callee()?;
-
-        let mac_name = match callee.kind {
-            ExpnKind::Macro(kind, name) => match kind {
-                MacroKind::Bang => name,
-
-                // Ignore attribute macros, their spans are usually mangled
-                // FIXME(eddyb) is this really the case anymore?
-                MacroKind::Attr | MacroKind::Derive => return None,
-            },
-
-            // These are not macros.
-            // FIXME(eddyb) maybe there is a way to handle them usefully?
-            ExpnKind::Inlined | ExpnKind::Root | ExpnKind::AstPass(_) | ExpnKind::Desugaring(_) => {
-                return None;
-            }
-        };
-
-        let callee_span = self.span_from_span(callee.def_site);
-        Some(MacroRef {
-            span: callsite_span,
-            qualname: mac_name.to_string(), // FIXME: generate the real qualname
-            callee_span,
-        })
-    }
-
-    fn lookup_def_id(&self, ref_id: hir::HirId) -> Option<DefId> {
-        match self.get_path_res(ref_id) {
-            Res::PrimTy(_) | Res::SelfTyParam { .. } | Res::SelfTyAlias { .. } | Res::Err => None,
-            def => def.opt_def_id(),
-        }
-    }
-
-    fn docs_for_attrs(&self, attrs: &[ast::Attribute]) -> String {
-        let mut result = String::new();
-
-        for attr in attrs {
-            if let Some((val, kind)) = attr.doc_str_and_comment_kind() {
-                // FIXME: Should save-analysis beautify doc strings itself or leave it to users?
-                result.push_str(beautify_doc_string(val, kind).as_str());
-                result.push('\n');
-            }
-        }
-
-        if !self.config.full_docs {
-            if let Some(index) = result.find("\n\n") {
-                result.truncate(index);
-            }
-        }
-
-        result
-    }
-
-    fn next_impl_id(&self) -> u32 {
-        let next = self.impl_counter.get();
-        self.impl_counter.set(next + 1);
-        next
-    }
-}
-
-// An AST visitor for collecting paths (e.g., the names of structs) and formal
-// variables (idents) from patterns.
-struct PathCollector<'l> {
-    tcx: TyCtxt<'l>,
-    collected_paths: Vec<(hir::HirId, &'l hir::QPath<'l>)>,
-    collected_idents: Vec<(hir::HirId, Ident, hir::Mutability)>,
-}
-
-impl<'l> PathCollector<'l> {
-    fn new(tcx: TyCtxt<'l>) -> PathCollector<'l> {
-        PathCollector { tcx, collected_paths: vec![], collected_idents: vec![] }
-    }
-}
-
-impl<'l> Visitor<'l> for PathCollector<'l> {
-    type NestedFilter = nested_filter::All;
-
-    fn nested_visit_map(&mut self) -> Self::Map {
-        self.tcx.hir()
-    }
-
-    fn visit_pat(&mut self, p: &'l hir::Pat<'l>) {
-        match p.kind {
-            hir::PatKind::Struct(ref path, ..) => {
-                self.collected_paths.push((p.hir_id, path));
-            }
-            hir::PatKind::TupleStruct(ref path, ..) | hir::PatKind::Path(ref path) => {
-                self.collected_paths.push((p.hir_id, path));
-            }
-            hir::PatKind::Binding(hir::BindingAnnotation(_, mutbl), _, ident, _) => {
-                debug!(
-                    "PathCollector, visit ident in pat {}: {:?} {:?}",
-                    ident, p.span, ident.span
-                );
-                self.collected_idents.push((p.hir_id, ident, mutbl));
-            }
-            _ => {}
-        }
-        intravisit::walk_pat(self, p);
-    }
-}
-
-/// Defines what to do with the results of saving the analysis.
-pub trait SaveHandler {
-    fn save(&mut self, save_ctxt: &SaveContext<'_>, analysis: &Analysis);
-}
-
-/// Dump the save-analysis results to a file.
-pub struct DumpHandler<'a> {
-    odir: Option<&'a Path>,
-    cratename: String,
-}
-
-impl<'a> DumpHandler<'a> {
-    pub fn new(odir: Option<&'a Path>, cratename: Symbol) -> DumpHandler<'a> {
-        DumpHandler { odir, cratename: cratename.to_string() }
-    }
-
-    fn output_file(&self, ctx: &SaveContext<'_>) -> (BufWriter<File>, PathBuf) {
-        let sess = &ctx.tcx.sess;
-        let file_name = match ctx.config.output_file {
-            Some(ref s) => PathBuf::from(s),
-            None => {
-                let mut root_path = match self.odir {
-                    Some(val) => val.join("save-analysis"),
-                    None => PathBuf::from("save-analysis-temp"),
-                };
-
-                if let Err(e) = std::fs::create_dir_all(&root_path) {
-                    error!("Could not create directory {}: {}", root_path.display(), e);
-                }
-
-                let executable = sess.crate_types().iter().any(|ct| *ct == CrateType::Executable);
-                let mut out_name = if executable { String::new() } else { "lib".to_owned() };
-                out_name.push_str(&self.cratename);
-                out_name.push_str(&sess.opts.cg.extra_filename);
-                out_name.push_str(".json");
-                root_path.push(&out_name);
-
-                root_path
-            }
-        };
-
-        info!("Writing output to {}", file_name.display());
-
-        let output_file = BufWriter::new(File::create(&file_name).unwrap_or_else(|e| {
-            sess.emit_fatal(errors::CouldNotOpen { file_name: file_name.as_path(), err: e })
-        }));
-
-        (output_file, file_name)
-    }
-}
-
-impl SaveHandler for DumpHandler<'_> {
-    fn save(&mut self, save_ctxt: &SaveContext<'_>, analysis: &Analysis) {
-        let sess = &save_ctxt.tcx.sess;
-        let (output, file_name) = self.output_file(&save_ctxt);
-        if let Err(e) = serde_json::to_writer(output, &analysis) {
-            error!("Can't serialize save-analysis: {:?}", e);
-        }
-
-        if sess.opts.json_artifact_notifications {
-            sess.parse_sess.span_diagnostic.emit_artifact_notification(&file_name, "save-analysis");
-        }
-    }
-}
-
-/// Call a callback with the results of save-analysis.
-pub struct CallbackHandler<'b> {
-    pub callback: &'b mut dyn FnMut(&rls_data::Analysis),
-}
-
-impl SaveHandler for CallbackHandler<'_> {
-    fn save(&mut self, _: &SaveContext<'_>, analysis: &Analysis) {
-        (self.callback)(analysis)
-    }
-}
-
-pub fn process_crate<H: SaveHandler>(
-    tcx: TyCtxt<'_>,
-    cratename: Symbol,
-    input: &Input,
-    config: Option<Config>,
-    mut handler: H,
-) {
-    with_no_trimmed_paths!({
-        tcx.dep_graph.with_ignore(|| {
-            info!("Dumping crate {}", cratename);
-
-            // Privacy checking must be done outside of type inference; use a
-            // fallback in case effective visibilities couldn't have been correctly computed.
-            let effective_visibilities = match tcx.sess.compile_status() {
-                Ok(..) => tcx.effective_visibilities(()),
-                Err(..) => tcx.arena.alloc(EffectiveVisibilities::default()),
-            };
-
-            let save_ctxt = SaveContext {
-                tcx,
-                maybe_typeck_results: None,
-                effective_visibilities: &effective_visibilities,
-                span_utils: SpanUtils::new(&tcx.sess),
-                config: find_config(config),
-                impl_counter: Cell::new(0),
-            };
-
-            let mut visitor = DumpVisitor::new(save_ctxt);
-
-            visitor.dump_crate_info(cratename);
-            visitor.dump_compilation_options(input, cratename);
-            visitor.process_crate();
-
-            handler.save(&visitor.save_ctxt, &visitor.analysis())
-        })
-    })
-}
-
-fn find_config(supplied: Option<Config>) -> Config {
-    if let Some(config) = supplied {
-        return config;
-    }
-
-    match env::var_os("RUST_SAVE_ANALYSIS_CONFIG") {
-        None => Config::default(),
-        Some(config) => config
-            .to_str()
-            .ok_or(())
-            .map_err(|_| error!("`RUST_SAVE_ANALYSIS_CONFIG` isn't UTF-8"))
-            .and_then(|cfg| {
-                serde_json::from_str(cfg)
-                    .map_err(|_| error!("Could not deserialize save-analysis config"))
-            })
-            .unwrap_or_default(),
-    }
-}
-
-// Utility functions for the module.
-
-// Helper function to escape quotes in a string
-fn escape(s: String) -> String {
-    s.replace('\"', "\"\"")
-}
-
-// Helper function to determine if a span came from a
-// macro expansion or syntax extension.
-fn generated_code(span: Span) -> bool {
-    span.from_expansion() || span.is_dummy()
-}
-
-// DefId::index is a newtype and so the JSON serialisation is ugly. Therefore
-// we use our own Id which is the same, but without the newtype.
-fn id_from_def_id(id: DefId) -> rls_data::Id {
-    rls_data::Id { krate: id.krate.as_u32(), index: id.index.as_u32() }
-}
-
-fn id_from_hir_id(id: hir::HirId, scx: &SaveContext<'_>) -> rls_data::Id {
-    let def_id = scx.tcx.hir().opt_local_def_id(id);
-    def_id.map(|id| id_from_def_id(id.to_def_id())).unwrap_or_else(|| {
-        // Create a *fake* `DefId` out of a `HirId` by combining the owner
-        // `local_def_index` and the `local_id`.
-        // This will work unless you have *billions* of definitions in a single
-        // crate (very unlikely to actually happen).
-        rls_data::Id {
-            krate: LOCAL_CRATE.as_u32(),
-            index: id.owner.def_id.local_def_index.as_u32() | id.local_id.as_u32().reverse_bits(),
-        }
-    })
-}
-
-fn null_id() -> rls_data::Id {
-    rls_data::Id { krate: u32::MAX, index: u32::MAX }
-}
-
-fn lower_attributes(attrs: Vec<ast::Attribute>, scx: &SaveContext<'_>) -> Vec<rls_data::Attribute> {
-    attrs
-        .into_iter()
-        // Only retain real attributes. Doc comments are lowered separately.
-        .filter(|attr| !attr.has_name(sym::doc))
-        .map(|mut attr| {
-            // Remove the surrounding '#[..]' or '#![..]' of the pretty printed
-            // attribute. First normalize all inner attribute (#![..]) to outer
-            // ones (#[..]), then remove the two leading and the one trailing character.
-            attr.style = ast::AttrStyle::Outer;
-            let value = attribute_to_string(&attr);
-            // This str slicing works correctly, because the leading and trailing characters
-            // are in the ASCII range and thus exactly one byte each.
-            let value = value[2..value.len() - 1].to_string();
-
-            rls_data::Attribute { value, span: scx.span_from_span(attr.span) }
-        })
-        .collect()
-}
diff --git a/compiler/rustc_save_analysis/src/sig.rs b/compiler/rustc_save_analysis/src/sig.rs
deleted file mode 100644
index 5a1bcb8fd..000000000
--- a/compiler/rustc_save_analysis/src/sig.rs
+++ /dev/null
@@ -1,923 +0,0 @@
-// A signature is a string representation of an item's type signature, excluding
-// any body. It also includes ids for any defs or refs in the signature. For
-// example:
-//
-// ```
-// fn foo(x: String) {
-//     println!("{}", x);
-// }
-// ```
-// The signature string is something like "fn foo(x: String) {}" and the signature
-// will have defs for `foo` and `x` and a ref for `String`.
-//
-// All signature text should parse in the correct context (i.e., in a module or
-// impl, etc.). Clients may want to trim trailing `{}` or `;`. The text of a
-// signature is not guaranteed to be stable (it may improve or change as the
-// syntax changes, or whitespace or punctuation may change). It is also likely
-// not to be pretty - no attempt is made to prettify the text. It is recommended
-// that clients run the text through Rustfmt.
-//
-// This module generates Signatures for items by walking the AST and looking up
-// references.
-//
-// Signatures do not include visibility info. I'm not sure if this is a feature
-// or an omission (FIXME).
-//
-// FIXME where clauses need implementing, defs/refs in generics are mostly missing.
-
-use crate::{id_from_def_id, id_from_hir_id, SaveContext};
-
-use rls_data::{SigElement, Signature};
-
-use rustc_ast::Mutability;
-use rustc_hir as hir;
-use rustc_hir::def::{DefKind, Res};
-use rustc_hir_pretty::id_to_string;
-use rustc_hir_pretty::{bounds_to_string, path_segment_to_string, path_to_string, ty_to_string};
-use rustc_span::symbol::{Ident, Symbol};
-
-pub fn item_signature(item: &hir::Item<'_>, scx: &SaveContext<'_>) -> Option<Signature> {
-    if !scx.config.signatures {
-        return None;
-    }
-    item.make(0, None, scx).ok()
-}
-
-pub fn foreign_item_signature(
-    item: &hir::ForeignItem<'_>,
-    scx: &SaveContext<'_>,
-) -> Option<Signature> {
-    if !scx.config.signatures {
-        return None;
-    }
-    item.make(0, None, scx).ok()
-}
-
-/// Signature for a struct or tuple field declaration.
-/// Does not include a trailing comma.
-pub fn field_signature(field: &hir::FieldDef<'_>, scx: &SaveContext<'_>) -> Option<Signature> {
-    if !scx.config.signatures {
-        return None;
-    }
-    field.make(0, None, scx).ok()
-}
-
-/// Does not include a trailing comma.
-pub fn variant_signature(variant: &hir::Variant<'_>, scx: &SaveContext<'_>) -> Option<Signature> {
-    if !scx.config.signatures {
-        return None;
-    }
-    variant.make(0, None, scx).ok()
-}
-
-pub fn method_signature(
-    id: hir::HirId,
-    ident: Ident,
-    generics: &hir::Generics<'_>,
-    m: &hir::FnSig<'_>,
-    scx: &SaveContext<'_>,
-) -> Option<Signature> {
-    if !scx.config.signatures {
-        return None;
-    }
-    make_method_signature(id, ident, generics, m, scx).ok()
-}
-
-pub fn assoc_const_signature(
-    id: hir::HirId,
-    ident: Symbol,
-    ty: &hir::Ty<'_>,
-    default: Option<&hir::Expr<'_>>,
-    scx: &SaveContext<'_>,
-) -> Option<Signature> {
-    if !scx.config.signatures {
-        return None;
-    }
-    make_assoc_const_signature(id, ident, ty, default, scx).ok()
-}
-
-pub fn assoc_type_signature(
-    id: hir::HirId,
-    ident: Ident,
-    bounds: Option<hir::GenericBounds<'_>>,
-    default: Option<&hir::Ty<'_>>,
-    scx: &SaveContext<'_>,
-) -> Option<Signature> {
-    if !scx.config.signatures {
-        return None;
-    }
-    make_assoc_type_signature(id, ident, bounds, default, scx).ok()
-}
-
-type Result = std::result::Result<Signature, &'static str>;
-
-trait Sig {
-    fn make(&self, offset: usize, id: Option<hir::HirId>, scx: &SaveContext<'_>) -> Result;
-}
-
-fn extend_sig(
-    mut sig: Signature,
-    text: String,
-    defs: Vec<SigElement>,
-    refs: Vec<SigElement>,
-) -> Signature {
-    sig.text = text;
-    sig.defs.extend(defs.into_iter());
-    sig.refs.extend(refs.into_iter());
-    sig
-}
-
-fn replace_text(mut sig: Signature, text: String) -> Signature {
-    sig.text = text;
-    sig
-}
-
-fn merge_sigs(text: String, sigs: Vec<Signature>) -> Signature {
-    let mut result = Signature { text, defs: vec![], refs: vec![] };
-
-    let (defs, refs): (Vec<_>, Vec<_>) = sigs.into_iter().map(|s| (s.defs, s.refs)).unzip();
-
-    result.defs.extend(defs.into_iter().flat_map(|ds| ds.into_iter()));
-    result.refs.extend(refs.into_iter().flat_map(|rs| rs.into_iter()));
-
-    result
-}
-
-fn text_sig(text: String) -> Signature {
-    Signature { text, defs: vec![], refs: vec![] }
-}
-
-impl<'hir> Sig for hir::Ty<'hir> {
-    fn make(&self, offset: usize, _parent_id: Option<hir::HirId>, scx: &SaveContext<'_>) -> Result {
-        let id = Some(self.hir_id);
-        match self.kind {
-            hir::TyKind::Slice(ref ty) => {
-                let nested = ty.make(offset + 1, id, scx)?;
-                let text = format!("[{}]", nested.text);
-                Ok(replace_text(nested, text))
-            }
-            hir::TyKind::Ptr(ref mt) => {
-                let prefix = match mt.mutbl {
-                    hir::Mutability::Mut => "*mut ",
-                    hir::Mutability::Not => "*const ",
-                };
-                let nested = mt.ty.make(offset + prefix.len(), id, scx)?;
-                let text = format!("{}{}", prefix, nested.text);
-                Ok(replace_text(nested, text))
-            }
-            hir::TyKind::Ref(ref lifetime, ref mt) => {
-                let mut prefix = "&".to_owned();
-                prefix.push_str(&lifetime.ident.to_string());
-                prefix.push(' ');
-                if mt.mutbl.is_mut() {
-                    prefix.push_str("mut ");
-                };
-
-                let nested = mt.ty.make(offset + prefix.len(), id, scx)?;
-                let text = format!("{}{}", prefix, nested.text);
-                Ok(replace_text(nested, text))
-            }
-            hir::TyKind::Never => Ok(text_sig("!".to_owned())),
-            hir::TyKind::Tup(ts) => {
-                let mut text = "(".to_owned();
-                let mut defs = vec![];
-                let mut refs = vec![];
-                for t in ts {
-                    let nested = t.make(offset + text.len(), id, scx)?;
-                    text.push_str(&nested.text);
-                    text.push(',');
-                    defs.extend(nested.defs.into_iter());
-                    refs.extend(nested.refs.into_iter());
-                }
-                text.push(')');
-                Ok(Signature { text, defs, refs })
-            }
-            hir::TyKind::BareFn(ref f) => {
-                let mut text = String::new();
-                if !f.generic_params.is_empty() {
-                    // FIXME defs, bounds on lifetimes
-                    text.push_str("for<");
-                    text.push_str(
-                        &f.generic_params
-                            .iter()
-                            .filter_map(|param| match param.kind {
-                                hir::GenericParamKind::Lifetime { .. } => {
-                                    Some(param.name.ident().to_string())
-                                }
-                                _ => None,
-                            })
-                            .collect::<Vec<_>>()
-                            .join(", "),
-                    );
-                    text.push('>');
-                }
-
-                if let hir::Unsafety::Unsafe = f.unsafety {
-                    text.push_str("unsafe ");
-                }
-                text.push_str("fn(");
-
-                let mut defs = vec![];
-                let mut refs = vec![];
-                for i in f.decl.inputs {
-                    let nested = i.make(offset + text.len(), Some(i.hir_id), scx)?;
-                    text.push_str(&nested.text);
-                    text.push(',');
-                    defs.extend(nested.defs.into_iter());
-                    refs.extend(nested.refs.into_iter());
-                }
-                text.push(')');
-                if let hir::FnRetTy::Return(ref t) = f.decl.output {
-                    text.push_str(" -> ");
-                    let nested = t.make(offset + text.len(), None, scx)?;
-                    text.push_str(&nested.text);
-                    text.push(',');
-                    defs.extend(nested.defs.into_iter());
-                    refs.extend(nested.refs.into_iter());
-                }
-
-                Ok(Signature { text, defs, refs })
-            }
-            hir::TyKind::Path(hir::QPath::Resolved(None, ref path)) => path.make(offset, id, scx),
-            hir::TyKind::Path(hir::QPath::Resolved(Some(ref qself), ref path)) => {
-                let nested_ty = qself.make(offset + 1, id, scx)?;
-                let prefix = format!(
-                    "<{} as {}>::",
-                    nested_ty.text,
-                    path_segment_to_string(&path.segments[0])
-                );
-
-                let name = path_segment_to_string(path.segments.last().ok_or("Bad path")?);
-                let res = scx.get_path_res(id.ok_or("Missing id for Path")?);
-                let id = id_from_def_id(res.def_id());
-                if path.segments.len() == 2 {
-                    let start = offset + prefix.len();
-                    let end = start + name.len();
-
-                    Ok(Signature {
-                        text: prefix + &name,
-                        defs: vec![],
-                        refs: vec![SigElement { id, start, end }],
-                    })
-                } else {
-                    let start = offset + prefix.len() + 5;
-                    let end = start + name.len();
-                    // FIXME should put the proper path in there, not ellipsis.
-                    Ok(Signature {
-                        text: prefix + "...::" + &name,
-                        defs: vec![],
-                        refs: vec![SigElement { id, start, end }],
-                    })
-                }
-            }
-            hir::TyKind::Path(hir::QPath::TypeRelative(ty, segment)) => {
-                let nested_ty = ty.make(offset + 1, id, scx)?;
-                let prefix = format!("<{}>::", nested_ty.text);
-
-                let name = path_segment_to_string(segment);
-                let res = scx.get_path_res(id.ok_or("Missing id for Path")?);
-                let id = id_from_def_id(res.def_id());
-
-                let start = offset + prefix.len();
-                let end = start + name.len();
-                Ok(Signature {
-                    text: prefix + &name,
-                    defs: vec![],
-                    refs: vec![SigElement { id, start, end }],
-                })
-            }
-            hir::TyKind::Path(hir::QPath::LangItem(lang_item, _, _)) => {
-                Ok(text_sig(format!("#[lang = \"{}\"]", lang_item.name())))
-            }
-            hir::TyKind::TraitObject(bounds, ..) => {
-                // FIXME recurse into bounds
-                let bounds: Vec<hir::GenericBound<'_>> = bounds
-                    .iter()
-                    .map(|hir::PolyTraitRef { bound_generic_params, trait_ref, span }| {
-                        hir::GenericBound::Trait(
-                            hir::PolyTraitRef {
-                                bound_generic_params,
-                                trait_ref: hir::TraitRef {
-                                    path: trait_ref.path,
-                                    hir_ref_id: trait_ref.hir_ref_id,
-                                },
-                                span: *span,
-                            },
-                            hir::TraitBoundModifier::None,
-                        )
-                    })
-                    .collect();
-                let nested = bounds_to_string(&bounds);
-                Ok(text_sig(nested))
-            }
-            hir::TyKind::Array(ref ty, ref length) => {
-                let nested_ty = ty.make(offset + 1, id, scx)?;
-                let expr = id_to_string(&scx.tcx.hir(), length.hir_id()).replace('\n', " ");
-                let text = format!("[{}; {}]", nested_ty.text, expr);
-                Ok(replace_text(nested_ty, text))
-            }
-            hir::TyKind::OpaqueDef(item_id, _, _) => {
-                let item = scx.tcx.hir().item(item_id);
-                item.make(offset, Some(item_id.hir_id()), scx)
-            }
-            hir::TyKind::Typeof(_) | hir::TyKind::Infer | hir::TyKind::Err => Err("Ty"),
-        }
-    }
-}
-
-impl<'hir> Sig for hir::Item<'hir> {
-    fn make(&self, offset: usize, _parent_id: Option<hir::HirId>, scx: &SaveContext<'_>) -> Result {
-        let id = Some(self.hir_id());
-
-        match self.kind {
-            hir::ItemKind::Static(ref ty, m, ref body) => {
-                let mut text = "static ".to_owned();
-                if m.is_mut() {
-                    text.push_str("mut ");
-                }
-                let name = self.ident.to_string();
-                let defs = vec![SigElement {
-                    id: id_from_def_id(self.owner_id.to_def_id()),
-                    start: offset + text.len(),
-                    end: offset + text.len() + name.len(),
-                }];
-                text.push_str(&name);
-                text.push_str(": ");
-
-                let ty = ty.make(offset + text.len(), id, scx)?;
-                text.push_str(&ty.text);
-
-                text.push_str(" = ");
-                let expr = id_to_string(&scx.tcx.hir(), body.hir_id).replace('\n', " ");
-                text.push_str(&expr);
-
-                text.push(';');
-
-                Ok(extend_sig(ty, text, defs, vec![]))
-            }
-            hir::ItemKind::Const(ref ty, ref body) => {
-                let mut text = "const ".to_owned();
-                let name = self.ident.to_string();
-                let defs = vec![SigElement {
-                    id: id_from_def_id(self.owner_id.to_def_id()),
-                    start: offset + text.len(),
-                    end: offset + text.len() + name.len(),
-                }];
-                text.push_str(&name);
-                text.push_str(": ");
-
-                let ty = ty.make(offset + text.len(), id, scx)?;
-                text.push_str(&ty.text);
-
-                text.push_str(" = ");
-                let expr = id_to_string(&scx.tcx.hir(), body.hir_id).replace('\n', " ");
-                text.push_str(&expr);
-
-                text.push(';');
-
-                Ok(extend_sig(ty, text, defs, vec![]))
-            }
-            hir::ItemKind::Fn(hir::FnSig { ref decl, header, span: _ }, ref generics, _) => {
-                let mut text = String::new();
-                if let hir::Constness::Const = header.constness {
-                    text.push_str("const ");
-                }
-                if hir::IsAsync::Async == header.asyncness {
-                    text.push_str("async ");
-                }
-                if let hir::Unsafety::Unsafe = header.unsafety {
-                    text.push_str("unsafe ");
-                }
-                text.push_str("fn ");
-
-                let mut sig =
-                    name_and_generics(text, offset, generics, self.hir_id(), self.ident, scx)?;
-
-                sig.text.push('(');
-                for i in decl.inputs {
-                    // FIXME should descend into patterns to add defs.
-                    sig.text.push_str(": ");
-                    let nested = i.make(offset + sig.text.len(), Some(i.hir_id), scx)?;
-                    sig.text.push_str(&nested.text);
-                    sig.text.push(',');
-                    sig.defs.extend(nested.defs.into_iter());
-                    sig.refs.extend(nested.refs.into_iter());
-                }
-                sig.text.push(')');
-
-                if let hir::FnRetTy::Return(ref t) = decl.output {
-                    sig.text.push_str(" -> ");
-                    let nested = t.make(offset + sig.text.len(), None, scx)?;
-                    sig.text.push_str(&nested.text);
-                    sig.defs.extend(nested.defs.into_iter());
-                    sig.refs.extend(nested.refs.into_iter());
-                }
-                sig.text.push_str(" {}");
-
-                Ok(sig)
-            }
-            hir::ItemKind::Macro(..) => {
-                let mut text = "macro".to_owned();
-                let name = self.ident.to_string();
-                text.push_str(&name);
-                text.push_str(&"! {}");
-
-                Ok(text_sig(text))
-            }
-            hir::ItemKind::Mod(ref _mod) => {
-                let mut text = "mod ".to_owned();
-                let name = self.ident.to_string();
-                let defs = vec![SigElement {
-                    id: id_from_def_id(self.owner_id.to_def_id()),
-                    start: offset + text.len(),
-                    end: offset + text.len() + name.len(),
-                }];
-                text.push_str(&name);
-                // Could be either `mod foo;` or `mod foo { ... }`, but we'll just pick one.
-                text.push(';');
-
-                Ok(Signature { text, defs, refs: vec![] })
-            }
-            hir::ItemKind::TyAlias(ref ty, ref generics) => {
-                let text = "type ".to_owned();
-                let mut sig =
-                    name_and_generics(text, offset, generics, self.hir_id(), self.ident, scx)?;
-
-                sig.text.push_str(" = ");
-                let ty = ty.make(offset + sig.text.len(), id, scx)?;
-                sig.text.push_str(&ty.text);
-                sig.text.push(';');
-
-                Ok(merge_sigs(sig.text.clone(), vec![sig, ty]))
-            }
-            hir::ItemKind::Enum(_, ref generics) => {
-                let text = "enum ".to_owned();
-                let mut sig =
-                    name_and_generics(text, offset, generics, self.hir_id(), self.ident, scx)?;
-                sig.text.push_str(" {}");
-                Ok(sig)
-            }
-            hir::ItemKind::Struct(_, ref generics) => {
-                let text = "struct ".to_owned();
-                let mut sig =
-                    name_and_generics(text, offset, generics, self.hir_id(), self.ident, scx)?;
-                sig.text.push_str(" {}");
-                Ok(sig)
-            }
-            hir::ItemKind::Union(_, ref generics) => {
-                let text = "union ".to_owned();
-                let mut sig =
-                    name_and_generics(text, offset, generics, self.hir_id(), self.ident, scx)?;
-                sig.text.push_str(" {}");
-                Ok(sig)
-            }
-            hir::ItemKind::Trait(is_auto, unsafety, ref generics, bounds, _) => {
-                let mut text = String::new();
-
-                if is_auto == hir::IsAuto::Yes {
-                    text.push_str("auto ");
-                }
-
-                if let hir::Unsafety::Unsafe = unsafety {
-                    text.push_str("unsafe ");
-                }
-                text.push_str("trait ");
-                let mut sig =
-                    name_and_generics(text, offset, generics, self.hir_id(), self.ident, scx)?;
-
-                if !bounds.is_empty() {
-                    sig.text.push_str(": ");
-                    sig.text.push_str(&bounds_to_string(bounds));
-                }
-                // FIXME where clause
-                sig.text.push_str(" {}");
-
-                Ok(sig)
-            }
-            hir::ItemKind::TraitAlias(ref generics, bounds) => {
-                let mut text = String::new();
-                text.push_str("trait ");
-                let mut sig =
-                    name_and_generics(text, offset, generics, self.hir_id(), self.ident, scx)?;
-
-                if !bounds.is_empty() {
-                    sig.text.push_str(" = ");
-                    sig.text.push_str(&bounds_to_string(bounds));
-                }
-                // FIXME where clause
-                sig.text.push(';');
-
-                Ok(sig)
-            }
-            hir::ItemKind::Impl(hir::Impl {
-                unsafety,
-                polarity,
-                defaultness,
-                defaultness_span: _,
-                constness,
-                ref generics,
-                ref of_trait,
-                ref self_ty,
-                items: _,
-            }) => {
-                let mut text = String::new();
-                if let hir::Defaultness::Default { .. } = defaultness {
-                    text.push_str("default ");
-                }
-                if let hir::Unsafety::Unsafe = unsafety {
-                    text.push_str("unsafe ");
-                }
-                text.push_str("impl");
-                if let hir::Constness::Const = constness {
-                    text.push_str(" const");
-                }
-
-                let generics_sig = generics.make(offset + text.len(), id, scx)?;
-                text.push_str(&generics_sig.text);
-
-                text.push(' ');
-
-                let trait_sig = if let Some(ref t) = *of_trait {
-                    if let hir::ImplPolarity::Negative(_) = polarity {
-                        text.push('!');
-                    }
-                    let trait_sig = t.path.make(offset + text.len(), id, scx)?;
-                    text.push_str(&trait_sig.text);
-                    text.push_str(" for ");
-                    trait_sig
-                } else {
-                    text_sig(String::new())
-                };
-
-                let ty_sig = self_ty.make(offset + text.len(), id, scx)?;
-                text.push_str(&ty_sig.text);
-
-                text.push_str(" {}");
-
-                Ok(merge_sigs(text, vec![generics_sig, trait_sig, ty_sig]))
-
-                // FIXME where clause
-            }
-            hir::ItemKind::ForeignMod { .. } => Err("extern mod"),
-            hir::ItemKind::GlobalAsm(_) => Err("global asm"),
-            hir::ItemKind::ExternCrate(_) => Err("extern crate"),
-            hir::ItemKind::OpaqueTy(ref opaque) => {
-                if opaque.in_trait {
-                    Err("opaque type in trait")
-                } else {
-                    Err("opaque type")
-                }
-            }
-            // FIXME should implement this (e.g., pub use).
-            hir::ItemKind::Use(..) => Err("import"),
-        }
-    }
-}
-
-impl<'hir> Sig for hir::Path<'hir> {
-    fn make(&self, offset: usize, id: Option<hir::HirId>, scx: &SaveContext<'_>) -> Result {
-        let res = scx.get_path_res(id.ok_or("Missing id for Path")?);
-
-        let (name, start, end) = match res {
-            Res::PrimTy(..) | Res::SelfTyParam { .. } | Res::SelfTyAlias { .. } | Res::Err => {
-                return Ok(Signature { text: path_to_string(self), defs: vec![], refs: vec![] });
-            }
-            Res::Def(DefKind::AssocConst | DefKind::Variant | DefKind::Ctor(..), _) => {
-                let len = self.segments.len();
-                if len < 2 {
-                    return Err("Bad path");
-                }
-                // FIXME: really we should descend into the generics here and add SigElements for
-                // them.
-                // FIXME: would be nice to have a def for the first path segment.
-                let seg1 = path_segment_to_string(&self.segments[len - 2]);
-                let seg2 = path_segment_to_string(&self.segments[len - 1]);
-                let start = offset + seg1.len() + 2;
-                (format!("{}::{}", seg1, seg2), start, start + seg2.len())
-            }
-            _ => {
-                let name = path_segment_to_string(self.segments.last().ok_or("Bad path")?);
-                let end = offset + name.len();
-                (name, offset, end)
-            }
-        };
-
-        let id = id_from_def_id(res.def_id());
-        Ok(Signature { text: name, defs: vec![], refs: vec![SigElement { id, start, end }] })
-    }
-}
-
-// This does not cover the where clause, which must be processed separately.
-impl<'hir> Sig for hir::Generics<'hir> {
-    fn make(&self, offset: usize, _parent_id: Option<hir::HirId>, scx: &SaveContext<'_>) -> Result {
-        if self.params.is_empty() {
-            return Ok(text_sig(String::new()));
-        }
-
-        let mut text = "<".to_owned();
-
-        let mut defs = Vec::with_capacity(self.params.len());
-        for param in self.params {
-            let mut param_text = String::new();
-            if let hir::GenericParamKind::Const { .. } = param.kind {
-                param_text.push_str("const ");
-            }
-            param_text.push_str(param.name.ident().as_str());
-            defs.push(SigElement {
-                id: id_from_hir_id(param.hir_id, scx),
-                start: offset + text.len(),
-                end: offset + text.len() + param_text.as_str().len(),
-            });
-            if let hir::GenericParamKind::Const { ref ty, default } = param.kind {
-                param_text.push_str(": ");
-                param_text.push_str(&ty_to_string(&ty));
-                if let Some(default) = default {
-                    param_text.push_str(" = ");
-                    param_text.push_str(&id_to_string(&scx.tcx.hir(), default.hir_id));
-                }
-            }
-            text.push_str(&param_text);
-            text.push(',');
-        }
-
-        text.push('>');
-        Ok(Signature { text, defs, refs: vec![] })
-    }
-}
-
-impl<'hir> Sig for hir::FieldDef<'hir> {
-    fn make(&self, offset: usize, _parent_id: Option<hir::HirId>, scx: &SaveContext<'_>) -> Result {
-        let mut text = String::new();
-
-        text.push_str(&self.ident.to_string());
-        let defs = Some(SigElement {
-            id: id_from_hir_id(self.hir_id, scx),
-            start: offset,
-            end: offset + text.len(),
-        });
-        text.push_str(": ");
-
-        let mut ty_sig = self.ty.make(offset + text.len(), Some(self.hir_id), scx)?;
-        text.push_str(&ty_sig.text);
-        ty_sig.text = text;
-        ty_sig.defs.extend(defs.into_iter());
-        Ok(ty_sig)
-    }
-}
-
-impl<'hir> Sig for hir::Variant<'hir> {
-    fn make(&self, offset: usize, parent_id: Option<hir::HirId>, scx: &SaveContext<'_>) -> Result {
-        let mut text = self.ident.to_string();
-        match self.data {
-            hir::VariantData::Struct(fields, r) => {
-                let id = parent_id.ok_or("Missing id for Variant's parent")?;
-                let name_def = SigElement {
-                    id: id_from_hir_id(id, scx),
-                    start: offset,
-                    end: offset + text.len(),
-                };
-                text.push_str(" { ");
-                let mut defs = vec![name_def];
-                let mut refs = vec![];
-                if r {
-                    text.push_str("/* parse error */ ");
-                } else {
-                    for f in fields {
-                        let field_sig = f.make(offset + text.len(), Some(id), scx)?;
-                        text.push_str(&field_sig.text);
-                        text.push_str(", ");
-                        defs.extend(field_sig.defs.into_iter());
-                        refs.extend(field_sig.refs.into_iter());
-                    }
-                }
-                text.push('}');
-                Ok(Signature { text, defs, refs })
-            }
-            hir::VariantData::Tuple(fields, id, _) => {
-                let name_def = SigElement {
-                    id: id_from_hir_id(id, scx),
-                    start: offset,
-                    end: offset + text.len(),
-                };
-                text.push('(');
-                let mut defs = vec![name_def];
-                let mut refs = vec![];
-                for f in fields {
-                    let field_sig = f.make(offset + text.len(), Some(id), scx)?;
-                    text.push_str(&field_sig.text);
-                    text.push_str(", ");
-                    defs.extend(field_sig.defs.into_iter());
-                    refs.extend(field_sig.refs.into_iter());
-                }
-                text.push(')');
-                Ok(Signature { text, defs, refs })
-            }
-            hir::VariantData::Unit(id, _) => {
-                let name_def = SigElement {
-                    id: id_from_hir_id(id, scx),
-                    start: offset,
-                    end: offset + text.len(),
-                };
-                Ok(Signature { text, defs: vec![name_def], refs: vec![] })
-            }
-        }
-    }
-}
-
-impl<'hir> Sig for hir::ForeignItem<'hir> {
-    fn make(&self, offset: usize, _parent_id: Option<hir::HirId>, scx: &SaveContext<'_>) -> Result {
-        let id = Some(self.hir_id());
-        match self.kind {
-            hir::ForeignItemKind::Fn(decl, _, ref generics) => {
-                let mut text = String::new();
-                text.push_str("fn ");
-
-                let mut sig =
-                    name_and_generics(text, offset, generics, self.hir_id(), self.ident, scx)?;
-
-                sig.text.push('(');
-                for i in decl.inputs {
-                    sig.text.push_str(": ");
-                    let nested = i.make(offset + sig.text.len(), Some(i.hir_id), scx)?;
-                    sig.text.push_str(&nested.text);
-                    sig.text.push(',');
-                    sig.defs.extend(nested.defs.into_iter());
-                    sig.refs.extend(nested.refs.into_iter());
-                }
-                sig.text.push(')');
-
-                if let hir::FnRetTy::Return(ref t) = decl.output {
-                    sig.text.push_str(" -> ");
-                    let nested = t.make(offset + sig.text.len(), None, scx)?;
-                    sig.text.push_str(&nested.text);
-                    sig.defs.extend(nested.defs.into_iter());
-                    sig.refs.extend(nested.refs.into_iter());
-                }
-                sig.text.push(';');
-
-                Ok(sig)
-            }
-            hir::ForeignItemKind::Static(ref ty, m) => {
-                let mut text = "static ".to_owned();
-                if m == Mutability::Mut {
-                    text.push_str("mut ");
-                }
-                let name = self.ident.to_string();
-                let defs = vec![SigElement {
-                    id: id_from_def_id(self.owner_id.to_def_id()),
-                    start: offset + text.len(),
-                    end: offset + text.len() + name.len(),
-                }];
-                text.push_str(&name);
-                text.push_str(": ");
-
-                let ty_sig = ty.make(offset + text.len(), id, scx)?;
-                text.push(';');
-
-                Ok(extend_sig(ty_sig, text, defs, vec![]))
-            }
-            hir::ForeignItemKind::Type => {
-                let mut text = "type ".to_owned();
-                let name = self.ident.to_string();
-                let defs = vec![SigElement {
-                    id: id_from_def_id(self.owner_id.to_def_id()),
-                    start: offset + text.len(),
-                    end: offset + text.len() + name.len(),
-                }];
-                text.push_str(&name);
-                text.push(';');
-
-                Ok(Signature { text, defs, refs: vec![] })
-            }
-        }
-    }
-}
-
-fn name_and_generics(
-    mut text: String,
-    offset: usize,
-    generics: &hir::Generics<'_>,
-    id: hir::HirId,
-    name: Ident,
-    scx: &SaveContext<'_>,
-) -> Result {
-    let name = name.to_string();
-    let def = SigElement {
-        id: id_from_hir_id(id, scx),
-        start: offset + text.len(),
-        end: offset + text.len() + name.len(),
-    };
-    text.push_str(&name);
-    let generics: Signature = generics.make(offset + text.len(), Some(id), scx)?;
-    // FIXME where clause
-    let text = format!("{}{}", text, generics.text);
-    Ok(extend_sig(generics, text, vec![def], vec![]))
-}
-
-fn make_assoc_type_signature(
-    id: hir::HirId,
-    ident: Ident,
-    bounds: Option<hir::GenericBounds<'_>>,
-    default: Option<&hir::Ty<'_>>,
-    scx: &SaveContext<'_>,
-) -> Result {
-    let mut text = "type ".to_owned();
-    let name = ident.to_string();
-    let mut defs = vec![SigElement {
-        id: id_from_hir_id(id, scx),
-        start: text.len(),
-        end: text.len() + name.len(),
-    }];
-    let mut refs = vec![];
-    text.push_str(&name);
-    if let Some(bounds) = bounds {
-        text.push_str(": ");
-        // FIXME should descend into bounds
-        text.push_str(&bounds_to_string(bounds));
-    }
-    if let Some(default) = default {
-        text.push_str(" = ");
-        let ty_sig = default.make(text.len(), Some(id), scx)?;
-        text.push_str(&ty_sig.text);
-        defs.extend(ty_sig.defs.into_iter());
-        refs.extend(ty_sig.refs.into_iter());
-    }
-    text.push(';');
-    Ok(Signature { text, defs, refs })
-}
-
-fn make_assoc_const_signature(
-    id: hir::HirId,
-    ident: Symbol,
-    ty: &hir::Ty<'_>,
-    default: Option<&hir::Expr<'_>>,
-    scx: &SaveContext<'_>,
-) -> Result {
-    let mut text = "const ".to_owned();
-    let name = ident.to_string();
-    let mut defs = vec![SigElement {
-        id: id_from_hir_id(id, scx),
-        start: text.len(),
-        end: text.len() + name.len(),
-    }];
-    let mut refs = vec![];
-    text.push_str(&name);
-    text.push_str(": ");
-
-    let ty_sig = ty.make(text.len(), Some(id), scx)?;
-    text.push_str(&ty_sig.text);
-    defs.extend(ty_sig.defs.into_iter());
-    refs.extend(ty_sig.refs.into_iter());
-
-    if let Some(default) = default {
-        text.push_str(" = ");
-        text.push_str(&id_to_string(&scx.tcx.hir(), default.hir_id));
-    }
-    text.push(';');
-    Ok(Signature { text, defs, refs })
-}
-
-fn make_method_signature(
-    id: hir::HirId,
-    ident: Ident,
-    generics: &hir::Generics<'_>,
-    m: &hir::FnSig<'_>,
-    scx: &SaveContext<'_>,
-) -> Result {
-    // FIXME code dup with function signature
-    let mut text = String::new();
-    if let hir::Constness::Const = m.header.constness {
-        text.push_str("const ");
-    }
-    if hir::IsAsync::Async == m.header.asyncness {
-        text.push_str("async ");
-    }
-    if let hir::Unsafety::Unsafe = m.header.unsafety {
-        text.push_str("unsafe ");
-    }
-    text.push_str("fn ");
-
-    let mut sig = name_and_generics(text, 0, generics, id, ident, scx)?;
-
-    sig.text.push('(');
-    for i in m.decl.inputs {
-        sig.text.push_str(": ");
-        let nested = i.make(sig.text.len(), Some(i.hir_id), scx)?;
-        sig.text.push_str(&nested.text);
-        sig.text.push(',');
-        sig.defs.extend(nested.defs.into_iter());
-        sig.refs.extend(nested.refs.into_iter());
-    }
-    sig.text.push(')');
-
-    if let hir::FnRetTy::Return(ref t) = m.decl.output {
-        sig.text.push_str(" -> ");
-        let nested = t.make(sig.text.len(), None, scx)?;
-        sig.text.push_str(&nested.text);
-        sig.defs.extend(nested.defs.into_iter());
-        sig.refs.extend(nested.refs.into_iter());
-    }
-    sig.text.push_str(" {}");
-
-    Ok(sig)
-}
diff --git a/compiler/rustc_save_analysis/src/span_utils.rs b/compiler/rustc_save_analysis/src/span_utils.rs
deleted file mode 100644
index e65d57bb3..000000000
--- a/compiler/rustc_save_analysis/src/span_utils.rs
+++ /dev/null
@@ -1,96 +0,0 @@
-use crate::generated_code;
-use rustc_data_structures::sync::Lrc;
-use rustc_lexer::{tokenize, TokenKind};
-use rustc_session::Session;
-use rustc_span::*;
-
-#[derive(Clone)]
-pub struct SpanUtils<'a> {
-    pub sess: &'a Session,
-}
-
-impl<'a> SpanUtils<'a> {
-    pub fn new(sess: &'a Session) -> SpanUtils<'a> {
-        SpanUtils { sess }
-    }
-
-    pub fn make_filename_string(&self, file: &SourceFile) -> String {
-        match &file.name {
-            FileName::Real(RealFileName::LocalPath(path)) => {
-                if path.is_absolute() {
-                    self.sess.source_map().path_mapping().map_prefix(path).0.display().to_string()
-                } else {
-                    self.sess
-                        .opts
-                        .working_dir
-                        .remapped_path_if_available()
-                        .join(&path)
-                        .display()
-                        .to_string()
-                }
-            }
-            filename => filename.prefer_remapped().to_string(),
-        }
-    }
-
-    pub fn snippet(&self, span: Span) -> String {
-        match self.sess.source_map().span_to_snippet(span) {
-            Ok(s) => s,
-            Err(_) => String::new(),
-        }
-    }
-
-    /// Finds the span of `*` token withing the larger `span`.
-    pub fn sub_span_of_star(&self, mut span: Span) -> Option<Span> {
-        let begin = self.sess.source_map().lookup_byte_offset(span.lo());
-        let end = self.sess.source_map().lookup_byte_offset(span.hi());
-        // Make the range zero-length if the span is invalid.
-        if begin.sf.start_pos != end.sf.start_pos {
-            span = span.shrink_to_lo();
-        }
-
-        let sf = Lrc::clone(&begin.sf);
-
-        self.sess.source_map().ensure_source_file_source_present(Lrc::clone(&sf));
-        let src =
-            sf.src.clone().or_else(|| sf.external_src.borrow().get_source().map(Lrc::clone))?;
-        let to_index = |pos: BytePos| -> usize { (pos - sf.start_pos).0 as usize };
-        let text = &src[to_index(span.lo())..to_index(span.hi())];
-        let start_pos = {
-            let mut pos = 0;
-            tokenize(text)
-                .map(|token| {
-                    let start = pos;
-                    pos += token.len;
-                    (start, token)
-                })
-                .find(|(_pos, token)| token.kind == TokenKind::Star)?
-                .0
-        };
-        let lo = span.lo() + BytePos(start_pos as u32);
-        let hi = lo + BytePos(1);
-        Some(span.with_lo(lo).with_hi(hi))
-    }
-
-    /// Return true if the span is generated code, and
-    /// it is not a subspan of the root callsite.
-    ///
-    /// Used to filter out spans of minimal value,
-    /// such as references to macro internal variables.
-    pub fn filter_generated(&self, span: Span) -> bool {
-        if generated_code(span) {
-            return true;
-        }
-
-        //If the span comes from a fake source_file, filter it.
-        !self.sess.source_map().lookup_char_pos(span.lo()).file.is_real_file()
-    }
-}
-
-macro_rules! filter {
-    ($util: expr, $parent: expr) => {
-        if $util.filter_generated($parent) {
-            return None;
-        }
-    };
-}
diff --git a/compiler/rustc_serialize/Cargo.toml b/compiler/rustc_serialize/Cargo.toml
index db0ef7354..c04465719 100644
--- a/compiler/rustc_serialize/Cargo.toml
+++ b/compiler/rustc_serialize/Cargo.toml
@@ -6,7 +6,7 @@ edition = "2021"
 [dependencies]
 indexmap = "1.9.1"
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-thin-vec = "0.2.9"
+thin-vec = "0.2.12"
 
 [dev-dependencies]
 rustc_macros = { path = "../rustc_macros" }
diff --git a/compiler/rustc_serialize/src/serialize.rs b/compiler/rustc_serialize/src/serialize.rs
index 751b209f1..567fe0610 100644
--- a/compiler/rustc_serialize/src/serialize.rs
+++ b/compiler/rustc_serialize/src/serialize.rs
@@ -43,7 +43,6 @@ pub trait Encoder {
     fn emit_str(&mut self, v: &str);
     fn emit_raw_bytes(&mut self, s: &[u8]);
 
-    // Convenience for the derive macro:
     fn emit_enum_variant<F>(&mut self, v_id: usize, f: F)
     where
         F: FnOnce(&mut Self),
@@ -51,17 +50,6 @@ pub trait Encoder {
         self.emit_usize(v_id);
         f(self);
     }
-
-    // We put the field index in a const generic to allow the emit_usize to be
-    // compiled into a more efficient form. In practice, the variant index is
-    // known at compile-time, and that knowledge allows much more efficient
-    // codegen than we'd otherwise get. LLVM isn't always able to make the
-    // optimization that would otherwise be necessary here, likely due to the
-    // multiple levels of inlining and const-prop that are needed.
-    #[inline]
-    fn emit_fieldless_enum_variant<const ID: usize>(&mut self) {
-        self.emit_usize(ID)
-    }
 }
 
 // Note: all the methods in this trait are infallible, which may be surprising.
@@ -430,11 +418,6 @@ impl<D: Decoder, T: Decodable<D> + Copy> Decodable<D> for Cell<T> {
     }
 }
 
-// FIXME: #15036
-// Should use `try_borrow`, returning an
-// `encoder.error("attempting to Encode borrowed RefCell")`
-// from `encode` when `try_borrow` returns `None`.
-
 impl<S: Encoder, T: Encodable<S>> Encodable<S> for RefCell<T> {
     fn encode(&self, s: &mut S) {
         self.borrow().encode(s);
diff --git a/compiler/rustc_error_messages/locales/en-US/session.ftl b/compiler/rustc_session/locales/en-US.ftl
index bc37d91a7..ff53f22d4 100644
--- a/compiler/rustc_error_messages/locales/en-US/session.ftl
+++ b/compiler/rustc_session/locales/en-US.ftl
@@ -5,7 +5,7 @@ session_incorrect_cgu_reuse_type =
     }`{$expected_reuse}`
 
 session_cgu_not_recorded =
-    CGU-reuse for `{$cgu_user_name}` is (mangled: `{$cgu_name}`) was not recorded`
+    CGU-reuse for `{$cgu_user_name}` is (mangled: `{$cgu_name}`) was not recorded
 
 session_feature_gate_error = {$explain}
 
@@ -25,6 +25,8 @@ session_profile_sample_use_file_does_not_exist = file `{$path}` passed to `-C pr
 
 session_target_requires_unwind_tables = target requires unwind tables, they cannot be disabled with `-C force-unwind-tables=no`
 
+session_instrumentation_not_supported = {$us} instrumentation is not supported for this target
+
 session_sanitizer_not_supported = {$us} sanitizer is not supported for this target
 
 session_sanitizers_not_supported = {$us} sanitizers are not supported for this target
@@ -89,3 +91,5 @@ session_int_literal_too_large = integer literal is too large
 
 session_invalid_int_literal_width = invalid width `{$width}` for integer literal
     .help = valid widths are 8, 16, 32, 64 and 128
+
+session_optimization_fuel_exhausted = optimization-fuel-exhausted: {$msg}
diff --git a/compiler/rustc_session/src/code_stats.rs b/compiler/rustc_session/src/code_stats.rs
index 1085bce44..551782504 100644
--- a/compiler/rustc_session/src/code_stats.rs
+++ b/compiler/rustc_session/src/code_stats.rs
@@ -20,7 +20,25 @@ pub enum SizeKind {
 }
 
 #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
+pub enum FieldKind {
+    AdtField,
+    Upvar,
+    GeneratorLocal,
+}
+
+impl std::fmt::Display for FieldKind {
+    fn fmt(&self, w: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            FieldKind::AdtField => write!(w, "field"),
+            FieldKind::Upvar => write!(w, "upvar"),
+            FieldKind::GeneratorLocal => write!(w, "local"),
+        }
+    }
+}
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
 pub struct FieldInfo {
+    pub kind: FieldKind,
     pub name: Symbol,
     pub offset: u64,
     pub size: u64,
@@ -66,7 +84,11 @@ impl CodeStats {
         // Sort variants so the largest ones are shown first. A stable sort is
         // used here so that source code order is preserved for all variants
         // that have the same size.
-        variants.sort_by(|info1, info2| info2.size.cmp(&info1.size));
+        // Except for Generators, whose variants are already sorted according to
+        // their yield points in `variant_info_for_generator`.
+        if kind != DataTypeKind::Generator {
+            variants.sort_by(|info1, info2| info2.size.cmp(&info1.size));
+        }
         let info = TypeSizeInfo {
             kind,
             type_description: type_desc.to_string(),
@@ -145,7 +167,7 @@ impl CodeStats {
                 fields.sort_by_key(|f| (f.offset, f.size));
 
                 for field in fields {
-                    let FieldInfo { ref name, offset, size, align } = field;
+                    let FieldInfo { kind, ref name, offset, size, align } = field;
 
                     if offset > min_offset {
                         let pad = offset - min_offset;
@@ -155,16 +177,16 @@ impl CodeStats {
                     if offset < min_offset {
                         // If this happens it's probably a union.
                         println!(
-                            "print-type-size {indent}field `.{name}`: {size} bytes, \
+                            "print-type-size {indent}{kind} `.{name}`: {size} bytes, \
                                   offset: {offset} bytes, \
                                   alignment: {align} bytes"
                         );
                     } else if info.packed || offset == min_offset {
-                        println!("print-type-size {indent}field `.{name}`: {size} bytes");
+                        println!("print-type-size {indent}{kind} `.{name}`: {size} bytes");
                     } else {
                         // Include field alignment in output only if it caused padding injection
                         println!(
-                            "print-type-size {indent}field `.{name}`: {size} bytes, \
+                            "print-type-size {indent}{kind} `.{name}`: {size} bytes, \
                                   alignment: {align} bytes"
                         );
                     }
diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index 586454f76..d4e4ace88 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -24,7 +24,7 @@ use rustc_span::RealFileName;
 use rustc_span::SourceFileHashAlgorithm;
 
 use rustc_errors::emitter::HumanReadableErrorType;
-use rustc_errors::{ColorConfig, HandlerFlags};
+use rustc_errors::{ColorConfig, DiagnosticArgValue, HandlerFlags, IntoDiagnosticArg};
 
 use std::collections::btree_map::{
     Iter as BTreeMapIter, Keys as BTreeMapKeysIter, Values as BTreeMapValuesIter,
@@ -174,6 +174,25 @@ pub enum InstrumentCoverage {
     Off,
 }
 
+/// Settings for `-Z instrument-xray` flag.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
+pub struct InstrumentXRay {
+    /// `-Z instrument-xray=always`, force instrumentation
+    pub always: bool,
+    /// `-Z instrument-xray=never`, disable instrumentation
+    pub never: bool,
+    /// `-Z instrument-xray=ignore-loops`, ignore presence of loops,
+    /// instrument functions based only on instruction count
+    pub ignore_loops: bool,
+    /// `-Z instrument-xray=instruction-threshold=N`, explicitly set instruction threshold
+    /// for instrumentation, or `None` to use compiler's default
+    pub instruction_threshold: Option<usize>,
+    /// `-Z instrument-xray=skip-entry`, do not instrument function entry
+    pub skip_entry: bool,
+    /// `-Z instrument-xray=skip-exit`, do not instrument function exit
+    pub skip_exit: bool,
+}
+
 #[derive(Clone, PartialEq, Hash, Debug)]
 pub enum LinkerPluginLto {
     LinkerPlugin(PathBuf),
@@ -400,6 +419,18 @@ pub enum TrimmedDefPaths {
     GoodPath,
 }
 
+#[derive(Clone, Hash)]
+pub enum ResolveDocLinks {
+    /// Do not resolve doc links.
+    None,
+    /// Resolve doc links on exported items only for crate types that have metadata.
+    ExportedMetadata,
+    /// Resolve doc links on exported items.
+    Exported,
+    /// Resolve doc links on all items.
+    All,
+}
+
 /// Use tree-based collections to cheaply get a deterministic `Hash` implementation.
 /// *Do not* switch `BTreeMap` out for an unsorted container type! That would break
 /// dependency tracking for command-line arguments. Also only hash keys, since tracking
@@ -769,6 +800,7 @@ impl Default for Options {
             unstable_features: UnstableFeatures::Disallow,
             debug_assertions: true,
             actually_rustdoc: false,
+            resolve_doc_links: ResolveDocLinks::None,
             trimmed_def_paths: TrimmedDefPaths::default(),
             cli_forced_codegen_units: None,
             cli_forced_local_thinlto_off: false,
@@ -865,13 +897,10 @@ pub enum CrateType {
 }
 
 impl CrateType {
-    /// When generated, is this crate type an archive?
-    pub fn is_archive(&self) -> bool {
-        match *self {
-            CrateType::Rlib | CrateType::Staticlib => true,
-            CrateType::Executable | CrateType::Dylib | CrateType::Cdylib | CrateType::ProcMacro => {
-                false
-            }
+    pub fn has_metadata(self) -> bool {
+        match self {
+            CrateType::Rlib | CrateType::Dylib | CrateType::ProcMacro => true,
+            CrateType::Executable | CrateType::Cdylib | CrateType::Staticlib => false,
         }
     }
 }
@@ -957,6 +986,7 @@ fn default_configuration(sess: &Session) -> CrateConfig {
     if sess.target.has_thread_local {
         ret.insert((sym::target_thread_local, None));
     }
+    let mut has_atomic = false;
     for (i, align) in [
         (8, layout.i8_align.abi),
         (16, layout.i16_align.abi),
@@ -965,6 +995,7 @@ fn default_configuration(sess: &Session) -> CrateConfig {
         (128, layout.i128_align.abi),
     ] {
         if i >= min_atomic_width && i <= max_atomic_width {
+            has_atomic = true;
             let mut insert_atomic = |s, align: Align| {
                 ret.insert((sym::target_has_atomic_load_store, Some(Symbol::intern(s))));
                 if atomic_cas {
@@ -981,11 +1012,23 @@ fn default_configuration(sess: &Session) -> CrateConfig {
             }
         }
     }
+    if sess.is_nightly_build() && has_atomic {
+        ret.insert((sym::target_has_atomic_load_store, None));
+        if atomic_cas {
+            ret.insert((sym::target_has_atomic, None));
+        }
+    }
 
     let panic_strategy = sess.panic_strategy();
     ret.insert((sym::panic, Some(panic_strategy.desc_symbol())));
 
-    for s in sess.opts.unstable_opts.sanitizer {
+    for mut s in sess.opts.unstable_opts.sanitizer {
+        // KASAN should use the same attribute name as ASAN, as it's still ASAN
+        // under the hood
+        if s == SanitizerSet::KERNELADDRESS {
+            s = SanitizerSet::ADDRESS;
+        }
+
         let symbol = Symbol::intern(&s.to_string());
         ret.insert((sym::sanitize, Some(symbol)));
     }
@@ -2202,7 +2245,7 @@ pub fn parse_externs(
                 early_error(
                     error_format,
                     "the `-Z unstable-options` flag must also be passed to \
-                     enable `--extern options",
+                     enable `--extern` options",
                 );
             }
             for opt in opts.split(',') {
@@ -2507,7 +2550,7 @@ pub fn build_session_options(matches: &getopts::Matches) -> Options {
         }
 
         // Only use this directory if it has a file we can expect to always find.
-        if candidate.join("library/std/src/lib.rs").is_file() { Some(candidate) } else { None }
+        candidate.join("library/std/src/lib.rs").is_file().then_some(candidate)
     };
 
     let working_dir = std::env::current_dir().unwrap_or_else(|e| {
@@ -2547,6 +2590,7 @@ pub fn build_session_options(matches: &getopts::Matches) -> Options {
         libs,
         debug_assertions,
         actually_rustdoc: false,
+        resolve_doc_links: ResolveDocLinks::ExportedMetadata,
         trimmed_def_paths: TrimmedDefPaths::default(),
         cli_forced_codegen_units: codegen_units,
         cli_forced_local_thinlto_off: disable_local_thinlto,
@@ -2577,6 +2621,7 @@ fn parse_pretty(unstable_opts: &UnstableOptions, efmt: ErrorOutputType) -> Optio
         "hir,typed" => Hir(PpHirMode::Typed),
         "hir-tree" => HirTree,
         "thir-tree" => ThirTree,
+        "thir-flat" => ThirFlat,
         "mir" => Mir,
         "mir-cfg" => MirCFG,
         name => early_error(
@@ -2585,7 +2630,8 @@ fn parse_pretty(unstable_opts: &UnstableOptions, efmt: ErrorOutputType) -> Optio
                 "argument to `unpretty` must be one of `normal`, `identified`, \
                             `expanded`, `expanded,identified`, `expanded,hygiene`, \
                             `ast-tree`, `ast-tree,expanded`, `hir`, `hir,identified`, \
-                            `hir,typed`, `hir-tree`, `thir-tree`, `mir` or `mir-cfg`; got {name}"
+                            `hir,typed`, `hir-tree`, `thir-tree`, `thir-flat`, `mir` or \
+                            `mir-cfg`; got {name}"
             ),
         ),
     };
@@ -2696,6 +2742,12 @@ impl fmt::Display for CrateType {
     }
 }
 
+impl IntoDiagnosticArg for CrateType {
+    fn into_diagnostic_arg(self) -> DiagnosticArgValue<'static> {
+        self.to_string().into_diagnostic_arg()
+    }
+}
+
 #[derive(Copy, Clone, PartialEq, Debug)]
 pub enum PpSourceMode {
     /// `-Zunpretty=normal`
@@ -2740,6 +2792,8 @@ pub enum PpMode {
     HirTree,
     /// `-Zunpretty=thir-tree`
     ThirTree,
+    /// `-Zunpretty=thir-flat`
+    ThirFlat,
     /// `-Zunpretty=mir`
     Mir,
     /// `-Zunpretty=mir-cfg`
@@ -2758,6 +2812,7 @@ impl PpMode {
             | Hir(_)
             | HirTree
             | ThirTree
+            | ThirFlat
             | Mir
             | MirCFG => true,
         }
@@ -2767,13 +2822,13 @@ impl PpMode {
         match *self {
             Source(_) | AstTree(_) => false,
 
-            Hir(_) | HirTree | ThirTree | Mir | MirCFG => true,
+            Hir(_) | HirTree | ThirTree | ThirFlat | Mir | MirCFG => true,
         }
     }
 
     pub fn needs_analysis(&self) -> bool {
         use PpMode::*;
-        matches!(*self, Mir | MirCFG | ThirTree)
+        matches!(*self, Mir | MirCFG | ThirTree | ThirFlat)
     }
 }
 
@@ -2798,9 +2853,10 @@ impl PpMode {
 pub(crate) mod dep_tracking {
     use super::{
         BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, ErrorOutputType,
-        InstrumentCoverage, LdImpl, LinkerPluginLto, LocationDetail, LtoCli, OomStrategy, OptLevel,
-        OutputType, OutputTypes, Passes, SourceFileHashAlgorithm, SplitDwarfKind,
-        SwitchWithOptPath, SymbolManglingVersion, TraitSolver, TrimmedDefPaths,
+        InstrumentCoverage, InstrumentXRay, LdImpl, LinkerPluginLto, LocationDetail, LtoCli,
+        OomStrategy, OptLevel, OutputType, OutputTypes, Passes, ResolveDocLinks,
+        SourceFileHashAlgorithm, SplitDwarfKind, SwitchWithOptPath, SymbolManglingVersion,
+        TraitSolver, TrimmedDefPaths,
     };
     use crate::lint;
     use crate::options::WasiExecModel;
@@ -2869,6 +2925,7 @@ pub(crate) mod dep_tracking {
         CodeModel,
         TlsModel,
         InstrumentCoverage,
+        InstrumentXRay,
         CrateType,
         MergeFunctions,
         PanicStrategy,
@@ -2886,6 +2943,7 @@ pub(crate) mod dep_tracking {
         TargetTriple,
         Edition,
         LinkerPluginLto,
+        ResolveDocLinks,
         SplitDebuginfo,
         SplitDwarfKind,
         StackProtector,
diff --git a/compiler/rustc_session/src/cstore.rs b/compiler/rustc_session/src/cstore.rs
index 4ae9a3fae..868ffdf0f 100644
--- a/compiler/rustc_session/src/cstore.rs
+++ b/compiler/rustc_session/src/cstore.rs
@@ -6,10 +6,9 @@ use crate::search_paths::PathKind;
 use crate::utils::NativeLibKind;
 use crate::Session;
 use rustc_ast as ast;
-use rustc_data_structures::sync::{self, MetadataRef, RwLock};
+use rustc_data_structures::sync::{self, AppendOnlyVec, MetadataRef, RwLock};
 use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, StableCrateId, LOCAL_CRATE};
 use rustc_hir::definitions::{DefKey, DefPath, DefPathHash, Definitions};
-use rustc_index::vec::IndexVec;
 use rustc_span::hygiene::{ExpnHash, ExpnId};
 use rustc_span::symbol::Symbol;
 use rustc_span::Span;
@@ -200,12 +199,12 @@ pub enum ExternCrateSource {
 /// At the time of this writing, there is only one backend and one way to store
 /// metadata in library -- this trait just serves to decouple rustc_metadata from
 /// the archive reader, which depends on LLVM.
-pub trait MetadataLoader {
+pub trait MetadataLoader: std::fmt::Debug {
     fn get_rlib_metadata(&self, target: &Target, filename: &Path) -> Result<MetadataRef, String>;
     fn get_dylib_metadata(&self, target: &Target, filename: &Path) -> Result<MetadataRef, String>;
 }
 
-pub type MetadataLoaderDyn = dyn MetadataLoader + Sync;
+pub type MetadataLoaderDyn = dyn MetadataLoader + Send + Sync;
 
 /// A store of Rust crates, through which their metadata can be accessed.
 ///
@@ -250,12 +249,11 @@ pub trait CrateStore: std::fmt::Debug {
     fn import_source_files(&self, sess: &Session, cnum: CrateNum);
 }
 
-pub type CrateStoreDyn = dyn CrateStore + sync::Sync;
+pub type CrateStoreDyn = dyn CrateStore + sync::Sync + sync::Send;
 
-#[derive(Debug)]
 pub struct Untracked {
-    pub cstore: Box<CrateStoreDyn>,
+    pub cstore: RwLock<Box<CrateStoreDyn>>,
     /// Reference span for definitions.
-    pub source_span: IndexVec<LocalDefId, Span>,
+    pub source_span: AppendOnlyVec<LocalDefId, Span>,
     pub definitions: RwLock<Definitions>,
 }
diff --git a/compiler/rustc_session/src/errors.rs b/compiler/rustc_session/src/errors.rs
index f5a72573d..bd32adbbd 100644
--- a/compiler/rustc_session/src/errors.rs
+++ b/compiler/rustc_session/src/errors.rs
@@ -4,7 +4,7 @@ use crate::cgu_reuse_tracker::CguReuse;
 use crate::parse::ParseSess;
 use rustc_ast::token;
 use rustc_ast::util::literal::LitError;
-use rustc_errors::MultiSpan;
+use rustc_errors::{error_code, DiagnosticMessage, EmissionGuarantee, IntoDiagnostic, MultiSpan};
 use rustc_macros::Diagnostic;
 use rustc_span::{Span, Symbol};
 use rustc_target::spec::{SplitDebuginfo, StackProtector, TargetTriple};
@@ -27,12 +27,22 @@ pub struct CguNotRecorded<'a> {
     pub cgu_name: &'a str,
 }
 
-#[derive(Diagnostic)]
-#[diag(session_feature_gate_error, code = "E0658")]
-pub struct FeatureGateError<'a> {
-    #[primary_span]
+pub struct FeatureGateError {
     pub span: MultiSpan,
-    pub explain: &'a str,
+    pub explain: DiagnosticMessage,
+}
+
+impl<'a, T: EmissionGuarantee> IntoDiagnostic<'a, T> for FeatureGateError {
+    #[track_caller]
+    fn into_diagnostic(
+        self,
+        handler: &'a rustc_errors::Handler,
+    ) -> rustc_errors::DiagnosticBuilder<'a, T> {
+        let mut diag = handler.struct_diagnostic(self.explain);
+        diag.set_span(self.span);
+        diag.code(error_code!(E0658));
+        diag
+    }
 }
 
 #[derive(Subdiagnostic)]
@@ -72,6 +82,12 @@ pub struct ProfileSampleUseFileDoesNotExist<'a> {
 pub struct TargetRequiresUnwindTables;
 
 #[derive(Diagnostic)]
+#[diag(session_instrumentation_not_supported)]
+pub struct InstrumentationNotSupported {
+    pub us: String,
+}
+
+#[derive(Diagnostic)]
 #[diag(session_sanitizer_not_supported)]
 pub struct SanitizerNotSupported {
     pub us: String,
@@ -316,11 +332,7 @@ pub fn report_lit_error(sess: &ParseSess, err: LitError, lit: token::Lit, span:
                 .take_while(|c| *c != 'i' && *c != 'u')
                 .all(|c| c.to_digit(base).is_some());
 
-        if valid {
-            Some(format!("0{}{}", base_char.to_ascii_lowercase(), &suffix[1..]))
-        } else {
-            None
-        }
+        valid.then(|| format!("0{}{}", base_char.to_ascii_lowercase(), &suffix[1..]))
     }
 
     let token::Lit { kind, symbol, suffix, .. } = lit;
@@ -375,3 +387,9 @@ pub fn report_lit_error(sess: &ParseSess, err: LitError, lit: token::Lit, span:
         }
     }
 }
+
+#[derive(Diagnostic)]
+#[diag(session_optimization_fuel_exhausted)]
+pub struct OptimisationFuelExhausted {
+    pub msg: String,
+}
diff --git a/compiler/rustc_session/src/filesearch.rs b/compiler/rustc_session/src/filesearch.rs
index b6a328908..f1fbf3821 100644
--- a/compiler/rustc_session/src/filesearch.rs
+++ b/compiler/rustc_session/src/filesearch.rs
@@ -182,7 +182,17 @@ pub fn get_or_default_sysroot() -> Result<PathBuf, String> {
         if dir.ends_with(crate::config::host_triple()) {
             dir.parent() // chop off `$target`
                 .and_then(|p| p.parent()) // chop off `rustlib`
-                .and_then(|p| p.parent()) // chop off `lib`
+                .and_then(|p| {
+                    // chop off `lib` (this could be also $arch dir if the host sysroot uses a
+                    // multi-arch layout like Debian or Ubuntu)
+                    match p.parent() {
+                        Some(p) => match p.file_name() {
+                            Some(f) if f == "lib" => p.parent(), // first chop went for $arch, so chop again for `lib`
+                            _ => Some(p),
+                        },
+                        None => None,
+                    }
+                })
                 .map(|s| s.to_owned())
                 .ok_or(format!(
                     "Could not move 3 levels upper using `parent()` on {}",
@@ -217,7 +227,7 @@ pub fn get_or_default_sysroot() -> Result<PathBuf, String> {
                 // Look for the target rustlib directory in the suspected sysroot.
                 let mut rustlib_path = rustc_target::target_rustlib_path(&p, "dummy");
                 rustlib_path.pop(); // pop off the dummy target.
-                if rustlib_path.exists() { Some(p) } else { None }
+                rustlib_path.exists().then_some(p)
             }
             None => None,
         }
diff --git a/compiler/rustc_session/src/lib.rs b/compiler/rustc_session/src/lib.rs
index 39e871f53..e1f1a5f6d 100644
--- a/compiler/rustc_session/src/lib.rs
+++ b/compiler/rustc_session/src/lib.rs
@@ -18,6 +18,9 @@ pub mod errors;
 #[macro_use]
 extern crate tracing;
 
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
+use rustc_macros::fluent_messages;
+
 pub mod cgu_reuse_tracker;
 pub mod utils;
 pub use lint::{declare_lint, declare_lint_pass, declare_tool_lint, impl_lint_pass};
@@ -39,6 +42,8 @@ pub mod output;
 
 pub use getopts;
 
+fluent_messages! { "../locales/en-US.ftl" }
+
 /// Requirements for a `StableHashingContext` to be used in this crate.
 /// This is a hack to allow using the `HashStable_Generic` derive macro
 /// instead of implementing everything in `rustc_middle`.
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index 7b5fd6cc2..b466a3fcd 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -4,7 +4,7 @@ use crate::early_error;
 use crate::lint;
 use crate::search_paths::SearchPath;
 use crate::utils::NativeLib;
-use rustc_errors::LanguageIdentifier;
+use rustc_errors::{LanguageIdentifier, TerminalUrl};
 use rustc_target::spec::{CodeModel, LinkerFlavorCli, MergeFunctions, PanicStrategy, SanitizerSet};
 use rustc_target::spec::{
     RelocModel, RelroLevel, SplitDebuginfo, StackProtector, TargetTriple, TlsModel,
@@ -169,6 +169,8 @@ top_level_options!(
         /// is currently just a hack and will be removed eventually, so please
         /// try to not rely on this too much.
         actually_rustdoc: bool [TRACKED],
+        /// Whether name resolver should resolve documentation links.
+        resolve_doc_links: ResolveDocLinks [TRACKED],
 
         /// Control path trimming.
         trimmed_def_paths: TrimmedDefPaths [TRACKED],
@@ -349,7 +351,7 @@ fn build_options<O: Default>(
 #[allow(non_upper_case_globals)]
 mod desc {
     pub const parse_no_flag: &str = "no value";
-    pub const parse_bool: &str = "one of: `y`, `yes`, `on`, `n`, `no`, or `off`";
+    pub const parse_bool: &str = "one of: `y`, `yes`, `on`, `true`, `n`, `no`, `off` or `false`";
     pub const parse_opt_bool: &str = parse_bool;
     pub const parse_string: &str = "a string";
     pub const parse_opt_string: &str = parse_string;
@@ -368,7 +370,7 @@ mod desc {
     pub const parse_opt_panic_strategy: &str = parse_panic_strategy;
     pub const parse_oom_strategy: &str = "either `panic` or `abort`";
     pub const parse_relro_level: &str = "one of: `full`, `partial`, or `off`";
-    pub const parse_sanitizers: &str = "comma separated list of sanitizers: `address`, `cfi`, `hwaddress`, `kcfi`, `leak`, `memory`, `memtag`, `shadow-call-stack`, or `thread`";
+    pub const parse_sanitizers: &str = "comma separated list of sanitizers: `address`, `cfi`, `hwaddress`, `kcfi`, `kernel-address`, `leak`, `memory`, `memtag`, `shadow-call-stack`, or `thread`";
     pub const parse_sanitizer_memory_track_origins: &str = "0, 1, or 2";
     pub const parse_cfguard: &str =
         "either a boolean (`yes`, `no`, `on`, `off`, etc), `checks`, or `nochecks`";
@@ -380,6 +382,7 @@ mod desc {
     pub const parse_dump_mono_stats: &str = "`markdown` (default) or `json`";
     pub const parse_instrument_coverage: &str =
         "`all` (default), `except-unused-generics`, `except-unused-functions`, or `off`";
+    pub const parse_instrument_xray: &str = "either a boolean (`yes`, `no`, `on`, `off`, etc), or a comma separated list of settings: `always` or `never` (mutually exclusive), `ignore-loops`, `instruction-threshold=N`, `skip-entry`, `skip-exit`";
     pub const parse_unpretty: &str = "`string` or `string=string`";
     pub const parse_treat_err_as_bug: &str = "either no value or a number bigger than 0";
     pub const parse_trait_solver: &str =
@@ -399,6 +402,8 @@ mod desc {
     pub const parse_code_model: &str = "one of supported code models (`rustc --print code-models`)";
     pub const parse_tls_model: &str = "one of supported TLS models (`rustc --print tls-models`)";
     pub const parse_target_feature: &str = parse_string;
+    pub const parse_terminal_url: &str =
+        "either a boolean (`yes`, `no`, `on`, `off`, etc), or `auto`";
     pub const parse_wasi_exec_model: &str = "either `command` or `reactor`";
     pub const parse_split_debuginfo: &str =
         "one of supported split-debuginfo modes (`off`, `packed`, or `unpacked`)";
@@ -432,11 +437,11 @@ mod parse {
     /// Use this for any boolean option that has a static default.
     pub(crate) fn parse_bool(slot: &mut bool, v: Option<&str>) -> bool {
         match v {
-            Some("y") | Some("yes") | Some("on") | None => {
+            Some("y") | Some("yes") | Some("on") | Some("true") | None => {
                 *slot = true;
                 true
             }
-            Some("n") | Some("no") | Some("off") => {
+            Some("n") | Some("no") | Some("off") | Some("false") => {
                 *slot = false;
                 true
             }
@@ -449,11 +454,11 @@ mod parse {
     /// other factors, such as other options, or target options.)
     pub(crate) fn parse_opt_bool(slot: &mut Option<bool>, v: Option<&str>) -> bool {
         match v {
-            Some("y") | Some("yes") | Some("on") | None => {
+            Some("y") | Some("yes") | Some("on") | Some("true") | None => {
                 *slot = Some(true);
                 true
             }
-            Some("n") | Some("no") | Some("off") => {
+            Some("n") | Some("no") | Some("off") | Some("false") => {
                 *slot = Some(false);
                 true
             }
@@ -679,6 +684,7 @@ mod parse {
                     "address" => SanitizerSet::ADDRESS,
                     "cfi" => SanitizerSet::CFI,
                     "kcfi" => SanitizerSet::KCFI,
+                    "kernel-address" => SanitizerSet::KERNELADDRESS,
                     "leak" => SanitizerSet::LEAK,
                     "memory" => SanitizerSet::MEMORY,
                     "memtag" => SanitizerSet::MEMTAG,
@@ -804,7 +810,7 @@ mod parse {
         if v.is_some() {
             let mut bool_arg = None;
             if parse_opt_bool(&mut bool_arg, v) {
-                *slot = if bool_arg.unwrap() { Some(MirSpanview::Statement) } else { None };
+                *slot = bool_arg.unwrap().then_some(MirSpanview::Statement);
                 return true;
             }
         }
@@ -845,7 +851,7 @@ mod parse {
         if v.is_some() {
             let mut bool_arg = None;
             if parse_opt_bool(&mut bool_arg, v) {
-                *slot = if bool_arg.unwrap() { Some(InstrumentCoverage::All) } else { None };
+                *slot = bool_arg.unwrap().then_some(InstrumentCoverage::All);
                 return true;
             }
         }
@@ -869,6 +875,68 @@ mod parse {
         true
     }
 
+    pub(crate) fn parse_instrument_xray(
+        slot: &mut Option<InstrumentXRay>,
+        v: Option<&str>,
+    ) -> bool {
+        if v.is_some() {
+            let mut bool_arg = None;
+            if parse_opt_bool(&mut bool_arg, v) {
+                *slot = if bool_arg.unwrap() { Some(InstrumentXRay::default()) } else { None };
+                return true;
+            }
+        }
+
+        let mut options = slot.get_or_insert_default();
+        let mut seen_always = false;
+        let mut seen_never = false;
+        let mut seen_ignore_loops = false;
+        let mut seen_instruction_threshold = false;
+        let mut seen_skip_entry = false;
+        let mut seen_skip_exit = false;
+        for option in v.into_iter().map(|v| v.split(',')).flatten() {
+            match option {
+                "always" if !seen_always && !seen_never => {
+                    options.always = true;
+                    options.never = false;
+                    seen_always = true;
+                }
+                "never" if !seen_never && !seen_always => {
+                    options.never = true;
+                    options.always = false;
+                    seen_never = true;
+                }
+                "ignore-loops" if !seen_ignore_loops => {
+                    options.ignore_loops = true;
+                    seen_ignore_loops = true;
+                }
+                option
+                    if option.starts_with("instruction-threshold")
+                        && !seen_instruction_threshold =>
+                {
+                    let Some(("instruction-threshold", n)) = option.split_once('=') else {
+                        return false;
+                    };
+                    match n.parse() {
+                        Ok(n) => options.instruction_threshold = Some(n),
+                        Err(_) => return false,
+                    }
+                    seen_instruction_threshold = true;
+                }
+                "skip-entry" if !seen_skip_entry => {
+                    options.skip_entry = true;
+                    seen_skip_entry = true;
+                }
+                "skip-exit" if !seen_skip_exit => {
+                    options.skip_exit = true;
+                    seen_skip_exit = true;
+                }
+                _ => return false,
+            }
+        }
+        true
+    }
+
     pub(crate) fn parse_treat_err_as_bug(slot: &mut Option<NonZeroUsize>, v: Option<&str>) -> bool {
         match v {
             Some(s) => {
@@ -979,6 +1047,16 @@ mod parse {
         true
     }
 
+    pub(crate) fn parse_terminal_url(slot: &mut TerminalUrl, v: Option<&str>) -> bool {
+        *slot = match v {
+            Some("on" | "" | "yes" | "y") | None => TerminalUrl::Yes,
+            Some("off" | "no" | "n") => TerminalUrl::No,
+            Some("auto") => TerminalUrl::Auto,
+            _ => return false,
+        };
+        true
+    }
+
     pub(crate) fn parse_symbol_mangling_version(
         slot: &mut Option<SymbolManglingVersion>,
         v: Option<&str>,
@@ -1290,6 +1368,8 @@ options! {
         (default: no)"),
     drop_tracking: bool = (false, parse_bool, [TRACKED],
         "enables drop tracking in generators (default: no)"),
+    drop_tracking_mir: bool = (false, parse_bool, [TRACKED],
+        "enables drop tracking on MIR in generators (default: no)"),
     dual_proc_macros: bool = (false, parse_bool, [TRACKED],
         "load proc macros for both target and host, but only link to the target (default: no)"),
     dump_dep_graph: bool = (false, parse_bool, [UNTRACKED],
@@ -1395,10 +1475,22 @@ options! {
         `=off` (default)"),
     instrument_mcount: bool = (false, parse_bool, [TRACKED],
         "insert function instrument code for mcount-based tracing (default: no)"),
+    instrument_xray: Option<InstrumentXRay> = (None, parse_instrument_xray, [TRACKED],
+        "insert function instrument code for XRay-based tracing (default: no)
+         Optional extra settings:
+         `=always`
+         `=never`
+         `=ignore-loops`
+         `=instruction-threshold=N`
+         `=skip-entry`
+         `=skip-exit`
+         Multiple options can be combined with commas."),
     keep_hygiene_data: bool = (false, parse_bool, [UNTRACKED],
         "keep hygiene data after analysis (default: no)"),
     layout_seed: Option<u64> = (None, parse_opt_number, [TRACKED],
         "seed layout randomization"),
+    link_directives: bool = (true, parse_bool, [TRACKED],
+        "honor #[link] directives in the compiled crate (default: yes)"),
     link_native_libraries: bool = (true, parse_bool, [UNTRACKED],
         "link native libraries in the linker invocation (default: yes)"),
     link_only: bool = (false, parse_bool, [TRACKED],
@@ -1411,8 +1503,9 @@ options! {
         "what location details should be tracked when using caller_location, either \
         `none`, or a comma separated list of location details, for which \
         valid options are `file`, `line`, and `column` (default: `file,line,column`)"),
-    log_backtrace: Option<String> = (None, parse_opt_string, [TRACKED],
-        "add a backtrace along with logging"),
+    lower_impl_trait_in_trait_to_assoc_ty: bool = (false, parse_bool, [TRACKED],
+        "modify the lowering strategy for `impl Trait` in traits so that they are lowered to \
+        generic associated types"),
     ls: bool = (false, parse_bool, [UNTRACKED],
         "list the symbols defined by a library crate (default: no)"),
     macro_backtrace: bool = (false, parse_bool, [UNTRACKED],
@@ -1479,8 +1572,6 @@ options! {
         "parse only; do not compile, assemble, or link (default: no)"),
     perf_stats: bool = (false, parse_bool, [UNTRACKED],
         "print some performance-related statistics (default: no)"),
-    pick_stable_methods_before_any_unstable: bool = (true, parse_bool, [TRACKED],
-        "try to pick stable methods first before picking any unstable methods (default: yes)"),
     plt: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "whether to use the PLT when calling into shared libraries;
         only has effect for PIC code on systems with ELF binaries
@@ -1542,9 +1633,6 @@ options! {
     saturating_float_casts: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "make float->int casts UB-free: numbers outside the integer type's range are clipped to \
         the max/min integer respectively, and NaN is mapped to 0 (default: yes)"),
-    save_analysis: bool = (false, parse_bool, [UNTRACKED],
-        "write syntax and type analysis (in JSON format) information, in \
-        addition to normal output (default: no)"),
     self_profile: SwitchWithOptPath = (SwitchWithOptPath::Disabled,
         parse_switch_with_opt_path, [UNTRACKED],
         "run the self profiler and output the raw event data"),
@@ -1600,6 +1688,8 @@ options! {
         "show extended diagnostic help (default: no)"),
     temps_dir: Option<String> = (None, parse_opt_string, [UNTRACKED],
         "the directory the intermediate files are written to"),
+    terminal_urls: TerminalUrl = (TerminalUrl::No, parse_terminal_url, [UNTRACKED],
+        "use the OSC 8 hyperlink terminal specification to print hyperlinks in the compiler output"),
     #[rustc_lint_opt_deny_field_access("use `Session::lto` instead of this field")]
     thinlto: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "enable ThinLTO when possible"),
@@ -1616,6 +1706,8 @@ options! {
         "measure time of each LLVM pass (default: no)"),
     time_passes: bool = (false, parse_bool, [UNTRACKED],
         "measure time of each rustc pass (default: no)"),
+    tiny_const_eval_limit: bool = (false, parse_bool, [TRACKED],
+        "sets a tiny, non-configurable limit for const eval; useful for compiler tests"),
     #[rustc_lint_opt_deny_field_access("use `Session::tls_model` instead of this field")]
     tls_model: Option<TlsModel> = (None, parse_tls_model, [TRACKED],
         "choose the TLS model to use (`rustc --print tls-models` for details)"),
diff --git a/compiler/rustc_session/src/parse.rs b/compiler/rustc_session/src/parse.rs
index 2aa8ca9e4..4e8c3f73e 100644
--- a/compiler/rustc_session/src/parse.rs
+++ b/compiler/rustc_session/src/parse.rs
@@ -88,7 +88,7 @@ pub fn feature_err<'a>(
     sess: &'a ParseSess,
     feature: Symbol,
     span: impl Into<MultiSpan>,
-    explain: &str,
+    explain: impl Into<DiagnosticMessage>,
 ) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
     feature_err_issue(sess, feature, span, GateIssue::Language, explain)
 }
@@ -103,7 +103,7 @@ pub fn feature_err_issue<'a>(
     feature: Symbol,
     span: impl Into<MultiSpan>,
     issue: GateIssue,
-    explain: &str,
+    explain: impl Into<DiagnosticMessage>,
 ) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
     let span = span.into();
 
@@ -114,7 +114,7 @@ pub fn feature_err_issue<'a>(
             .map(|err| err.cancel());
     }
 
-    let mut err = sess.create_err(FeatureGateError { span, explain });
+    let mut err = sess.create_err(FeatureGateError { span, explain: explain.into() });
     add_feature_diagnostics_for_issue(&mut err, sess, feature, issue);
     err
 }
@@ -226,8 +226,8 @@ pub struct ParseSess {
 
 impl ParseSess {
     /// Used for testing.
-    pub fn new(file_path_mapping: FilePathMapping) -> Self {
-        let fallback_bundle = fallback_fluent_bundle(rustc_errors::DEFAULT_LOCALE_RESOURCES, false);
+    pub fn new(locale_resources: Vec<&'static str>, file_path_mapping: FilePathMapping) -> Self {
+        let fallback_bundle = fallback_fluent_bundle(locale_resources, false);
         let sm = Lrc::new(SourceMap::new(file_path_mapping));
         let handler = Handler::with_tty_emitter(
             ColorConfig::Auto,
@@ -265,7 +265,7 @@ impl ParseSess {
     }
 
     pub fn with_silent_emitter(fatal_note: Option<String>) -> Self {
-        let fallback_bundle = fallback_fluent_bundle(rustc_errors::DEFAULT_LOCALE_RESOURCES, false);
+        let fallback_bundle = fallback_fluent_bundle(Vec::new(), false);
         let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
         let fatal_handler =
             Handler::with_tty_emitter(ColorConfig::Auto, false, None, None, None, fallback_bundle);
diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs
index 95f199de6..12634f671 100644
--- a/compiler/rustc_session/src/session.rs
+++ b/compiler/rustc_session/src/session.rs
@@ -1,16 +1,9 @@
 use crate::cgu_reuse_tracker::CguReuseTracker;
 use crate::code_stats::CodeStats;
-pub use crate::code_stats::{DataTypeKind, FieldInfo, SizeKind, VariantInfo};
+pub use crate::code_stats::{DataTypeKind, FieldInfo, FieldKind, SizeKind, VariantInfo};
 use crate::config::Input;
 use crate::config::{self, CrateType, InstrumentCoverage, OptLevel, OutputType, SwitchWithOptPath};
-use crate::errors::{
-    BranchProtectionRequiresAArch64, CannotEnableCrtStaticLinux, CannotMixAndMatchSanitizers,
-    LinkerPluginToWindowsNotSupported, NotCircumventFeature, ProfileSampleUseFileDoesNotExist,
-    ProfileUseFileDoesNotExist, SanitizerCfiEnabled, SanitizerNotSupported, SanitizersNotSupported,
-    SkippingConstChecks, SplitDebugInfoUnstablePlatform, StackProtectorNotSupportedForTarget,
-    TargetRequiresUnwindTables, UnleashedFeatureHelp, UnstableVirtualFunctionElimination,
-    UnsupportedDwarfVersion,
-};
+use crate::errors;
 use crate::parse::{add_feature_diagnostics, ParseSess};
 use crate::search_paths::{PathKind, SearchPath};
 use crate::{filesearch, lint};
@@ -31,6 +24,7 @@ use rustc_errors::registry::Registry;
 use rustc_errors::{
     error_code, fallback_fluent_bundle, DiagnosticBuilder, DiagnosticId, DiagnosticMessage,
     ErrorGuaranteed, FluentBundle, IntoDiagnostic, LazyFallbackBundle, MultiSpan, Noted,
+    TerminalUrl,
 };
 use rustc_macros::HashStable_Generic;
 pub use rustc_span::def_id::StableCrateId;
@@ -163,7 +157,7 @@ pub struct Session {
     /// `-C metadata` arguments passed to the compiler. Its value forms a unique
     /// global identifier for the crate. It is used to allow multiple crates
     /// with the same name to coexist. See the
-    /// `rustc_codegen_llvm::back::symbol_names` module for more information.
+    /// `rustc_symbol_mangling` crate for more information.
     pub stable_crate_id: OnceCell<StableCrateId>,
 
     features: OnceCell<rustc_feature::Features>,
@@ -245,15 +239,15 @@ impl Session {
         if !unleashed_features.is_empty() {
             let mut must_err = false;
             // Create a diagnostic pointing at where things got unleashed.
-            self.emit_warning(SkippingConstChecks {
+            self.emit_warning(errors::SkippingConstChecks {
                 unleashed_features: unleashed_features
                     .iter()
                     .map(|(span, gate)| {
                         gate.map(|gate| {
                             must_err = true;
-                            UnleashedFeatureHelp::Named { span: *span, gate }
+                            errors::UnleashedFeatureHelp::Named { span: *span, gate }
                         })
-                        .unwrap_or(UnleashedFeatureHelp::Unnamed { span: *span })
+                        .unwrap_or(errors::UnleashedFeatureHelp::Unnamed { span: *span })
                     })
                     .collect(),
             });
@@ -261,7 +255,7 @@ impl Session {
             // If we should err, make sure we did.
             if must_err && self.has_errors().is_none() {
                 // We have skipped a feature gate, and not run into other errors... reject.
-                self.emit_err(NotCircumventFeature);
+                self.emit_err(errors::NotCircumventFeature);
             }
         }
     }
@@ -483,6 +477,8 @@ impl Session {
         self.diagnostic().span_err_with_code(sp, msg, code)
     }
     #[rustc_lint_diagnostics]
+    #[allow(rustc::untranslatable_diagnostic)]
+    #[allow(rustc::diagnostic_outside_of_impl)]
     pub fn err(&self, msg: impl Into<DiagnosticMessage>) -> ErrorGuaranteed {
         self.diagnostic().err(msg)
     }
@@ -583,12 +579,16 @@ impl Session {
             ))
         }
     }
+
+    #[rustc_lint_diagnostics]
     #[allow(rustc::untranslatable_diagnostic)]
     #[allow(rustc::diagnostic_outside_of_impl)]
     #[track_caller]
     pub fn span_warn<S: Into<MultiSpan>>(&self, sp: S, msg: impl Into<DiagnosticMessage>) {
         self.diagnostic().span_warn(sp, msg)
     }
+
+    #[rustc_lint_diagnostics]
     #[allow(rustc::untranslatable_diagnostic)]
     #[allow(rustc::diagnostic_outside_of_impl)]
     pub fn span_warn_with_code<S: Into<MultiSpan>>(
@@ -599,6 +599,10 @@ impl Session {
     ) {
         self.diagnostic().span_warn_with_code(sp, msg, code)
     }
+
+    #[rustc_lint_diagnostics]
+    #[allow(rustc::untranslatable_diagnostic)]
+    #[allow(rustc::diagnostic_outside_of_impl)]
     pub fn warn(&self, msg: impl Into<DiagnosticMessage>) {
         self.diagnostic().warn(msg)
     }
@@ -641,11 +645,17 @@ impl Session {
         self.diagnostic().delay_good_path_bug(msg)
     }
 
+    #[rustc_lint_diagnostics]
+    #[allow(rustc::untranslatable_diagnostic)]
+    #[allow(rustc::diagnostic_outside_of_impl)]
     pub fn note_without_error(&self, msg: impl Into<DiagnosticMessage>) {
         self.diagnostic().note_without_error(msg)
     }
 
     #[track_caller]
+    #[rustc_lint_diagnostics]
+    #[allow(rustc::untranslatable_diagnostic)]
+    #[allow(rustc::diagnostic_outside_of_impl)]
     pub fn span_note_without_error<S: Into<MultiSpan>>(
         &self,
         sp: S,
@@ -653,6 +663,8 @@ impl Session {
     ) {
         self.diagnostic().span_note_without_error(sp, msg)
     }
+
+    #[rustc_lint_diagnostics]
     #[allow(rustc::untranslatable_diagnostic)]
     #[allow(rustc::diagnostic_outside_of_impl)]
     pub fn struct_note_without_error(
@@ -870,10 +882,14 @@ impl Session {
 
     /// We want to know if we're allowed to do an optimization for crate foo from -z fuel=foo=n.
     /// This expends fuel if applicable, and records fuel if applicable.
-    pub fn consider_optimizing<T: Fn() -> String>(&self, crate_name: &str, msg: T) -> bool {
+    pub fn consider_optimizing(
+        &self,
+        get_crate_name: impl Fn() -> Symbol,
+        msg: impl Fn() -> String,
+    ) -> bool {
         let mut ret = true;
         if let Some((ref c, _)) = self.opts.unstable_opts.fuel {
-            if c == crate_name {
+            if c == get_crate_name().as_str() {
                 assert_eq!(self.threads(), 1);
                 let mut fuel = self.optimization_fuel.lock();
                 ret = fuel.remaining != 0;
@@ -882,7 +898,7 @@ impl Session {
                         // We only call `msg` in case we can actually emit warnings.
                         // Otherwise, this could cause a `delay_good_path_bug` to
                         // trigger (issue #79546).
-                        self.warn(&format!("optimization-fuel-exhausted: {}", msg()));
+                        self.emit_warning(errors::OptimisationFuelExhausted { msg: msg() });
                     }
                     fuel.out_of_fuel = true;
                 } else if fuel.remaining > 0 {
@@ -891,7 +907,7 @@ impl Session {
             }
         }
         if let Some(ref c) = self.opts.unstable_opts.print_fuel {
-            if c == crate_name {
+            if c == get_crate_name().as_str() {
                 assert_eq!(self.threads(), 1);
                 self.print_fuel.fetch_add(1, SeqCst);
             }
@@ -899,23 +915,24 @@ impl Session {
         ret
     }
 
-    pub fn rust_2015(&self) -> bool {
-        self.edition() == Edition::Edition2015
+    /// Is this edition 2015?
+    pub fn is_rust_2015(&self) -> bool {
+        self.edition().is_rust_2015()
     }
 
     /// Are we allowed to use features from the Rust 2018 edition?
     pub fn rust_2018(&self) -> bool {
-        self.edition() >= Edition::Edition2018
+        self.edition().rust_2018()
     }
 
     /// Are we allowed to use features from the Rust 2021 edition?
     pub fn rust_2021(&self) -> bool {
-        self.edition() >= Edition::Edition2021
+        self.edition().rust_2021()
     }
 
     /// Are we allowed to use features from the Rust 2024 edition?
     pub fn rust_2024(&self) -> bool {
-        self.edition() >= Edition::Edition2024
+        self.edition().rust_2024()
     }
 
     /// Returns `true` if we cannot skip the PLT for shared library calls.
@@ -941,10 +958,10 @@ impl Session {
     /// Checks if LLVM lifetime markers should be emitted.
     pub fn emit_lifetime_markers(&self) -> bool {
         self.opts.optimize != config::OptLevel::No
-        // AddressSanitizer uses lifetimes to detect use after scope bugs.
+        // AddressSanitizer and KernelAddressSanitizer uses lifetimes to detect use after scope bugs.
         // MemorySanitizer uses lifetimes to detect use of uninitialized stack variables.
         // HWAddressSanitizer will use lifetimes to detect use after scope bugs in the future.
-        || self.opts.unstable_opts.sanitizer.intersects(SanitizerSet::ADDRESS | SanitizerSet::MEMORY | SanitizerSet::HWADDRESS)
+        || self.opts.unstable_opts.sanitizer.intersects(SanitizerSet::ADDRESS | SanitizerSet::KERNELADDRESS | SanitizerSet::MEMORY | SanitizerSet::HWADDRESS)
     }
 
     pub fn is_proc_macro_attr(&self, attr: &Attribute) -> bool {
@@ -1261,6 +1278,19 @@ fn default_emitter(
 ) -> Box<dyn Emitter + sync::Send> {
     let macro_backtrace = sopts.unstable_opts.macro_backtrace;
     let track_diagnostics = sopts.unstable_opts.track_diagnostics;
+    let terminal_url = match sopts.unstable_opts.terminal_urls {
+        TerminalUrl::Auto => {
+            match (std::env::var("COLORTERM").as_deref(), std::env::var("TERM").as_deref()) {
+                (Ok("truecolor"), Ok("xterm-256color"))
+                    if sopts.unstable_features.is_nightly_build() =>
+                {
+                    TerminalUrl::Yes
+                }
+                _ => TerminalUrl::No,
+            }
+        }
+        t => t,
+    };
     match sopts.error_format {
         config::ErrorOutputType::HumanReadable(kind) => {
             let (short, color_config) = kind.unzip();
@@ -1285,6 +1315,7 @@ fn default_emitter(
                     sopts.diagnostic_width,
                     macro_backtrace,
                     track_diagnostics,
+                    terminal_url,
                 );
                 Box::new(emitter.ui_testing(sopts.unstable_opts.ui_testing))
             }
@@ -1300,6 +1331,7 @@ fn default_emitter(
                 sopts.diagnostic_width,
                 macro_backtrace,
                 track_diagnostics,
+                terminal_url,
             )
             .ui_testing(sopts.unstable_opts.ui_testing),
         ),
@@ -1313,6 +1345,7 @@ pub fn build_session(
     io: CompilerIO,
     bundle: Option<Lrc<rustc_errors::FluentBundle>>,
     registry: rustc_errors::registry::Registry,
+    fluent_resources: Vec<&'static str>,
     driver_lint_caps: FxHashMap<lint::LintId, lint::Level>,
     file_loader: Option<Box<dyn FileLoader + Send + Sync + 'static>>,
     target_override: Option<Target>,
@@ -1357,7 +1390,7 @@ pub fn build_session(
     ));
 
     let fallback_bundle = fallback_fluent_bundle(
-        rustc_errors::DEFAULT_LOCALE_RESOURCES,
+        fluent_resources,
         sopts.unstable_opts.translate_directionality_markers,
     );
     let emitter = default_emitter(&sopts, registry, source_map.clone(), bundle, fallback_bundle);
@@ -1482,28 +1515,28 @@ fn validate_commandline_args_with_session_available(sess: &Session) {
         && sess.opts.cg.prefer_dynamic
         && sess.target.is_like_windows
     {
-        sess.emit_err(LinkerPluginToWindowsNotSupported);
+        sess.emit_err(errors::LinkerPluginToWindowsNotSupported);
     }
 
     // Make sure that any given profiling data actually exists so LLVM can't
     // decide to silently skip PGO.
     if let Some(ref path) = sess.opts.cg.profile_use {
         if !path.exists() {
-            sess.emit_err(ProfileUseFileDoesNotExist { path });
+            sess.emit_err(errors::ProfileUseFileDoesNotExist { path });
         }
     }
 
     // Do the same for sample profile data.
     if let Some(ref path) = sess.opts.unstable_opts.profile_sample_use {
         if !path.exists() {
-            sess.emit_err(ProfileSampleUseFileDoesNotExist { path });
+            sess.emit_err(errors::ProfileSampleUseFileDoesNotExist { path });
         }
     }
 
     // Unwind tables cannot be disabled if the target requires them.
     if let Some(include_uwtables) = sess.opts.cg.force_unwind_tables {
         if sess.target.requires_uwtable && !include_uwtables {
-            sess.emit_err(TargetRequiresUnwindTables);
+            sess.emit_err(errors::TargetRequiresUnwindTables);
         }
     }
 
@@ -1513,16 +1546,18 @@ fn validate_commandline_args_with_session_available(sess: &Session) {
     match unsupported_sanitizers.into_iter().count() {
         0 => {}
         1 => {
-            sess.emit_err(SanitizerNotSupported { us: unsupported_sanitizers.to_string() });
+            sess.emit_err(errors::SanitizerNotSupported { us: unsupported_sanitizers.to_string() });
         }
         _ => {
-            sess.emit_err(SanitizersNotSupported { us: unsupported_sanitizers.to_string() });
+            sess.emit_err(errors::SanitizersNotSupported {
+                us: unsupported_sanitizers.to_string(),
+            });
         }
     }
     // Cannot mix and match sanitizers.
     let mut sanitizer_iter = sess.opts.unstable_opts.sanitizer.into_iter();
     if let (Some(first), Some(second)) = (sanitizer_iter.next(), sanitizer_iter.next()) {
-        sess.emit_err(CannotMixAndMatchSanitizers {
+        sess.emit_err(errors::CannotMixAndMatchSanitizers {
             first: first.to_string(),
             second: second.to_string(),
         });
@@ -1530,22 +1565,22 @@ fn validate_commandline_args_with_session_available(sess: &Session) {
 
     // Cannot enable crt-static with sanitizers on Linux
     if sess.crt_static(None) && !sess.opts.unstable_opts.sanitizer.is_empty() {
-        sess.emit_err(CannotEnableCrtStaticLinux);
+        sess.emit_err(errors::CannotEnableCrtStaticLinux);
     }
 
     // LLVM CFI and VFE both require LTO.
     if sess.lto() != config::Lto::Fat {
         if sess.is_sanitizer_cfi_enabled() {
-            sess.emit_err(SanitizerCfiEnabled);
+            sess.emit_err(errors::SanitizerCfiEnabled);
         }
         if sess.opts.unstable_opts.virtual_function_elimination {
-            sess.emit_err(UnstableVirtualFunctionElimination);
+            sess.emit_err(errors::UnstableVirtualFunctionElimination);
         }
     }
 
     // LLVM CFI and KCFI are mutually exclusive
     if sess.is_sanitizer_cfi_enabled() && sess.is_sanitizer_kcfi_enabled() {
-        sess.emit_err(CannotMixAndMatchSanitizers {
+        sess.emit_err(errors::CannotMixAndMatchSanitizers {
             first: "cfi".to_string(),
             second: "kcfi".to_string(),
         });
@@ -1553,7 +1588,7 @@ fn validate_commandline_args_with_session_available(sess: &Session) {
 
     if sess.opts.unstable_opts.stack_protector != StackProtector::None {
         if !sess.target.options.supports_stack_protector {
-            sess.emit_warning(StackProtectorNotSupportedForTarget {
+            sess.emit_warning(errors::StackProtectorNotSupportedForTarget {
                 stack_protector: sess.opts.unstable_opts.stack_protector,
                 target_triple: &sess.opts.target_triple,
             });
@@ -1561,19 +1596,23 @@ fn validate_commandline_args_with_session_available(sess: &Session) {
     }
 
     if sess.opts.unstable_opts.branch_protection.is_some() && sess.target.arch != "aarch64" {
-        sess.emit_err(BranchProtectionRequiresAArch64);
+        sess.emit_err(errors::BranchProtectionRequiresAArch64);
     }
 
     if let Some(dwarf_version) = sess.opts.unstable_opts.dwarf_version {
         if dwarf_version > 5 {
-            sess.emit_err(UnsupportedDwarfVersion { dwarf_version });
+            sess.emit_err(errors::UnsupportedDwarfVersion { dwarf_version });
         }
     }
 
     if !sess.target.options.supported_split_debuginfo.contains(&sess.split_debuginfo())
         && !sess.opts.unstable_opts.unstable_options
     {
-        sess.emit_err(SplitDebugInfoUnstablePlatform { debuginfo: sess.split_debuginfo() });
+        sess.emit_err(errors::SplitDebugInfoUnstablePlatform { debuginfo: sess.split_debuginfo() });
+    }
+
+    if sess.opts.unstable_opts.instrument_xray.is_some() && !sess.target.options.supports_xray {
+        sess.emit_err(errors::InstrumentationNotSupported { us: "XRay".to_string() });
     }
 }
 
@@ -1596,7 +1635,10 @@ pub enum IncrCompSession {
 }
 
 fn early_error_handler(output: config::ErrorOutputType) -> rustc_errors::Handler {
-    let fallback_bundle = fallback_fluent_bundle(rustc_errors::DEFAULT_LOCALE_RESOURCES, false);
+    // FIXME(#100717): early errors aren't translated at the moment, so this is fine, but it will
+    // need to reference every crate that might emit an early error for translation to work.
+    let fallback_bundle =
+        fallback_fluent_bundle(vec![rustc_errors::DEFAULT_LOCALE_RESOURCE], false);
     let emitter: Box<dyn Emitter + sync::Send> = match output {
         config::ErrorOutputType::HumanReadable(kind) => {
             let (short, color_config) = kind.unzip();
@@ -1610,6 +1652,7 @@ fn early_error_handler(output: config::ErrorOutputType) -> rustc_errors::Handler
                 None,
                 false,
                 false,
+                TerminalUrl::No,
             ))
         }
         config::ErrorOutputType::Json { pretty, json_rendered } => Box::new(JsonEmitter::basic(
@@ -1620,6 +1663,7 @@ fn early_error_handler(output: config::ErrorOutputType) -> rustc_errors::Handler
             None,
             false,
             false,
+            TerminalUrl::No,
         )),
     };
     rustc_errors::Handler::with_emitter(true, None, emitter)
diff --git a/compiler/rustc_span/src/def_id.rs b/compiler/rustc_span/src/def_id.rs
index 7c5e1427d..162c15574 100644
--- a/compiler/rustc_span/src/def_id.rs
+++ b/compiler/rustc_span/src/def_id.rs
@@ -119,6 +119,12 @@ impl DefPathHash {
     }
 }
 
+impl Default for DefPathHash {
+    fn default() -> Self {
+        DefPathHash(Fingerprint::ZERO)
+    }
+}
+
 impl Borrow<Fingerprint> for DefPathHash {
     #[inline]
     fn borrow(&self) -> &Fingerprint {
@@ -229,7 +235,7 @@ impl<D: Decoder> Decodable<D> for DefIndex {
 pub struct DefId {
     // cfg-ing the order of fields so that the `DefIndex` which is high entropy always ends up in
     // the lower bits no matter the endianness. This allows the compiler to turn that `Hash` impl
-    // into a direct call to 'u64::hash(_)`.
+    // into a direct call to `u64::hash(_)`.
     #[cfg(not(all(target_pointer_width = "64", target_endian = "big")))]
     pub index: DefIndex,
     pub krate: CrateNum,
@@ -293,7 +299,7 @@ impl DefId {
 
     #[inline]
     pub fn as_local(self) -> Option<LocalDefId> {
-        if self.is_local() { Some(LocalDefId { local_def_index: self.index }) } else { None }
+        self.is_local().then(|| LocalDefId { local_def_index: self.index })
     }
 
     #[inline]
@@ -314,7 +320,7 @@ impl DefId {
 
     #[inline]
     pub fn as_crate_root(self) -> Option<CrateNum> {
-        if self.is_crate_root() { Some(self.krate) } else { None }
+        self.is_crate_root().then_some(self.krate)
     }
 
     #[inline]
diff --git a/compiler/rustc_span/src/edit_distance.rs b/compiler/rustc_span/src/edit_distance.rs
new file mode 100644
index 000000000..89f0386e3
--- /dev/null
+++ b/compiler/rustc_span/src/edit_distance.rs
@@ -0,0 +1,229 @@
+//! Edit distances.
+//!
+//! The [edit distance] is a metric for measuring the difference between two strings.
+//!
+//! [edit distance]: https://en.wikipedia.org/wiki/Edit_distance
+
+// The current implementation is the restricted Damerau-Levenshtein algorithm. It is restricted
+// because it does not permit modifying characters that have already been transposed. The specific
+// algorithm should not matter to the caller of the methods, which is why it is not noted in the
+// documentation.
+
+use crate::symbol::Symbol;
+use std::{cmp, mem};
+
+#[cfg(test)]
+mod tests;
+
+/// Finds the [edit distance] between two strings.
+///
+/// Returns `None` if the distance exceeds the limit.
+///
+/// [edit distance]: https://en.wikipedia.org/wiki/Edit_distance
+pub fn edit_distance(a: &str, b: &str, limit: usize) -> Option<usize> {
+    let mut a = &a.chars().collect::<Vec<_>>()[..];
+    let mut b = &b.chars().collect::<Vec<_>>()[..];
+
+    // Ensure that `b` is the shorter string, minimizing memory use.
+    if a.len() < b.len() {
+        mem::swap(&mut a, &mut b);
+    }
+
+    let min_dist = a.len() - b.len();
+    // If we know the limit will be exceeded, we can return early.
+    if min_dist > limit {
+        return None;
+    }
+
+    // Strip common prefix.
+    while let Some(((b_char, b_rest), (a_char, a_rest))) = b.split_first().zip(a.split_first())
+        && a_char == b_char
+    {
+        a = a_rest;
+        b = b_rest;
+    }
+    // Strip common suffix.
+    while let Some(((b_char, b_rest), (a_char, a_rest))) = b.split_last().zip(a.split_last())
+        && a_char == b_char
+    {
+        a = a_rest;
+        b = b_rest;
+    }
+
+    // If either string is empty, the distance is the length of the other.
+    // We know that `b` is the shorter string, so we don't need to check `a`.
+    if b.len() == 0 {
+        return Some(min_dist);
+    }
+
+    let mut prev_prev = vec![usize::MAX; b.len() + 1];
+    let mut prev = (0..=b.len()).collect::<Vec<_>>();
+    let mut current = vec![0; b.len() + 1];
+
+    // row by row
+    for i in 1..=a.len() {
+        current[0] = i;
+        let a_idx = i - 1;
+
+        // column by column
+        for j in 1..=b.len() {
+            let b_idx = j - 1;
+
+            // There is no cost to substitute a character with itself.
+            let substitution_cost = if a[a_idx] == b[b_idx] { 0 } else { 1 };
+
+            current[j] = cmp::min(
+                // deletion
+                prev[j] + 1,
+                cmp::min(
+                    // insertion
+                    current[j - 1] + 1,
+                    // substitution
+                    prev[j - 1] + substitution_cost,
+                ),
+            );
+
+            if (i > 1) && (j > 1) && (a[a_idx] == b[b_idx - 1]) && (a[a_idx - 1] == b[b_idx]) {
+                // transposition
+                current[j] = cmp::min(current[j], prev_prev[j - 2] + 1);
+            }
+        }
+
+        // Rotate the buffers, reusing the memory.
+        [prev_prev, prev, current] = [prev, current, prev_prev];
+    }
+
+    // `prev` because we already rotated the buffers.
+    let distance = prev[b.len()];
+    (distance <= limit).then_some(distance)
+}
+
+/// Provides a word similarity score between two words that accounts for substrings being more
+/// meaningful than a typical edit distance. The lower the score, the closer the match. 0 is an
+/// identical match.
+///
+/// Uses the edit distance between the two strings and removes the cost of the length difference.
+/// If this is 0 then it is either a substring match or a full word match, in the substring match
+/// case we detect this and return `1`. To prevent finding meaningless substrings, eg. "in" in
+/// "shrink", we only perform this subtraction of length difference if one of the words is not
+/// greater than twice the length of the other. For cases where the words are close in size but not
+/// an exact substring then the cost of the length difference is discounted by half.
+///
+/// Returns `None` if the distance exceeds the limit.
+pub fn edit_distance_with_substrings(a: &str, b: &str, limit: usize) -> Option<usize> {
+    let n = a.chars().count();
+    let m = b.chars().count();
+
+    // Check one isn't less than half the length of the other. If this is true then there is a
+    // big difference in length.
+    let big_len_diff = (n * 2) < m || (m * 2) < n;
+    let len_diff = if n < m { m - n } else { n - m };
+    let distance = edit_distance(a, b, limit + len_diff)?;
+
+    // This is the crux, subtracting length difference means exact substring matches will now be 0
+    let score = distance - len_diff;
+
+    // If the score is 0 but the words have different lengths then it's a substring match not a full
+    // word match
+    let score = if score == 0 && len_diff > 0 && !big_len_diff {
+        1 // Exact substring match, but not a total word match so return non-zero
+    } else if !big_len_diff {
+        // Not a big difference in length, discount cost of length difference
+        score + (len_diff + 1) / 2
+    } else {
+        // A big difference in length, add back the difference in length to the score
+        score + len_diff
+    };
+
+    (score <= limit).then_some(score)
+}
+
+/// Finds the best match for given word in the given iterator where substrings are meaningful.
+///
+/// A version of [`find_best_match_for_name`] that uses [`edit_distance_with_substrings`] as the
+/// score for word similarity. This takes an optional distance limit which defaults to one-third of
+/// the given word.
+///
+/// We use case insensitive comparison to improve accuracy on an edge case with a lower(upper)case
+/// letters mismatch.
+pub fn find_best_match_for_name_with_substrings(
+    candidates: &[Symbol],
+    lookup: Symbol,
+    dist: Option<usize>,
+) -> Option<Symbol> {
+    find_best_match_for_name_impl(true, candidates, lookup, dist)
+}
+
+/// Finds the best match for a given word in the given iterator.
+///
+/// As a loose rule to avoid the obviously incorrect suggestions, it takes
+/// an optional limit for the maximum allowable edit distance, which defaults
+/// to one-third of the given word.
+///
+/// We use case insensitive comparison to improve accuracy on an edge case with a lower(upper)case
+/// letters mismatch.
+pub fn find_best_match_for_name(
+    candidates: &[Symbol],
+    lookup: Symbol,
+    dist: Option<usize>,
+) -> Option<Symbol> {
+    find_best_match_for_name_impl(false, candidates, lookup, dist)
+}
+
+#[cold]
+fn find_best_match_for_name_impl(
+    use_substring_score: bool,
+    candidates: &[Symbol],
+    lookup: Symbol,
+    dist: Option<usize>,
+) -> Option<Symbol> {
+    let lookup = lookup.as_str();
+    let lookup_uppercase = lookup.to_uppercase();
+
+    // Priority of matches:
+    // 1. Exact case insensitive match
+    // 2. Edit distance match
+    // 3. Sorted word match
+    if let Some(c) = candidates.iter().find(|c| c.as_str().to_uppercase() == lookup_uppercase) {
+        return Some(*c);
+    }
+
+    let mut dist = dist.unwrap_or_else(|| cmp::max(lookup.len(), 3) / 3);
+    let mut best = None;
+    for c in candidates {
+        match if use_substring_score {
+            edit_distance_with_substrings(lookup, c.as_str(), dist)
+        } else {
+            edit_distance(lookup, c.as_str(), dist)
+        } {
+            Some(0) => return Some(*c),
+            Some(d) => {
+                dist = d - 1;
+                best = Some(*c);
+            }
+            None => {}
+        }
+    }
+    if best.is_some() {
+        return best;
+    }
+
+    find_match_by_sorted_words(candidates, lookup)
+}
+
+fn find_match_by_sorted_words(iter_names: &[Symbol], lookup: &str) -> Option<Symbol> {
+    iter_names.iter().fold(None, |result, candidate| {
+        if sort_by_words(candidate.as_str()) == sort_by_words(lookup) {
+            Some(*candidate)
+        } else {
+            result
+        }
+    })
+}
+
+fn sort_by_words(name: &str) -> String {
+    let mut split_words: Vec<&str> = name.split('_').collect();
+    // We are sorting primitive &strs and can use unstable sort here.
+    split_words.sort_unstable();
+    split_words.join("_")
+}
diff --git a/compiler/rustc_span/src/edit_distance/tests.rs b/compiler/rustc_span/src/edit_distance/tests.rs
new file mode 100644
index 000000000..c9c7a1f1b
--- /dev/null
+++ b/compiler/rustc_span/src/edit_distance/tests.rs
@@ -0,0 +1,80 @@
+use super::*;
+
+#[test]
+fn test_edit_distance() {
+    // Test bytelength agnosticity
+    for c in (0..char::MAX as u32).filter_map(char::from_u32).map(|i| i.to_string()) {
+        assert_eq!(edit_distance(&c[..], &c[..], usize::MAX), Some(0));
+    }
+
+    let a = "\nMäry häd ä little lämb\n\nLittle lämb\n";
+    let b = "\nMary häd ä little lämb\n\nLittle lämb\n";
+    let c = "Mary häd ä little lämb\n\nLittle lämb\n";
+    assert_eq!(edit_distance(a, b, usize::MAX), Some(1));
+    assert_eq!(edit_distance(b, a, usize::MAX), Some(1));
+    assert_eq!(edit_distance(a, c, usize::MAX), Some(2));
+    assert_eq!(edit_distance(c, a, usize::MAX), Some(2));
+    assert_eq!(edit_distance(b, c, usize::MAX), Some(1));
+    assert_eq!(edit_distance(c, b, usize::MAX), Some(1));
+}
+
+#[test]
+fn test_edit_distance_limit() {
+    assert_eq!(edit_distance("abc", "abcd", 1), Some(1));
+    assert_eq!(edit_distance("abc", "abcd", 0), None);
+    assert_eq!(edit_distance("abc", "xyz", 3), Some(3));
+    assert_eq!(edit_distance("abc", "xyz", 2), None);
+}
+
+#[test]
+fn test_method_name_similarity_score() {
+    assert_eq!(edit_distance_with_substrings("empty", "is_empty", 1), Some(1));
+    assert_eq!(edit_distance_with_substrings("shrunk", "rchunks", 2), None);
+    assert_eq!(edit_distance_with_substrings("abc", "abcd", 1), Some(1));
+    assert_eq!(edit_distance_with_substrings("a", "abcd", 1), None);
+    assert_eq!(edit_distance_with_substrings("edf", "eq", 1), None);
+    assert_eq!(edit_distance_with_substrings("abc", "xyz", 3), Some(3));
+    assert_eq!(edit_distance_with_substrings("abcdef", "abcdef", 2), Some(0));
+}
+
+#[test]
+fn test_find_best_match_for_name() {
+    use crate::create_default_session_globals_then;
+    create_default_session_globals_then(|| {
+        let input = vec![Symbol::intern("aaab"), Symbol::intern("aaabc")];
+        assert_eq!(
+            find_best_match_for_name(&input, Symbol::intern("aaaa"), None),
+            Some(Symbol::intern("aaab"))
+        );
+
+        assert_eq!(find_best_match_for_name(&input, Symbol::intern("1111111111"), None), None);
+
+        let input = vec![Symbol::intern("AAAA")];
+        assert_eq!(
+            find_best_match_for_name(&input, Symbol::intern("aaaa"), None),
+            Some(Symbol::intern("AAAA"))
+        );
+
+        let input = vec![Symbol::intern("AAAA")];
+        assert_eq!(
+            find_best_match_for_name(&input, Symbol::intern("aaaa"), Some(4)),
+            Some(Symbol::intern("AAAA"))
+        );
+
+        let input = vec![Symbol::intern("a_longer_variable_name")];
+        assert_eq!(
+            find_best_match_for_name(&input, Symbol::intern("a_variable_longer_name"), None),
+            Some(Symbol::intern("a_longer_variable_name"))
+        );
+    })
+}
+
+#[test]
+fn test_precise_algorithm() {
+    // Not Levenshtein distance.
+    assert_ne!(edit_distance("ab", "ba", usize::MAX), Some(2));
+    // Not unrestricted Damerau-Levenshtein distance.
+    assert_ne!(edit_distance("abde", "bcaed", usize::MAX), Some(3));
+    // The current implementation is a restricted Damerau-Levenshtein distance.
+    assert_eq!(edit_distance("abde", "bcaed", usize::MAX), Some(4));
+}
diff --git a/compiler/rustc_span/src/edition.rs b/compiler/rustc_span/src/edition.rs
index b43183916..f16db69aa 100644
--- a/compiler/rustc_span/src/edition.rs
+++ b/compiler/rustc_span/src/edition.rs
@@ -49,8 +49,8 @@ impl fmt::Display for Edition {
 }
 
 impl Edition {
-    pub fn lint_name(&self) -> &'static str {
-        match *self {
+    pub fn lint_name(self) -> &'static str {
+        match self {
             Edition::Edition2015 => "rust_2015_compatibility",
             Edition::Edition2018 => "rust_2018_compatibility",
             Edition::Edition2021 => "rust_2021_compatibility",
@@ -58,8 +58,8 @@ impl Edition {
         }
     }
 
-    pub fn feature_name(&self) -> Symbol {
-        match *self {
+    pub fn feature_name(self) -> Symbol {
+        match self {
             Edition::Edition2015 => sym::rust_2015_preview,
             Edition::Edition2018 => sym::rust_2018_preview,
             Edition::Edition2021 => sym::rust_2021_preview,
@@ -67,8 +67,8 @@ impl Edition {
         }
     }
 
-    pub fn is_stable(&self) -> bool {
-        match *self {
+    pub fn is_stable(self) -> bool {
+        match self {
             Edition::Edition2015 => true,
             Edition::Edition2018 => true,
             Edition::Edition2021 => true,
@@ -76,23 +76,24 @@ impl Edition {
         }
     }
 
-    pub fn rust_2015(&self) -> bool {
-        *self == Edition::Edition2015
+    /// Is this edition 2015?
+    pub fn is_rust_2015(self) -> bool {
+        self == Edition::Edition2015
     }
 
     /// Are we allowed to use features from the Rust 2018 edition?
-    pub fn rust_2018(&self) -> bool {
-        *self >= Edition::Edition2018
+    pub fn rust_2018(self) -> bool {
+        self >= Edition::Edition2018
     }
 
     /// Are we allowed to use features from the Rust 2021 edition?
-    pub fn rust_2021(&self) -> bool {
-        *self >= Edition::Edition2021
+    pub fn rust_2021(self) -> bool {
+        self >= Edition::Edition2021
     }
 
     /// Are we allowed to use features from the Rust 2024 edition?
-    pub fn rust_2024(&self) -> bool {
-        *self >= Edition::Edition2024
+    pub fn rust_2024(self) -> bool {
+        self >= Edition::Edition2024
     }
 }
 
diff --git a/compiler/rustc_span/src/lev_distance.rs b/compiler/rustc_span/src/lev_distance.rs
deleted file mode 100644
index 61e4b98a8..000000000
--- a/compiler/rustc_span/src/lev_distance.rs
+++ /dev/null
@@ -1,177 +0,0 @@
-//! Levenshtein distances.
-//!
-//! The [Levenshtein distance] is a metric for measuring the difference between two strings.
-//!
-//! [Levenshtein distance]: https://en.wikipedia.org/wiki/Levenshtein_distance
-
-use crate::symbol::Symbol;
-use std::cmp;
-
-#[cfg(test)]
-mod tests;
-
-/// Finds the Levenshtein distance between two strings.
-///
-/// Returns None if the distance exceeds the limit.
-pub fn lev_distance(a: &str, b: &str, limit: usize) -> Option<usize> {
-    let n = a.chars().count();
-    let m = b.chars().count();
-    let min_dist = if n < m { m - n } else { n - m };
-
-    if min_dist > limit {
-        return None;
-    }
-    if n == 0 || m == 0 {
-        return (min_dist <= limit).then_some(min_dist);
-    }
-
-    let mut dcol: Vec<_> = (0..=m).collect();
-
-    for (i, sc) in a.chars().enumerate() {
-        let mut current = i;
-        dcol[0] = current + 1;
-
-        for (j, tc) in b.chars().enumerate() {
-            let next = dcol[j + 1];
-            if sc == tc {
-                dcol[j + 1] = current;
-            } else {
-                dcol[j + 1] = cmp::min(current, next);
-                dcol[j + 1] = cmp::min(dcol[j + 1], dcol[j]) + 1;
-            }
-            current = next;
-        }
-    }
-
-    (dcol[m] <= limit).then_some(dcol[m])
-}
-
-/// Provides a word similarity score between two words that accounts for substrings being more
-/// meaningful than a typical Levenshtein distance. The lower the score, the closer the match.
-/// 0 is an identical match.
-///
-/// Uses the Levenshtein distance between the two strings and removes the cost of the length
-/// difference. If this is 0 then it is either a substring match or a full word match, in the
-/// substring match case we detect this and return `1`. To prevent finding meaningless substrings,
-/// eg. "in" in "shrink", we only perform this subtraction of length difference if one of the words
-/// is not greater than twice the length of the other. For cases where the words are close in size
-/// but not an exact substring then the cost of the length difference is discounted by half.
-///
-/// Returns `None` if the distance exceeds the limit.
-pub fn lev_distance_with_substrings(a: &str, b: &str, limit: usize) -> Option<usize> {
-    let n = a.chars().count();
-    let m = b.chars().count();
-
-    // Check one isn't less than half the length of the other. If this is true then there is a
-    // big difference in length.
-    let big_len_diff = (n * 2) < m || (m * 2) < n;
-    let len_diff = if n < m { m - n } else { n - m };
-    let lev = lev_distance(a, b, limit + len_diff)?;
-
-    // This is the crux, subtracting length difference means exact substring matches will now be 0
-    let score = lev - len_diff;
-
-    // If the score is 0 but the words have different lengths then it's a substring match not a full
-    // word match
-    let score = if score == 0 && len_diff > 0 && !big_len_diff {
-        1 // Exact substring match, but not a total word match so return non-zero
-    } else if !big_len_diff {
-        // Not a big difference in length, discount cost of length difference
-        score + (len_diff + 1) / 2
-    } else {
-        // A big difference in length, add back the difference in length to the score
-        score + len_diff
-    };
-
-    (score <= limit).then_some(score)
-}
-
-/// Finds the best match for given word in the given iterator where substrings are meaningful.
-///
-/// A version of [`find_best_match_for_name`] that uses [`lev_distance_with_substrings`] as the score
-/// for word similarity. This takes an optional distance limit which defaults to one-third of the
-/// given word.
-///
-/// Besides the modified Levenshtein, we use case insensitive comparison to improve accuracy
-/// on an edge case with a lower(upper)case letters mismatch.
-pub fn find_best_match_for_name_with_substrings(
-    candidates: &[Symbol],
-    lookup: Symbol,
-    dist: Option<usize>,
-) -> Option<Symbol> {
-    find_best_match_for_name_impl(true, candidates, lookup, dist)
-}
-
-/// Finds the best match for a given word in the given iterator.
-///
-/// As a loose rule to avoid the obviously incorrect suggestions, it takes
-/// an optional limit for the maximum allowable edit distance, which defaults
-/// to one-third of the given word.
-///
-/// Besides Levenshtein, we use case insensitive comparison to improve accuracy
-/// on an edge case with a lower(upper)case letters mismatch.
-pub fn find_best_match_for_name(
-    candidates: &[Symbol],
-    lookup: Symbol,
-    dist: Option<usize>,
-) -> Option<Symbol> {
-    find_best_match_for_name_impl(false, candidates, lookup, dist)
-}
-
-#[cold]
-fn find_best_match_for_name_impl(
-    use_substring_score: bool,
-    candidates: &[Symbol],
-    lookup: Symbol,
-    dist: Option<usize>,
-) -> Option<Symbol> {
-    let lookup = lookup.as_str();
-    let lookup_uppercase = lookup.to_uppercase();
-
-    // Priority of matches:
-    // 1. Exact case insensitive match
-    // 2. Levenshtein distance match
-    // 3. Sorted word match
-    if let Some(c) = candidates.iter().find(|c| c.as_str().to_uppercase() == lookup_uppercase) {
-        return Some(*c);
-    }
-
-    let mut dist = dist.unwrap_or_else(|| cmp::max(lookup.len(), 3) / 3);
-    let mut best = None;
-    for c in candidates {
-        match if use_substring_score {
-            lev_distance_with_substrings(lookup, c.as_str(), dist)
-        } else {
-            lev_distance(lookup, c.as_str(), dist)
-        } {
-            Some(0) => return Some(*c),
-            Some(d) => {
-                dist = d - 1;
-                best = Some(*c);
-            }
-            None => {}
-        }
-    }
-    if best.is_some() {
-        return best;
-    }
-
-    find_match_by_sorted_words(candidates, lookup)
-}
-
-fn find_match_by_sorted_words(iter_names: &[Symbol], lookup: &str) -> Option<Symbol> {
-    iter_names.iter().fold(None, |result, candidate| {
-        if sort_by_words(candidate.as_str()) == sort_by_words(lookup) {
-            Some(*candidate)
-        } else {
-            result
-        }
-    })
-}
-
-fn sort_by_words(name: &str) -> String {
-    let mut split_words: Vec<&str> = name.split('_').collect();
-    // We are sorting primitive &strs and can use unstable sort here.
-    split_words.sort_unstable();
-    split_words.join("_")
-}
diff --git a/compiler/rustc_span/src/lev_distance/tests.rs b/compiler/rustc_span/src/lev_distance/tests.rs
deleted file mode 100644
index b17d6588c..000000000
--- a/compiler/rustc_span/src/lev_distance/tests.rs
+++ /dev/null
@@ -1,71 +0,0 @@
-use super::*;
-
-#[test]
-fn test_lev_distance() {
-    use std::char::{from_u32, MAX};
-    // Test bytelength agnosticity
-    for c in (0..MAX as u32).filter_map(from_u32).map(|i| i.to_string()) {
-        assert_eq!(lev_distance(&c[..], &c[..], usize::MAX), Some(0));
-    }
-
-    let a = "\nMäry häd ä little lämb\n\nLittle lämb\n";
-    let b = "\nMary häd ä little lämb\n\nLittle lämb\n";
-    let c = "Mary häd ä little lämb\n\nLittle lämb\n";
-    assert_eq!(lev_distance(a, b, usize::MAX), Some(1));
-    assert_eq!(lev_distance(b, a, usize::MAX), Some(1));
-    assert_eq!(lev_distance(a, c, usize::MAX), Some(2));
-    assert_eq!(lev_distance(c, a, usize::MAX), Some(2));
-    assert_eq!(lev_distance(b, c, usize::MAX), Some(1));
-    assert_eq!(lev_distance(c, b, usize::MAX), Some(1));
-}
-
-#[test]
-fn test_lev_distance_limit() {
-    assert_eq!(lev_distance("abc", "abcd", 1), Some(1));
-    assert_eq!(lev_distance("abc", "abcd", 0), None);
-    assert_eq!(lev_distance("abc", "xyz", 3), Some(3));
-    assert_eq!(lev_distance("abc", "xyz", 2), None);
-}
-
-#[test]
-fn test_method_name_similarity_score() {
-    assert_eq!(lev_distance_with_substrings("empty", "is_empty", 1), Some(1));
-    assert_eq!(lev_distance_with_substrings("shrunk", "rchunks", 2), None);
-    assert_eq!(lev_distance_with_substrings("abc", "abcd", 1), Some(1));
-    assert_eq!(lev_distance_with_substrings("a", "abcd", 1), None);
-    assert_eq!(lev_distance_with_substrings("edf", "eq", 1), None);
-    assert_eq!(lev_distance_with_substrings("abc", "xyz", 3), Some(3));
-    assert_eq!(lev_distance_with_substrings("abcdef", "abcdef", 2), Some(0));
-}
-
-#[test]
-fn test_find_best_match_for_name() {
-    use crate::create_default_session_globals_then;
-    create_default_session_globals_then(|| {
-        let input = vec![Symbol::intern("aaab"), Symbol::intern("aaabc")];
-        assert_eq!(
-            find_best_match_for_name(&input, Symbol::intern("aaaa"), None),
-            Some(Symbol::intern("aaab"))
-        );
-
-        assert_eq!(find_best_match_for_name(&input, Symbol::intern("1111111111"), None), None);
-
-        let input = vec![Symbol::intern("AAAA")];
-        assert_eq!(
-            find_best_match_for_name(&input, Symbol::intern("aaaa"), None),
-            Some(Symbol::intern("AAAA"))
-        );
-
-        let input = vec![Symbol::intern("AAAA")];
-        assert_eq!(
-            find_best_match_for_name(&input, Symbol::intern("aaaa"), Some(4)),
-            Some(Symbol::intern("AAAA"))
-        );
-
-        let input = vec![Symbol::intern("a_longer_variable_name")];
-        assert_eq!(
-            find_best_match_for_name(&input, Symbol::intern("a_variable_longer_name"), None),
-            Some(Symbol::intern("a_longer_variable_name"))
-        );
-    })
-}
diff --git a/compiler/rustc_span/src/lib.rs b/compiler/rustc_span/src/lib.rs
index 7e61f2f9f..873cd33f6 100644
--- a/compiler/rustc_span/src/lib.rs
+++ b/compiler/rustc_span/src/lib.rs
@@ -19,6 +19,7 @@
 #![feature(negative_impls)]
 #![feature(min_specialization)]
 #![feature(rustc_attrs)]
+#![feature(let_chains)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 
@@ -46,7 +47,7 @@ pub use hygiene::{ExpnData, ExpnHash, ExpnId, LocalExpnId, SyntaxContext};
 use rustc_data_structures::stable_hasher::HashingControls;
 pub mod def_id;
 use def_id::{CrateNum, DefId, DefPathHash, LocalDefId, LOCAL_CRATE};
-pub mod lev_distance;
+pub mod edit_distance;
 mod span_encoding;
 pub use span_encoding::{Span, DUMMY_SP};
 
@@ -705,23 +706,23 @@ impl Span {
     }
 
     #[inline]
-    pub fn rust_2015(self) -> bool {
-        self.edition() == edition::Edition::Edition2015
+    pub fn is_rust_2015(self) -> bool {
+        self.edition().is_rust_2015()
     }
 
     #[inline]
     pub fn rust_2018(self) -> bool {
-        self.edition() >= edition::Edition::Edition2018
+        self.edition().rust_2018()
     }
 
     #[inline]
     pub fn rust_2021(self) -> bool {
-        self.edition() >= edition::Edition::Edition2021
+        self.edition().rust_2021()
     }
 
     #[inline]
     pub fn rust_2024(self) -> bool {
-        self.edition() >= edition::Edition::Edition2024
+        self.edition().rust_2024()
     }
 
     /// Returns the source callee.
@@ -2148,3 +2149,17 @@ where
         Hash::hash(&len, hasher);
     }
 }
+
+/// Useful type to use with `Result<>` indicate that an error has already
+/// been reported to the user, so no need to continue checking.
+#[derive(Clone, Copy, Debug, Encodable, Decodable, Hash, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(HashStable_Generic)]
+pub struct ErrorGuaranteed(());
+
+impl ErrorGuaranteed {
+    /// To be used only if you really know what you are doing... ideally, we would find a way to
+    /// eliminate all calls to this method.
+    pub fn unchecked_claim_error_was_emitted() -> Self {
+        ErrorGuaranteed(())
+    }
+}
diff --git a/compiler/rustc_span/src/span_encoding.rs b/compiler/rustc_span/src/span_encoding.rs
index d48c4f7e5..c600298c5 100644
--- a/compiler/rustc_span/src/span_encoding.rs
+++ b/compiler/rustc_span/src/span_encoding.rs
@@ -110,11 +110,16 @@ impl Span {
                 // Inline format with parent.
                 let len_or_tag = len_or_tag | PARENT_MASK;
                 let parent2 = parent.local_def_index.as_u32();
-                if ctxt2 == SyntaxContext::root().as_u32() && parent2 <= MAX_CTXT {
+                if ctxt2 == SyntaxContext::root().as_u32()
+                    && parent2 <= MAX_CTXT
+                    && len_or_tag < LEN_TAG
+                {
+                    debug_assert_ne!(len_or_tag, LEN_TAG);
                     return Span { base_or_index: base, len_or_tag, ctxt_or_tag: parent2 as u16 };
                 }
             } else {
                 // Inline format with ctxt.
+                debug_assert_ne!(len_or_tag, LEN_TAG);
                 return Span {
                     base_or_index: base,
                     len_or_tag: len as u16,
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 7597b8d12..6272bf7f2 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -288,6 +288,7 @@ symbols! {
         Target,
         ToOwned,
         ToString,
+        TokenStream,
         Try,
         TryCaptureGeneric,
         TryCapturePrintable,
@@ -723,16 +724,23 @@ symbols! {
         forbid,
         forget,
         format,
+        format_alignment,
         format_args,
         format_args_capture,
         format_args_macro,
         format_args_nl,
+        format_argument,
+        format_arguments,
+        format_count,
         format_macro,
+        format_placeholder,
+        format_unsafe_arg,
         freeze,
         freg,
         frem_fast,
         from,
         from_desugaring,
+        from_fn,
         from_iter,
         from_method,
         from_output,
@@ -948,6 +956,7 @@ symbols! {
         mul,
         mul_assign,
         mul_with_overflow,
+        multiple_supertrait_upcastable,
         must_not_suspend,
         must_use,
         naked,
@@ -1008,6 +1017,7 @@ symbols! {
         non_ascii_idents,
         non_exhaustive,
         non_exhaustive_omitted_patterns_lint,
+        non_lifetime_binders,
         non_modrs_mods,
         nontemporal_store,
         noop_method_borrow,
@@ -1041,6 +1051,7 @@ symbols! {
         overlapping_marker_traits,
         owned_box,
         packed,
+        packed_bundled_libs,
         panic,
         panic_2015,
         panic_2021,
@@ -1076,7 +1087,7 @@ symbols! {
         plugins,
         pointee_trait,
         pointer,
-        pointer_sized,
+        pointer_like,
         poll,
         position,
         post_dash_lto: "post-lto",
@@ -1215,6 +1226,7 @@ symbols! {
         rustc_capture_analysis,
         rustc_clean,
         rustc_coherence_is_core,
+        rustc_coinductive,
         rustc_const_stable,
         rustc_const_unstable,
         rustc_conversion_suggestion,
@@ -1942,7 +1954,7 @@ impl Interner {
         let name = Symbol::new(inner.strings.len() as u32);
 
         // SAFETY: we convert from `&str` to `&[u8]`, clone it into the arena,
-        // and immediately convert the clone back to `&[u8], all because there
+        // and immediately convert the clone back to `&[u8]`, all because there
         // is no `inner.arena.alloc_str()` method. This is clearly safe.
         let string: &str =
             unsafe { str::from_utf8_unchecked(inner.arena.alloc_slice(string.as_bytes())) };
diff --git a/compiler/rustc_error_messages/locales/en-US/symbol_mangling.ftl b/compiler/rustc_symbol_mangling/locales/en-US.ftl
index b7d48280f..b7d48280f 100644
--- a/compiler/rustc_error_messages/locales/en-US/symbol_mangling.ftl
+++ b/compiler/rustc_symbol_mangling/locales/en-US.ftl
diff --git a/compiler/rustc_symbol_mangling/src/legacy.rs b/compiler/rustc_symbol_mangling/src/legacy.rs
index 23ff6b333..2368468c8 100644
--- a/compiler/rustc_symbol_mangling/src/legacy.rs
+++ b/compiler/rustc_symbol_mangling/src/legacy.rs
@@ -3,7 +3,7 @@ use rustc_hir::def_id::CrateNum;
 use rustc_hir::definitions::{DefPathData, DisambiguatedDefPathData};
 use rustc_middle::ty::print::{PrettyPrinter, Print, Printer};
 use rustc_middle::ty::subst::{GenericArg, GenericArgKind};
-use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeVisitableExt};
 use rustc_middle::util::common::record_time;
 
 use std::fmt::{self, Write};
@@ -26,7 +26,7 @@ pub(super) fn mangle<'tcx>(
         let key = tcx.def_key(ty_def_id);
         match key.disambiguated_data.data {
             DefPathData::TypeNs(_) | DefPathData::ValueNs(_) => {
-                instance_ty = tcx.type_of(ty_def_id);
+                instance_ty = tcx.type_of(ty_def_id).subst_identity();
                 debug!(?instance_ty);
                 break;
             }
diff --git a/compiler/rustc_symbol_mangling/src/lib.rs b/compiler/rustc_symbol_mangling/src/lib.rs
index 547a59076..d9ce73734 100644
--- a/compiler/rustc_symbol_mangling/src/lib.rs
+++ b/compiler/rustc_symbol_mangling/src/lib.rs
@@ -100,8 +100,10 @@ extern crate rustc_middle;
 #[macro_use]
 extern crate tracing;
 
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
+use rustc_macros::fluent_messages;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::mir::mono::{InstantiationMode, MonoItem};
@@ -117,6 +119,8 @@ pub mod errors;
 pub mod test;
 pub mod typeid;
 
+fluent_messages! { "../locales/en-US.ftl" }
+
 /// This function computes the symbol name for the given `instance` and the
 /// given instantiating crate. That is, if you know that instance X is
 /// instantiated in crate Y, this is the symbol name this instance would have.
@@ -244,8 +248,7 @@ fn compute_symbol_name<'tcx>(
     // project.
     let avoid_cross_crate_conflicts = is_generic(substs) || is_globally_shared_function;
 
-    let instantiating_crate =
-        if avoid_cross_crate_conflicts { Some(compute_instantiating_crate()) } else { None };
+    let instantiating_crate = avoid_cross_crate_conflicts.then(compute_instantiating_crate);
 
     // Pick the crate responsible for the symbol mangling version, which has to:
     // 1. be stable for each instance, whether it's being defined or imported
diff --git a/compiler/rustc_symbol_mangling/src/typeid/typeid_itanium_cxx_abi.rs b/compiler/rustc_symbol_mangling/src/typeid/typeid_itanium_cxx_abi.rs
index 0759b95bd..1a679f32c 100644
--- a/compiler/rustc_symbol_mangling/src/typeid/typeid_itanium_cxx_abi.rs
+++ b/compiler/rustc_symbol_mangling/src/typeid/typeid_itanium_cxx_abi.rs
@@ -299,6 +299,7 @@ fn encode_region<'tcx>(
         RegionKind::ReEarlyBound(..)
         | RegionKind::ReFree(..)
         | RegionKind::ReStatic
+        | RegionKind::ReError(_)
         | RegionKind::ReVar(..)
         | RegionKind::RePlaceholder(..) => {
             bug!("encode_region: unexpected `{:?}`", region.kind());
@@ -395,6 +396,7 @@ fn encode_ty_name(tcx: TyCtxt<'_>, def_id: DefId) -> String {
             hir::definitions::DefPathData::CrateRoot
             | hir::definitions::DefPathData::Use
             | hir::definitions::DefPathData::GlobalAsm
+            | hir::definitions::DefPathData::ImplTraitAssocTy
             | hir::definitions::DefPathData::MacroNs(..)
             | hir::definitions::DefPathData::LifetimeNs(..) => {
                 bug!("encode_ty_name: unexpected `{:?}`", disambiguated_data.data);
@@ -640,6 +642,7 @@ fn encode_ty<'tcx>(
         ty::Bound(..)
         | ty::Error(..)
         | ty::GeneratorWitness(..)
+        | ty::GeneratorWitnessMIR(..)
         | ty::Infer(..)
         | ty::Alias(..)
         | ty::Param(..)
@@ -672,7 +675,7 @@ fn transform_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, options: TransformTyOptio
         _ if ty.is_unit() => {}
 
         ty::Tuple(tys) => {
-            ty = tcx.mk_tup(tys.iter().map(|ty| transform_ty(tcx, ty, options)));
+            ty = tcx.mk_tup_from_iter(tys.iter().map(|ty| transform_ty(tcx, ty, options)));
         }
 
         ty::Array(ty0, len) => {
@@ -694,13 +697,13 @@ fn transform_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, options: TransformTyOptio
                 let variant = adt_def.non_enum_variant();
                 let param_env = tcx.param_env(variant.def_id);
                 let field = variant.fields.iter().find(|field| {
-                    let ty = tcx.type_of(field.did);
+                    let ty = tcx.type_of(field.did).subst_identity();
                     let is_zst =
                         tcx.layout_of(param_env.and(ty)).map_or(false, |layout| layout.is_zst());
                     !is_zst
                 });
                 if let Some(field) = field {
-                    let ty0 = tcx.bound_type_of(field.did).subst(tcx, substs);
+                    let ty0 = tcx.type_of(field.did).subst(tcx, substs);
                     // Generalize any repr(transparent) user-defined type that is either a pointer
                     // or reference, and either references itself or any other type that contains or
                     // references itself, to avoid a reference cycle.
@@ -779,8 +782,8 @@ fn transform_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, options: TransformTyOptio
                 let output = transform_ty(tcx, fn_sig.skip_binder().output(), options);
                 ty = tcx.mk_fn_ptr(ty::Binder::bind_with_vars(
                     tcx.mk_fn_sig(
-                        parameters.iter(),
-                        &output,
+                        parameters,
+                        output,
                         fn_sig.c_variadic(),
                         fn_sig.unsafety(),
                         fn_sig.abi(),
@@ -793,6 +796,7 @@ fn transform_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, options: TransformTyOptio
         ty::Bound(..)
         | ty::Error(..)
         | ty::GeneratorWitness(..)
+        | ty::GeneratorWitnessMIR(..)
         | ty::Infer(..)
         | ty::Alias(..)
         | ty::Param(..)
@@ -810,21 +814,18 @@ fn transform_substs<'tcx>(
     substs: SubstsRef<'tcx>,
     options: TransformTyOptions,
 ) -> SubstsRef<'tcx> {
-    let substs: Vec<GenericArg<'tcx>> = substs
-        .iter()
-        .map(|subst| {
-            if let GenericArgKind::Type(ty) = subst.unpack() {
-                if is_c_void_ty(tcx, ty) {
-                    tcx.mk_unit().into()
-                } else {
-                    transform_ty(tcx, ty, options).into()
-                }
+    let substs = substs.iter().map(|subst| {
+        if let GenericArgKind::Type(ty) = subst.unpack() {
+            if is_c_void_ty(tcx, ty) {
+                tcx.mk_unit().into()
             } else {
-                subst
+                transform_ty(tcx, ty, options).into()
             }
-        })
-        .collect();
-    tcx.mk_substs(substs.iter())
+        } else {
+            subst
+        }
+    });
+    tcx.mk_substs_from_iter(substs)
 }
 
 /// Returns a type metadata identifier for the specified FnAbi using the Itanium C++ ABI with vendor
diff --git a/compiler/rustc_symbol_mangling/src/v0.rs b/compiler/rustc_symbol_mangling/src/v0.rs
index 0d446d654..2f20d4213 100644
--- a/compiler/rustc_symbol_mangling/src/v0.rs
+++ b/compiler/rustc_symbol_mangling/src/v0.rs
@@ -8,7 +8,8 @@ use rustc_hir::definitions::{DefPathData, DisambiguatedDefPathData};
 use rustc_middle::ty::layout::IntegerExt;
 use rustc_middle::ty::print::{Print, Printer};
 use rustc_middle::ty::{
-    self, EarlyBinder, FloatTy, Instance, IntTy, Ty, TyCtxt, TypeVisitable, UintTy,
+    self, EarlyBinder, FloatTy, Instance, IntTy, Ty, TyCtxt, TypeVisitable, TypeVisitableExt,
+    UintTy,
 };
 use rustc_middle::ty::{GenericArg, GenericArgKind};
 use rustc_span::symbol::kw;
@@ -204,7 +205,7 @@ impl<'tcx> SymbolMangler<'tcx> {
         print_value: impl FnOnce(&'a mut Self, &T) -> Result<&'a mut Self, !>,
     ) -> Result<&'a mut Self, !>
     where
-        T: TypeVisitable<'tcx>,
+        T: TypeVisitable<TyCtxt<'tcx>>,
     {
         // FIXME(non-lifetime-binders): What to do here?
         let regions = if value.has_late_bound_regions() {
@@ -490,6 +491,7 @@ impl<'tcx> Printer<'tcx> for &mut SymbolMangler<'tcx> {
             }
 
             ty::GeneratorWitness(_) => bug!("symbol_names: unexpected `GeneratorWitness`"),
+            ty::GeneratorWitnessMIR(..) => bug!("symbol_names: unexpected `GeneratorWitnessMIR`"),
         }
 
         // Only cache types that do not refer to an enclosing
@@ -539,7 +541,7 @@ impl<'tcx> Printer<'tcx> for &mut SymbolMangler<'tcx> {
                 match predicate.as_ref().skip_binder() {
                     ty::ExistentialPredicate::Trait(trait_ref) => {
                         // Use a type that can't appear in defaults of type parameters.
-                        let dummy_self = cx.tcx.mk_ty_infer(ty::FreshTy(0));
+                        let dummy_self = cx.tcx.mk_fresh_ty(0);
                         let trait_ref = trait_ref.with_self_ty(cx.tcx, dummy_self);
                         cx = cx.print_def_path(trait_ref.def_id, trait_ref.substs)?;
                     }
@@ -790,6 +792,7 @@ impl<'tcx> Printer<'tcx> for &mut SymbolMangler<'tcx> {
             | DefPathData::Use
             | DefPathData::GlobalAsm
             | DefPathData::Impl
+            | DefPathData::ImplTraitAssocTy
             | DefPathData::MacroNs(_)
             | DefPathData::LifetimeNs(_) => {
                 bug!("symbol_names: unexpected DefPathData: {:?}", disambiguated_data.data)
diff --git a/compiler/rustc_target/src/abi/call/loongarch.rs b/compiler/rustc_target/src/abi/call/loongarch.rs
index 4a2d39cc7..247256f07 100644
--- a/compiler/rustc_target/src/abi/call/loongarch.rs
+++ b/compiler/rustc_target/src/abi/call/loongarch.rs
@@ -39,7 +39,7 @@ where
 {
     match arg_layout.abi {
         Abi::Scalar(scalar) => match scalar.primitive() {
-            abi::Int(..) | abi::Pointer => {
+            abi::Int(..) | abi::Pointer(_) => {
                 if arg_layout.size.bits() > xlen {
                     return Err(CannotUseFpConv);
                 }
diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs
index 3b8c867d3..a0730fbb6 100644
--- a/compiler/rustc_target/src/abi/call/mod.rs
+++ b/compiler/rustc_target/src/abi/call/mod.rs
@@ -346,7 +346,7 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
             // The primitive for this algorithm.
             Abi::Scalar(scalar) => {
                 let kind = match scalar.primitive() {
-                    abi::Int(..) | abi::Pointer => RegKind::Integer,
+                    abi::Int(..) | abi::Pointer(_) => RegKind::Integer,
                     abi::F32 | abi::F64 => RegKind::Float,
                 };
                 Ok(HomogeneousAggregate::Homogeneous(Reg { kind, size: self.size }))
diff --git a/compiler/rustc_target/src/abi/call/riscv.rs b/compiler/rustc_target/src/abi/call/riscv.rs
index 34280d38e..d90dce2a0 100644
--- a/compiler/rustc_target/src/abi/call/riscv.rs
+++ b/compiler/rustc_target/src/abi/call/riscv.rs
@@ -45,7 +45,7 @@ where
 {
     match arg_layout.abi {
         Abi::Scalar(scalar) => match scalar.primitive() {
-            abi::Int(..) | abi::Pointer => {
+            abi::Int(..) | abi::Pointer(_) => {
                 if arg_layout.size.bits() > xlen {
                     return Err(CannotUseFpConv);
                 }
diff --git a/compiler/rustc_target/src/abi/call/sparc64.rs b/compiler/rustc_target/src/abi/call/sparc64.rs
index c8b6ac5ae..cbed5b4af 100644
--- a/compiler/rustc_target/src/abi/call/sparc64.rs
+++ b/compiler/rustc_target/src/abi/call/sparc64.rs
@@ -20,7 +20,7 @@ where
 {
     let dl = cx.data_layout();
 
-    if !scalar.primitive().is_float() {
+    if !matches!(scalar.primitive(), abi::F32 | abi::F64) {
         return data;
     }
 
@@ -83,11 +83,11 @@ where
         (abi::F32, _) => offset += Reg::f32().size,
         (_, abi::F64) => offset += Reg::f64().size,
         (abi::Int(i, _signed), _) => offset += i.size(),
-        (abi::Pointer, _) => offset += Reg::i64().size,
+        (abi::Pointer(_), _) => offset += Reg::i64().size,
         _ => {}
     }
 
-    if (offset.bytes() % 4) != 0 && scalar2.primitive().is_float() {
+    if (offset.bytes() % 4) != 0 && matches!(scalar2.primitive(), abi::F32 | abi::F64) {
         offset += Size::from_bytes(4 - (offset.bytes() % 4));
     }
     data = arg_scalar(cx, scalar2, offset, data);
diff --git a/compiler/rustc_target/src/abi/call/x86_64.rs b/compiler/rustc_target/src/abi/call/x86_64.rs
index c0c071a61..9427f27d1 100644
--- a/compiler/rustc_target/src/abi/call/x86_64.rs
+++ b/compiler/rustc_target/src/abi/call/x86_64.rs
@@ -50,7 +50,7 @@ where
             Abi::Uninhabited => return Ok(()),
 
             Abi::Scalar(scalar) => match scalar.primitive() {
-                abi::Int(..) | abi::Pointer => Class::Int,
+                abi::Int(..) | abi::Pointer(_) => Class::Int,
                 abi::F32 | abi::F64 => Class::Sse,
             },
 
diff --git a/compiler/rustc_target/src/abi/mod.rs b/compiler/rustc_target/src/abi/mod.rs
index 88a0a1f8e..8d2e92cc7 100644
--- a/compiler/rustc_target/src/abi/mod.rs
+++ b/compiler/rustc_target/src/abi/mod.rs
@@ -3,10 +3,8 @@ pub use Primitive::*;
 
 use crate::json::{Json, ToJson};
 
-use std::fmt;
 use std::ops::Deref;
 
-use rustc_data_structures::intern::Interned;
 use rustc_macros::HashStable_Generic;
 
 pub mod call;
@@ -19,48 +17,6 @@ impl ToJson for Endian {
     }
 }
 
-rustc_index::newtype_index! {
-    #[derive(HashStable_Generic)]
-    pub struct VariantIdx {}
-}
-
-#[derive(Copy, Clone, PartialEq, Eq, Hash, HashStable_Generic)]
-#[rustc_pass_by_value]
-pub struct Layout<'a>(pub Interned<'a, LayoutS<VariantIdx>>);
-
-impl<'a> fmt::Debug for Layout<'a> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        // See comment on `<LayoutS as Debug>::fmt` above.
-        self.0.0.fmt(f)
-    }
-}
-
-impl<'a> Layout<'a> {
-    pub fn fields(self) -> &'a FieldsShape {
-        &self.0.0.fields
-    }
-
-    pub fn variants(self) -> &'a Variants<VariantIdx> {
-        &self.0.0.variants
-    }
-
-    pub fn abi(self) -> Abi {
-        self.0.0.abi
-    }
-
-    pub fn largest_niche(self) -> Option<Niche> {
-        self.0.0.largest_niche
-    }
-
-    pub fn align(self) -> AbiAndPrefAlign {
-        self.0.0.align
-    }
-
-    pub fn size(self) -> Size {
-        self.0.0.size
-    }
-}
-
 /// The layout of a type, alongside the type itself.
 /// Provides various type traversal APIs (e.g., recursing into fields).
 ///
@@ -75,8 +31,8 @@ pub struct TyAndLayout<'a, Ty> {
 }
 
 impl<'a, Ty> Deref for TyAndLayout<'a, Ty> {
-    type Target = &'a LayoutS<VariantIdx>;
-    fn deref(&self) -> &&'a LayoutS<VariantIdx> {
+    type Target = &'a LayoutS;
+    fn deref(&self) -> &&'a LayoutS {
         &self.layout.0.0
     }
 }
@@ -129,7 +85,7 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
         C: HasDataLayout,
     {
         match self.abi {
-            Abi::Scalar(scalar) => scalar.primitive().is_float(),
+            Abi::Scalar(scalar) => matches!(scalar.primitive(), F32 | F64),
             Abi::Aggregate { .. } => {
                 if self.fields.count() == 1 && self.fields.offset(0).bytes() == 0 {
                     self.field(cx, 0).is_single_fp_element(cx)
diff --git a/compiler/rustc_target/src/spec/aarch64_apple_ios.rs b/compiler/rustc_target/src/spec/aarch64_apple_ios.rs
index b5f9eb125..e9edfd287 100644
--- a/compiler/rustc_target/src/spec/aarch64_apple_ios.rs
+++ b/compiler/rustc_target/src/spec/aarch64_apple_ios.rs
@@ -1,8 +1,11 @@
 use super::apple_base::{ios_llvm_target, opts, Arch};
-use crate::spec::{FramePointer, Target, TargetOptions};
+use crate::spec::{FramePointer, SanitizerSet, Target, TargetOptions};
 
 pub fn target() -> Target {
     let arch = Arch::Arm64;
+    let mut base = opts("ios", arch);
+    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::THREAD;
+
     Target {
         // Clang automatically chooses a more specific target based on
         // IPHONEOS_DEPLOYMENT_TARGET.
@@ -28,7 +31,7 @@ pub fn target() -> Target {
                 darwinpcs\0\
                 -Os\0"
                 .into(),
-            ..opts("ios", arch)
+            ..base
         },
     }
 }
diff --git a/compiler/rustc_target/src/spec/aarch64_apple_ios_macabi.rs b/compiler/rustc_target/src/spec/aarch64_apple_ios_macabi.rs
index 0009972cf..2b135b670 100644
--- a/compiler/rustc_target/src/spec/aarch64_apple_ios_macabi.rs
+++ b/compiler/rustc_target/src/spec/aarch64_apple_ios_macabi.rs
@@ -2,7 +2,7 @@ use super::apple_base::{opts, Arch};
 use crate::spec::{Cc, FramePointer, LinkerFlavor, Lld, Target, TargetOptions};
 
 pub fn target() -> Target {
-    let llvm_target = "arm64-apple-ios14.0-macabi";
+    let llvm_target = "arm64-apple-ios-macabi";
 
     let arch = Arch::Arm64_macabi;
     let mut base = opts("ios", arch);
@@ -22,7 +22,7 @@ pub fn target() -> Target {
             // These arguments are not actually invoked - they just have
             // to look right to pass App Store validation.
             bitcode_llvm_cmdline: "-triple\0\
-                arm64-apple-ios14.0-macabi\0\
+                arm64-apple-ios-macabi\0\
                 -emit-obj\0\
                 -disable-llvm-passes\0\
                 -Os\0"
diff --git a/compiler/rustc_target/src/spec/aarch64_apple_ios_sim.rs b/compiler/rustc_target/src/spec/aarch64_apple_ios_sim.rs
index 3374755e2..6e2d62b6e 100644
--- a/compiler/rustc_target/src/spec/aarch64_apple_ios_sim.rs
+++ b/compiler/rustc_target/src/spec/aarch64_apple_ios_sim.rs
@@ -1,8 +1,11 @@
 use super::apple_base::{ios_sim_llvm_target, opts, Arch};
-use crate::spec::{FramePointer, Target, TargetOptions};
+use crate::spec::{FramePointer, SanitizerSet, Target, TargetOptions};
 
 pub fn target() -> Target {
     let arch = Arch::Arm64_sim;
+    let mut base = opts("ios", arch);
+    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::THREAD;
+
     Target {
         // Clang automatically chooses a more specific target based on
         // IPHONEOS_DEPLOYMENT_TARGET.
@@ -28,7 +31,7 @@ pub fn target() -> Target {
                 darwinpcs\0\
                 -Os\0"
                 .into(),
-            ..opts("ios", arch)
+            ..base
         },
     }
 }
diff --git a/compiler/rustc_target/src/spec/aarch64_be_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/aarch64_be_unknown_linux_gnu.rs
index 9bce82a19..b84783c0a 100644
--- a/compiler/rustc_target/src/spec/aarch64_be_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/aarch64_be_unknown_linux_gnu.rs
@@ -8,7 +8,7 @@ pub fn target() -> Target {
         data_layout: "E-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128".into(),
         arch: "aarch64".into(),
         options: TargetOptions {
-            features: "+outline-atomics".into(),
+            features: "+v8a,+outline-atomics".into(),
             max_atomic_width: Some(128),
             mcount: "\u{1}_mcount".into(),
             endian: Endian::Big,
diff --git a/compiler/rustc_target/src/spec/aarch64_be_unknown_linux_gnu_ilp32.rs b/compiler/rustc_target/src/spec/aarch64_be_unknown_linux_gnu_ilp32.rs
index c9ceb55dd..a24e0119f 100644
--- a/compiler/rustc_target/src/spec/aarch64_be_unknown_linux_gnu_ilp32.rs
+++ b/compiler/rustc_target/src/spec/aarch64_be_unknown_linux_gnu_ilp32.rs
@@ -12,7 +12,7 @@ pub fn target() -> Target {
         arch: "aarch64".into(),
         options: TargetOptions {
             abi: "ilp32".into(),
-            features: "+outline-atomics".into(),
+            features: "+v8a,+outline-atomics".into(),
             mcount: "\u{1}_mcount".into(),
             endian: Endian::Big,
             ..base
diff --git a/compiler/rustc_target/src/spec/aarch64_kmc_solid_asp3.rs b/compiler/rustc_target/src/spec/aarch64_kmc_solid_asp3.rs
index 6ea9ae266..437fd6015 100644
--- a/compiler/rustc_target/src/spec/aarch64_kmc_solid_asp3.rs
+++ b/compiler/rustc_target/src/spec/aarch64_kmc_solid_asp3.rs
@@ -9,7 +9,7 @@ pub fn target() -> Target {
         arch: "aarch64".into(),
         options: TargetOptions {
             linker: Some("aarch64-kmc-elf-gcc".into()),
-            features: "+neon,+fp-armv8".into(),
+            features: "+v8a,+neon,+fp-armv8".into(),
             relocation_model: RelocModel::Static,
             disable_redzone: true,
             max_atomic_width: Some(128),
diff --git a/compiler/rustc_target/src/spec/aarch64_linux_android.rs b/compiler/rustc_target/src/spec/aarch64_linux_android.rs
index c85f7f62a..071b727b3 100644
--- a/compiler/rustc_target/src/spec/aarch64_linux_android.rs
+++ b/compiler/rustc_target/src/spec/aarch64_linux_android.rs
@@ -13,12 +13,13 @@ pub fn target() -> Target {
             max_atomic_width: Some(128),
             // As documented in https://developer.android.com/ndk/guides/cpu-features.html
             // the neon (ASIMD) and FP must exist on all android aarch64 targets.
-            features: "+neon,+fp-armv8".into(),
+            features: "+v8a,+neon,+fp-armv8".into(),
             supported_sanitizers: SanitizerSet::CFI
                 | SanitizerSet::HWADDRESS
                 | SanitizerSet::MEMTAG
                 | SanitizerSet::SHADOWCALLSTACK
                 | SanitizerSet::ADDRESS,
+            supports_xray: true,
             ..super::android_base::opts()
         },
     }
diff --git a/compiler/rustc_target/src/spec/aarch64_nintendo_switch_freestanding.rs b/compiler/rustc_target/src/spec/aarch64_nintendo_switch_freestanding.rs
index 529e98d2c..e271bdc8a 100644
--- a/compiler/rustc_target/src/spec/aarch64_nintendo_switch_freestanding.rs
+++ b/compiler/rustc_target/src/spec/aarch64_nintendo_switch_freestanding.rs
@@ -10,6 +10,7 @@ pub fn target() -> Target {
         data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128".into(),
         arch: "aarch64".into(),
         options: TargetOptions {
+            features: "+v8a".into(),
             linker_flavor: LinkerFlavor::Gnu(Cc::No, Lld::Yes),
             linker: Some("rust-lld".into()),
             link_script: Some(LINKER_SCRIPT.into()),
diff --git a/compiler/rustc_target/src/spec/aarch64_pc_windows_gnullvm.rs b/compiler/rustc_target/src/spec/aarch64_pc_windows_gnullvm.rs
index 98d3e79c8..cf1d7ca11 100644
--- a/compiler/rustc_target/src/spec/aarch64_pc_windows_gnullvm.rs
+++ b/compiler/rustc_target/src/spec/aarch64_pc_windows_gnullvm.rs
@@ -3,7 +3,7 @@ use crate::spec::Target;
 pub fn target() -> Target {
     let mut base = super::windows_gnullvm_base::opts();
     base.max_atomic_width = Some(128);
-    base.features = "+neon,+fp-armv8".into();
+    base.features = "+v8a,+neon,+fp-armv8".into();
     base.linker = Some("aarch64-w64-mingw32-clang".into());
 
     Target {
diff --git a/compiler/rustc_target/src/spec/aarch64_pc_windows_msvc.rs b/compiler/rustc_target/src/spec/aarch64_pc_windows_msvc.rs
index 7c4544b3f..56b76bc7a 100644
--- a/compiler/rustc_target/src/spec/aarch64_pc_windows_msvc.rs
+++ b/compiler/rustc_target/src/spec/aarch64_pc_windows_msvc.rs
@@ -3,7 +3,7 @@ use crate::spec::Target;
 pub fn target() -> Target {
     let mut base = super::windows_msvc_base::opts();
     base.max_atomic_width = Some(128);
-    base.features = "+neon,+fp-armv8".into();
+    base.features = "+v8a,+neon,+fp-armv8".into();
 
     Target {
         llvm_target: "aarch64-pc-windows-msvc".into(),
diff --git a/compiler/rustc_target/src/spec/aarch64_unknown_freebsd.rs b/compiler/rustc_target/src/spec/aarch64_unknown_freebsd.rs
index 2f39c4862..84fa9814b 100644
--- a/compiler/rustc_target/src/spec/aarch64_unknown_freebsd.rs
+++ b/compiler/rustc_target/src/spec/aarch64_unknown_freebsd.rs
@@ -7,6 +7,7 @@ pub fn target() -> Target {
         data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128".into(),
         arch: "aarch64".into(),
         options: TargetOptions {
+            features: "+v8a".into(),
             max_atomic_width: Some(128),
             supported_sanitizers: SanitizerSet::ADDRESS
                 | SanitizerSet::CFI
diff --git a/compiler/rustc_target/src/spec/aarch64_unknown_fuchsia.rs b/compiler/rustc_target/src/spec/aarch64_unknown_fuchsia.rs
index ef2ab304f..a5683fa73 100644
--- a/compiler/rustc_target/src/spec/aarch64_unknown_fuchsia.rs
+++ b/compiler/rustc_target/src/spec/aarch64_unknown_fuchsia.rs
@@ -7,6 +7,7 @@ pub fn target() -> Target {
         data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128".into(),
         arch: "aarch64".into(),
         options: TargetOptions {
+            features: "+v8a".into(),
             max_atomic_width: Some(128),
             supported_sanitizers: SanitizerSet::ADDRESS
                 | SanitizerSet::CFI
diff --git a/compiler/rustc_target/src/spec/aarch64_unknown_hermit.rs b/compiler/rustc_target/src/spec/aarch64_unknown_hermit.rs
index 1d7269c8d..87e8d6270 100644
--- a/compiler/rustc_target/src/spec/aarch64_unknown_hermit.rs
+++ b/compiler/rustc_target/src/spec/aarch64_unknown_hermit.rs
@@ -3,7 +3,7 @@ use crate::spec::Target;
 pub fn target() -> Target {
     let mut base = super::hermit_base::opts();
     base.max_atomic_width = Some(128);
-    base.features = "+strict-align,+neon,+fp-armv8".into();
+    base.features = "+v8a,+strict-align,+neon,+fp-armv8".into();
 
     Target {
         llvm_target: "aarch64-unknown-hermit".into(),
diff --git a/compiler/rustc_target/src/spec/aarch64_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/aarch64_unknown_linux_gnu.rs
index 3006044d5..da2460894 100644
--- a/compiler/rustc_target/src/spec/aarch64_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/aarch64_unknown_linux_gnu.rs
@@ -7,7 +7,7 @@ pub fn target() -> Target {
         data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128".into(),
         arch: "aarch64".into(),
         options: TargetOptions {
-            features: "+outline-atomics".into(),
+            features: "+v8a,+outline-atomics".into(),
             mcount: "\u{1}_mcount".into(),
             max_atomic_width: Some(128),
             supported_sanitizers: SanitizerSet::ADDRESS
@@ -17,6 +17,7 @@ pub fn target() -> Target {
                 | SanitizerSet::MEMTAG
                 | SanitizerSet::THREAD
                 | SanitizerSet::HWADDRESS,
+            supports_xray: true,
             ..super::linux_gnu_base::opts()
         },
     }
diff --git a/compiler/rustc_target/src/spec/aarch64_unknown_linux_gnu_ilp32.rs b/compiler/rustc_target/src/spec/aarch64_unknown_linux_gnu_ilp32.rs
index 63023df1d..ad9df53c2 100644
--- a/compiler/rustc_target/src/spec/aarch64_unknown_linux_gnu_ilp32.rs
+++ b/compiler/rustc_target/src/spec/aarch64_unknown_linux_gnu_ilp32.rs
@@ -8,7 +8,7 @@ pub fn target() -> Target {
         arch: "aarch64".into(),
         options: TargetOptions {
             abi: "ilp32".into(),
-            features: "+outline-atomics".into(),
+            features: "+v8a,+outline-atomics".into(),
             max_atomic_width: Some(128),
             mcount: "\u{1}_mcount".into(),
             ..super::linux_gnu_base::opts()
diff --git a/compiler/rustc_target/src/spec/aarch64_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/aarch64_unknown_linux_musl.rs
index 002d0dac2..d0c950c2e 100644
--- a/compiler/rustc_target/src/spec/aarch64_unknown_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/aarch64_unknown_linux_musl.rs
@@ -3,6 +3,8 @@ use crate::spec::{Target, TargetOptions};
 pub fn target() -> Target {
     let mut base = super::linux_musl_base::opts();
     base.max_atomic_width = Some(128);
+    base.supports_xray = true;
+    base.features = "+v8a".into();
 
     Target {
         llvm_target: "aarch64-unknown-linux-musl".into(),
diff --git a/compiler/rustc_target/src/spec/aarch64_unknown_netbsd.rs b/compiler/rustc_target/src/spec/aarch64_unknown_netbsd.rs
index 703f75022..a58b64d3d 100644
--- a/compiler/rustc_target/src/spec/aarch64_unknown_netbsd.rs
+++ b/compiler/rustc_target/src/spec/aarch64_unknown_netbsd.rs
@@ -7,6 +7,7 @@ pub fn target() -> Target {
         data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128".into(),
         arch: "aarch64".into(),
         options: TargetOptions {
+            features: "+v8a".into(),
             mcount: "__mcount".into(),
             max_atomic_width: Some(128),
             ..super::netbsd_base::opts()
diff --git a/compiler/rustc_target/src/spec/aarch64_unknown_none.rs b/compiler/rustc_target/src/spec/aarch64_unknown_none.rs
index aca52e147..be27302f7 100644
--- a/compiler/rustc_target/src/spec/aarch64_unknown_none.rs
+++ b/compiler/rustc_target/src/spec/aarch64_unknown_none.rs
@@ -14,8 +14,8 @@ pub fn target() -> Target {
     let opts = TargetOptions {
         linker_flavor: LinkerFlavor::Gnu(Cc::No, Lld::Yes),
         linker: Some("rust-lld".into()),
-        features: "+strict-align,+neon,+fp-armv8".into(),
-        supported_sanitizers: SanitizerSet::KCFI,
+        features: "+v8a,+strict-align,+neon,+fp-armv8".into(),
+        supported_sanitizers: SanitizerSet::KCFI | SanitizerSet::KERNELADDRESS,
         relocation_model: RelocModel::Static,
         disable_redzone: true,
         max_atomic_width: Some(128),
diff --git a/compiler/rustc_target/src/spec/aarch64_unknown_none_softfloat.rs b/compiler/rustc_target/src/spec/aarch64_unknown_none_softfloat.rs
index 2385cb69a..9dfa1f268 100644
--- a/compiler/rustc_target/src/spec/aarch64_unknown_none_softfloat.rs
+++ b/compiler/rustc_target/src/spec/aarch64_unknown_none_softfloat.rs
@@ -13,7 +13,7 @@ pub fn target() -> Target {
         abi: "softfloat".into(),
         linker_flavor: LinkerFlavor::Gnu(Cc::No, Lld::Yes),
         linker: Some("rust-lld".into()),
-        features: "+strict-align,-neon,-fp-armv8".into(),
+        features: "+v8a,+strict-align,-neon,-fp-armv8".into(),
         relocation_model: RelocModel::Static,
         disable_redzone: true,
         max_atomic_width: Some(128),
diff --git a/compiler/rustc_target/src/spec/aarch64_unknown_nto_qnx_710.rs b/compiler/rustc_target/src/spec/aarch64_unknown_nto_qnx_710.rs
index 916b6137b..8c1126ae6 100644
--- a/compiler/rustc_target/src/spec/aarch64_unknown_nto_qnx_710.rs
+++ b/compiler/rustc_target/src/spec/aarch64_unknown_nto_qnx_710.rs
@@ -17,6 +17,7 @@ pub fn target() -> Target {
         data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128".into(),
         arch: "aarch64".into(),
         options: TargetOptions {
+            features: "+v8a".into(),
             max_atomic_width: Some(128),
             pre_link_args: TargetOptions::link_args(
                 LinkerFlavor::Gnu(Cc::Yes, Lld::No),
diff --git a/compiler/rustc_target/src/spec/aarch64_unknown_openbsd.rs b/compiler/rustc_target/src/spec/aarch64_unknown_openbsd.rs
index 3d99040f0..224e31af2 100644
--- a/compiler/rustc_target/src/spec/aarch64_unknown_openbsd.rs
+++ b/compiler/rustc_target/src/spec/aarch64_unknown_openbsd.rs
@@ -6,6 +6,10 @@ pub fn target() -> Target {
         pointer_width: 64,
         data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128".into(),
         arch: "aarch64".into(),
-        options: TargetOptions { max_atomic_width: Some(128), ..super::openbsd_base::opts() },
+        options: TargetOptions {
+            features: "+v8a".into(),
+            max_atomic_width: Some(128),
+            ..super::openbsd_base::opts()
+        },
     }
 }
diff --git a/compiler/rustc_target/src/spec/aarch64_unknown_redox.rs b/compiler/rustc_target/src/spec/aarch64_unknown_redox.rs
index 6c9be4c8e..5650162cd 100644
--- a/compiler/rustc_target/src/spec/aarch64_unknown_redox.rs
+++ b/compiler/rustc_target/src/spec/aarch64_unknown_redox.rs
@@ -3,6 +3,7 @@ use crate::spec::Target;
 pub fn target() -> Target {
     let mut base = super::redox_base::opts();
     base.max_atomic_width = Some(128);
+    base.features = "+v8a".into();
 
     Target {
         llvm_target: "aarch64-unknown-redox".into(),
diff --git a/compiler/rustc_target/src/spec/aarch64_unknown_uefi.rs b/compiler/rustc_target/src/spec/aarch64_unknown_uefi.rs
index 817ff2422..82fb01556 100644
--- a/compiler/rustc_target/src/spec/aarch64_unknown_uefi.rs
+++ b/compiler/rustc_target/src/spec/aarch64_unknown_uefi.rs
@@ -9,6 +9,7 @@ pub fn target() -> Target {
 
     base.max_atomic_width = Some(128);
     base.add_pre_link_args(LinkerFlavor::Msvc(Lld::No), &["/machine:arm64"]);
+    base.features = "+v8a".into();
 
     Target {
         llvm_target: "aarch64-unknown-windows".into(),
diff --git a/compiler/rustc_target/src/spec/aarch64_uwp_windows_msvc.rs b/compiler/rustc_target/src/spec/aarch64_uwp_windows_msvc.rs
index db4dbf817..d39442d91 100644
--- a/compiler/rustc_target/src/spec/aarch64_uwp_windows_msvc.rs
+++ b/compiler/rustc_target/src/spec/aarch64_uwp_windows_msvc.rs
@@ -3,6 +3,7 @@ use crate::spec::Target;
 pub fn target() -> Target {
     let mut base = super::windows_uwp_msvc_base::opts();
     base.max_atomic_width = Some(128);
+    base.features = "+v8a".into();
 
     Target {
         llvm_target: "aarch64-pc-windows-msvc".into(),
diff --git a/compiler/rustc_target/src/spec/aarch64_wrs_vxworks.rs b/compiler/rustc_target/src/spec/aarch64_wrs_vxworks.rs
index e118553df..7e2af4c7a 100644
--- a/compiler/rustc_target/src/spec/aarch64_wrs_vxworks.rs
+++ b/compiler/rustc_target/src/spec/aarch64_wrs_vxworks.rs
@@ -6,6 +6,10 @@ pub fn target() -> Target {
         pointer_width: 64,
         data_layout: "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128".into(),
         arch: "aarch64".into(),
-        options: TargetOptions { max_atomic_width: Some(128), ..super::vxworks_base::opts() },
+        options: TargetOptions {
+            features: "+v8a".into(),
+            max_atomic_width: Some(128),
+            ..super::vxworks_base::opts()
+        },
     }
 }
diff --git a/compiler/rustc_target/src/spec/arm64_32_apple_watchos.rs b/compiler/rustc_target/src/spec/arm64_32_apple_watchos.rs
index 52ee68e75..c757ed45e 100644
--- a/compiler/rustc_target/src/spec/arm64_32_apple_watchos.rs
+++ b/compiler/rustc_target/src/spec/arm64_32_apple_watchos.rs
@@ -9,7 +9,7 @@ pub fn target() -> Target {
         data_layout: "e-m:o-p:32:32-i64:64-i128:128-n32:64-S128".into(),
         arch: "aarch64".into(),
         options: TargetOptions {
-            features: "+neon,+fp-armv8,+apple-a7".into(),
+            features: "+v8a,+neon,+fp-armv8,+apple-a7".into(),
             max_atomic_width: Some(128),
             forces_embed_bitcode: true,
             dynamic_linking: false,
diff --git a/compiler/rustc_target/src/spec/armebv7r_none_eabi.rs b/compiler/rustc_target/src/spec/armebv7r_none_eabi.rs
index 8c65d6afc..f6f46aac4 100644
--- a/compiler/rustc_target/src/spec/armebv7r_none_eabi.rs
+++ b/compiler/rustc_target/src/spec/armebv7r_none_eabi.rs
@@ -19,7 +19,7 @@ pub fn target() -> Target {
             max_atomic_width: Some(32),
             emit_debug_gdb_scripts: false,
             // GCC and Clang default to 8 for arm-none here
-            c_enum_min_bits: 8,
+            c_enum_min_bits: Some(8),
             ..Default::default()
         },
     }
diff --git a/compiler/rustc_target/src/spec/armebv7r_none_eabihf.rs b/compiler/rustc_target/src/spec/armebv7r_none_eabihf.rs
index 7013bc60d..9608efe8b 100644
--- a/compiler/rustc_target/src/spec/armebv7r_none_eabihf.rs
+++ b/compiler/rustc_target/src/spec/armebv7r_none_eabihf.rs
@@ -20,7 +20,7 @@ pub fn target() -> Target {
             max_atomic_width: Some(32),
             emit_debug_gdb_scripts: false,
             // GCC and Clang default to 8 for arm-none here
-            c_enum_min_bits: 8,
+            c_enum_min_bits: Some(8),
             ..Default::default()
         },
     }
diff --git a/compiler/rustc_target/src/spec/armv4t_none_eabi.rs b/compiler/rustc_target/src/spec/armv4t_none_eabi.rs
index 7ac1aab3b..28b109889 100644
--- a/compiler/rustc_target/src/spec/armv4t_none_eabi.rs
+++ b/compiler/rustc_target/src/spec/armv4t_none_eabi.rs
@@ -49,7 +49,7 @@ pub fn target() -> Target {
             // from thumb_base, rust-lang/rust#44993.
             emit_debug_gdb_scripts: false,
             // from thumb_base, apparently gcc/clang give enums a minimum of 8 bits on no-os targets
-            c_enum_min_bits: 8,
+            c_enum_min_bits: Some(8),
             ..Default::default()
         },
     }
diff --git a/compiler/rustc_target/src/spec/armv6k_nintendo_3ds.rs b/compiler/rustc_target/src/spec/armv6k_nintendo_3ds.rs
index 40ec6f961..23f4a5abf 100644
--- a/compiler/rustc_target/src/spec/armv6k_nintendo_3ds.rs
+++ b/compiler/rustc_target/src/spec/armv6k_nintendo_3ds.rs
@@ -29,8 +29,7 @@ pub fn target() -> Target {
             pre_link_args,
             exe_suffix: ".elf".into(),
             no_default_libraries: false,
-            // There are some issues in debug builds with this enabled in certain programs.
-            has_thread_local: false,
+            has_thread_local: true,
             ..Default::default()
         },
     }
diff --git a/compiler/rustc_target/src/spec/armv7a_none_eabi.rs b/compiler/rustc_target/src/spec/armv7a_none_eabi.rs
index 4e20fb325..d59de86a2 100644
--- a/compiler/rustc_target/src/spec/armv7a_none_eabi.rs
+++ b/compiler/rustc_target/src/spec/armv7a_none_eabi.rs
@@ -27,7 +27,7 @@ pub fn target() -> Target {
         max_atomic_width: Some(64),
         panic_strategy: PanicStrategy::Abort,
         emit_debug_gdb_scripts: false,
-        c_enum_min_bits: 8,
+        c_enum_min_bits: Some(8),
         ..Default::default()
     };
     Target {
diff --git a/compiler/rustc_target/src/spec/armv7a_none_eabihf.rs b/compiler/rustc_target/src/spec/armv7a_none_eabihf.rs
index ae70129ae..8cdf3c36b 100644
--- a/compiler/rustc_target/src/spec/armv7a_none_eabihf.rs
+++ b/compiler/rustc_target/src/spec/armv7a_none_eabihf.rs
@@ -19,7 +19,7 @@ pub fn target() -> Target {
         panic_strategy: PanicStrategy::Abort,
         emit_debug_gdb_scripts: false,
         // GCC and Clang default to 8 for arm-none here
-        c_enum_min_bits: 8,
+        c_enum_min_bits: Some(8),
         ..Default::default()
     };
     Target {
diff --git a/compiler/rustc_target/src/spec/armv7r_none_eabi.rs b/compiler/rustc_target/src/spec/armv7r_none_eabi.rs
index 25f301ccc..5225abf44 100644
--- a/compiler/rustc_target/src/spec/armv7r_none_eabi.rs
+++ b/compiler/rustc_target/src/spec/armv7r_none_eabi.rs
@@ -18,7 +18,7 @@ pub fn target() -> Target {
             max_atomic_width: Some(32),
             emit_debug_gdb_scripts: false,
             // GCC and Clang default to 8 for arm-none here
-            c_enum_min_bits: 8,
+            c_enum_min_bits: Some(8),
             ..Default::default()
         },
     }
diff --git a/compiler/rustc_target/src/spec/armv7r_none_eabihf.rs b/compiler/rustc_target/src/spec/armv7r_none_eabihf.rs
index 40449759d..9a35e0461 100644
--- a/compiler/rustc_target/src/spec/armv7r_none_eabihf.rs
+++ b/compiler/rustc_target/src/spec/armv7r_none_eabihf.rs
@@ -19,7 +19,7 @@ pub fn target() -> Target {
             max_atomic_width: Some(32),
             emit_debug_gdb_scripts: false,
             // GCC and Clang default to 8 for arm-none here
-            c_enum_min_bits: 8,
+            c_enum_min_bits: Some(8),
             ..Default::default()
         },
     }
diff --git a/compiler/rustc_target/src/spec/bpf_base.rs b/compiler/rustc_target/src/spec/bpf_base.rs
index 2b00cda44..4d03747d0 100644
--- a/compiler/rustc_target/src/spec/bpf_base.rs
+++ b/compiler/rustc_target/src/spec/bpf_base.rs
@@ -6,7 +6,7 @@ pub fn opts(endian: Endian) -> TargetOptions {
         allow_asm: true,
         endian,
         linker_flavor: LinkerFlavor::Bpf,
-        atomic_cas: true,
+        atomic_cas: false,
         dynamic_linking: true,
         no_builtins: true,
         panic_strategy: PanicStrategy::Abort,
diff --git a/compiler/rustc_target/src/spec/hexagon_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/hexagon_unknown_linux_musl.rs
index 3aad05eb2..4c6ab5f5a 100644
--- a/compiler/rustc_target/src/spec/hexagon_unknown_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/hexagon_unknown_linux_musl.rs
@@ -11,7 +11,7 @@ pub fn target() -> Target {
     base.has_rpath = true;
     base.linker_flavor = LinkerFlavor::Unix(Cc::Yes);
 
-    base.c_enum_min_bits = 8;
+    base.c_enum_min_bits = Some(8);
 
     Target {
         llvm_target: "hexagon-unknown-linux-musl".into(),
diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
index a094c2c54..0d86a3032 100644
--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
@@ -812,6 +812,7 @@ bitflags::bitflags! {
         const MEMTAG  = 1 << 6;
         const SHADOWCALLSTACK = 1 << 7;
         const KCFI    = 1 << 8;
+        const KERNELADDRESS = 1 << 9;
     }
 }
 
@@ -824,6 +825,7 @@ impl SanitizerSet {
             SanitizerSet::ADDRESS => "address",
             SanitizerSet::CFI => "cfi",
             SanitizerSet::KCFI => "kcfi",
+            SanitizerSet::KERNELADDRESS => "kernel-address",
             SanitizerSet::LEAK => "leak",
             SanitizerSet::MEMORY => "memory",
             SanitizerSet::MEMTAG => "memtag",
@@ -866,6 +868,7 @@ impl IntoIterator for SanitizerSet {
             SanitizerSet::SHADOWCALLSTACK,
             SanitizerSet::THREAD,
             SanitizerSet::HWADDRESS,
+            SanitizerSet::KERNELADDRESS,
         ]
         .iter()
         .copied()
@@ -1344,10 +1347,18 @@ impl Target {
             });
         }
 
-        dl.c_enum_min_size = match Integer::from_size(Size::from_bits(self.c_enum_min_bits)) {
-            Ok(bits) => bits,
-            Err(err) => return Err(TargetDataLayoutErrors::InvalidBitsSize { err }),
-        };
+        dl.c_enum_min_size = self
+            .c_enum_min_bits
+            .map_or_else(
+                || {
+                    self.c_int_width
+                        .parse()
+                        .map_err(|_| String::from("failed to parse c_int_width"))
+                },
+                Ok,
+            )
+            .and_then(|i| Integer::from_size(Size::from_bits(i)))
+            .map_err(|err| TargetDataLayoutErrors::InvalidBitsSize { err })?;
 
         Ok(dl)
     }
@@ -1701,8 +1712,8 @@ pub struct TargetOptions {
     /// If present it's a default value to use for adjusting the C ABI.
     pub default_adjusted_cabi: Option<Abi>,
 
-    /// Minimum number of bits in #[repr(C)] enum. Defaults to 32.
-    pub c_enum_min_bits: u64,
+    /// Minimum number of bits in #[repr(C)] enum. Defaults to the size of c_int
+    pub c_enum_min_bits: Option<u64>,
 
     /// Whether or not the DWARF `.debug_aranges` section should be generated.
     pub generate_arange_section: bool,
@@ -1718,6 +1729,9 @@ pub struct TargetOptions {
     /// The ABI of entry function.
     /// Default value is `Conv::C`, i.e. C call convention
     pub entry_abi: Conv,
+
+    /// Whether the target supports XRay instrumentation.
+    pub supports_xray: bool,
 }
 
 /// Add arguments for the given flavor and also for its "twin" flavors
@@ -1932,11 +1946,12 @@ impl Default for TargetOptions {
             supported_split_debuginfo: Cow::Borrowed(&[SplitDebuginfo::Off]),
             supported_sanitizers: SanitizerSet::empty(),
             default_adjusted_cabi: None,
-            c_enum_min_bits: 32,
+            c_enum_min_bits: None,
             generate_arange_section: true,
             supports_stack_protector: true,
             entry_name: "main".into(),
             entry_abi: Conv::C,
+            supports_xray: false,
         }
     }
 }
@@ -2118,12 +2133,6 @@ impl Target {
                     base.$key_name = s;
                 }
             } );
-            ($key_name:ident, u64) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                if let Some(s) = obj.remove(&name).and_then(|j| Json::as_u64(&j)) {
-                    base.$key_name = s;
-                }
-            } );
             ($key_name:ident, u32) => ( {
                 let name = (stringify!($key_name)).replace("_", "-");
                 if let Some(s) = obj.remove(&name).and_then(|b| b.as_u64()) {
@@ -2335,6 +2344,7 @@ impl Target {
                                 Some("address") => SanitizerSet::ADDRESS,
                                 Some("cfi") => SanitizerSet::CFI,
                                 Some("kcfi") => SanitizerSet::KCFI,
+                                Some("kernel-address") => SanitizerSet::KERNELADDRESS,
                                 Some("leak") => SanitizerSet::LEAK,
                                 Some("memory") => SanitizerSet::MEMORY,
                                 Some("memtag") => SanitizerSet::MEMTAG,
@@ -2492,6 +2502,7 @@ impl Target {
 
         key!(is_builtin, bool);
         key!(c_int_width = "target-c-int-width");
+        key!(c_enum_min_bits, Option<u64>); // if None, matches c_int_width
         key!(os);
         key!(env);
         key!(abi);
@@ -2587,11 +2598,11 @@ impl Target {
         key!(supported_split_debuginfo, falliable_list)?;
         key!(supported_sanitizers, SanitizerSet)?;
         key!(default_adjusted_cabi, Option<Abi>)?;
-        key!(c_enum_min_bits, u64);
         key!(generate_arange_section, bool);
         key!(supports_stack_protector, bool);
         key!(entry_name);
         key!(entry_abi, Conv)?;
+        key!(supports_xray, bool);
 
         if base.is_builtin {
             // This can cause unfortunate ICEs later down the line.
@@ -2845,6 +2856,7 @@ impl ToJson for Target {
         target_option_val!(supports_stack_protector);
         target_option_val!(entry_name);
         target_option_val!(entry_abi);
+        target_option_val!(supports_xray);
 
         if let Some(abi) = self.default_adjusted_cabi {
             d.insert("default-adjusted-cabi".into(), Abi::name(abi).to_json());
diff --git a/compiler/rustc_target/src/spec/riscv64gc_unknown_none_elf.rs b/compiler/rustc_target/src/spec/riscv64gc_unknown_none_elf.rs
index 409b0b269..ab3c14e3f 100644
--- a/compiler/rustc_target/src/spec/riscv64gc_unknown_none_elf.rs
+++ b/compiler/rustc_target/src/spec/riscv64gc_unknown_none_elf.rs
@@ -1,6 +1,8 @@
 use crate::spec::{Cc, CodeModel, LinkerFlavor, Lld, PanicStrategy};
 use crate::spec::{RelocModel, Target, TargetOptions};
 
+use super::SanitizerSet;
+
 pub fn target() -> Target {
     Target {
         data_layout: "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128".into(),
@@ -20,6 +22,7 @@ pub fn target() -> Target {
             code_model: Some(CodeModel::Medium),
             emit_debug_gdb_scripts: false,
             eh_frame_header: false,
+            supported_sanitizers: SanitizerSet::KERNELADDRESS,
             ..Default::default()
         },
     }
diff --git a/compiler/rustc_target/src/spec/riscv64imac_unknown_none_elf.rs b/compiler/rustc_target/src/spec/riscv64imac_unknown_none_elf.rs
index 87aba9171..0f1821c99 100644
--- a/compiler/rustc_target/src/spec/riscv64imac_unknown_none_elf.rs
+++ b/compiler/rustc_target/src/spec/riscv64imac_unknown_none_elf.rs
@@ -1,5 +1,5 @@
 use crate::spec::{Cc, CodeModel, LinkerFlavor, Lld, PanicStrategy};
-use crate::spec::{RelocModel, Target, TargetOptions};
+use crate::spec::{RelocModel, SanitizerSet, Target, TargetOptions};
 
 pub fn target() -> Target {
     Target {
@@ -19,6 +19,7 @@ pub fn target() -> Target {
             code_model: Some(CodeModel::Medium),
             emit_debug_gdb_scripts: false,
             eh_frame_header: false,
+            supported_sanitizers: SanitizerSet::KERNELADDRESS,
             ..Default::default()
         },
     }
diff --git a/compiler/rustc_target/src/spec/s390x_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/s390x_unknown_linux_gnu.rs
index cda88de0e..f2c722b9a 100644
--- a/compiler/rustc_target/src/spec/s390x_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/s390x_unknown_linux_gnu.rs
@@ -1,5 +1,5 @@
 use crate::abi::Endian;
-use crate::spec::{StackProbeType, Target};
+use crate::spec::{SanitizerSet, StackProbeType, Target};
 
 pub fn target() -> Target {
     let mut base = super::linux_gnu_base::opts();
@@ -13,6 +13,8 @@ pub fn target() -> Target {
     base.max_atomic_width = Some(64);
     base.min_global_align = Some(16);
     base.stack_probes = StackProbeType::Inline;
+    base.supported_sanitizers =
+        SanitizerSet::ADDRESS | SanitizerSet::LEAK | SanitizerSet::MEMORY | SanitizerSet::THREAD;
 
     Target {
         llvm_target: "s390x-unknown-linux-gnu".into(),
diff --git a/compiler/rustc_target/src/spec/s390x_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/s390x_unknown_linux_musl.rs
index 91e63aee5..8fe9d023c 100644
--- a/compiler/rustc_target/src/spec/s390x_unknown_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/s390x_unknown_linux_musl.rs
@@ -1,5 +1,5 @@
 use crate::abi::Endian;
-use crate::spec::{StackProbeType, Target};
+use crate::spec::{SanitizerSet, StackProbeType, Target};
 
 pub fn target() -> Target {
     let mut base = super::linux_musl_base::opts();
@@ -14,6 +14,8 @@ pub fn target() -> Target {
     base.min_global_align = Some(16);
     base.static_position_independent_executables = true;
     base.stack_probes = StackProbeType::Inline;
+    base.supported_sanitizers =
+        SanitizerSet::ADDRESS | SanitizerSet::LEAK | SanitizerSet::MEMORY | SanitizerSet::THREAD;
 
     Target {
         llvm_target: "s390x-unknown-linux-musl".into(),
diff --git a/compiler/rustc_target/src/spec/thumb_base.rs b/compiler/rustc_target/src/spec/thumb_base.rs
index 000766c57..4dcf47fe4 100644
--- a/compiler/rustc_target/src/spec/thumb_base.rs
+++ b/compiler/rustc_target/src/spec/thumb_base.rs
@@ -53,7 +53,7 @@ pub fn opts() -> TargetOptions {
         frame_pointer: FramePointer::Always,
         // ARM supports multiple ABIs for enums, the linux one matches the default of 32 here
         // but any arm-none or thumb-none target will be defaulted to 8 on GCC and clang
-        c_enum_min_bits: 8,
+        c_enum_min_bits: Some(8),
         ..Default::default()
     }
 }
diff --git a/compiler/rustc_target/src/spec/thumbv4t_none_eabi.rs b/compiler/rustc_target/src/spec/thumbv4t_none_eabi.rs
index 5a3e4c88d..e3734932f 100644
--- a/compiler/rustc_target/src/spec/thumbv4t_none_eabi.rs
+++ b/compiler/rustc_target/src/spec/thumbv4t_none_eabi.rs
@@ -55,7 +55,7 @@ pub fn target() -> Target {
             // suggested from thumb_base, rust-lang/rust#44993.
             emit_debug_gdb_scripts: false,
             // suggested from thumb_base, with no-os gcc/clang use 8-bit enums
-            c_enum_min_bits: 8,
+            c_enum_min_bits: Some(8),
             frame_pointer: FramePointer::MayOmit,
 
             main_needs_argc_argv: false,
diff --git a/compiler/rustc_target/src/spec/x86_64_apple_ios.rs b/compiler/rustc_target/src/spec/x86_64_apple_ios.rs
index fbd3ebd4d..1dcb47056 100644
--- a/compiler/rustc_target/src/spec/x86_64_apple_ios.rs
+++ b/compiler/rustc_target/src/spec/x86_64_apple_ios.rs
@@ -1,8 +1,11 @@
 use super::apple_base::{ios_sim_llvm_target, opts, Arch};
-use crate::spec::{StackProbeType, Target, TargetOptions};
+use crate::spec::{SanitizerSet, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let arch = Arch::X86_64_sim;
+    let mut base = opts("ios", arch);
+    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::THREAD;
+
     Target {
         llvm_target: ios_sim_llvm_target(arch).into(),
         pointer_width: 64,
@@ -12,7 +15,7 @@ pub fn target() -> Target {
         options: TargetOptions {
             max_atomic_width: Some(64),
             stack_probes: StackProbeType::X86,
-            ..opts("ios", arch)
+            ..base
         },
     }
 }
diff --git a/compiler/rustc_target/src/spec/x86_64_apple_ios_macabi.rs b/compiler/rustc_target/src/spec/x86_64_apple_ios_macabi.rs
index 0f3f85199..5a3e2a79b 100644
--- a/compiler/rustc_target/src/spec/x86_64_apple_ios_macabi.rs
+++ b/compiler/rustc_target/src/spec/x86_64_apple_ios_macabi.rs
@@ -2,7 +2,7 @@ use super::apple_base::{opts, Arch};
 use crate::spec::{Cc, LinkerFlavor, Lld, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
-    let llvm_target = "x86_64-apple-ios13.0-macabi";
+    let llvm_target = "x86_64-apple-ios-macabi";
 
     let arch = Arch::X86_64_macabi;
     let mut base = opts("ios", arch);
diff --git a/compiler/rustc_target/src/spec/x86_64_linux_android.rs b/compiler/rustc_target/src/spec/x86_64_linux_android.rs
index 9c9137848..a3bdb5f54 100644
--- a/compiler/rustc_target/src/spec/x86_64_linux_android.rs
+++ b/compiler/rustc_target/src/spec/x86_64_linux_android.rs
@@ -8,6 +8,7 @@ pub fn target() -> Target {
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m64"]);
     base.stack_probes = StackProbeType::X86;
+    base.supports_xray = true;
 
     Target {
         llvm_target: "x86_64-linux-android".into(),
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_freebsd.rs b/compiler/rustc_target/src/spec/x86_64_unknown_freebsd.rs
index 98988ab35..b41e5842a 100644
--- a/compiler/rustc_target/src/spec/x86_64_unknown_freebsd.rs
+++ b/compiler/rustc_target/src/spec/x86_64_unknown_freebsd.rs
@@ -8,6 +8,7 @@ pub fn target() -> Target {
     base.stack_probes = StackProbeType::X86;
     base.supported_sanitizers =
         SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::MEMORY | SanitizerSet::THREAD;
+    base.supports_xray = true;
 
     Target {
         llvm_target: "x86_64-unknown-freebsd".into(),
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/x86_64_unknown_linux_gnu.rs
index a91ab365b..9af1049b8 100644
--- a/compiler/rustc_target/src/spec/x86_64_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/x86_64_unknown_linux_gnu.rs
@@ -12,6 +12,7 @@ pub fn target() -> Target {
         | SanitizerSet::LEAK
         | SanitizerSet::MEMORY
         | SanitizerSet::THREAD;
+    base.supports_xray = true;
 
     Target {
         llvm_target: "x86_64-unknown-linux-gnu".into(),
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/x86_64_unknown_linux_musl.rs
index 9087dc3df..bf4cf7d7b 100644
--- a/compiler/rustc_target/src/spec/x86_64_unknown_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/x86_64_unknown_linux_musl.rs
@@ -12,6 +12,7 @@ pub fn target() -> Target {
         | SanitizerSet::LEAK
         | SanitizerSet::MEMORY
         | SanitizerSet::THREAD;
+    base.supports_xray = true;
 
     Target {
         llvm_target: "x86_64-unknown-linux-musl".into(),
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_netbsd.rs b/compiler/rustc_target/src/spec/x86_64_unknown_netbsd.rs
index 64ae425d8..74c434935 100644
--- a/compiler/rustc_target/src/spec/x86_64_unknown_netbsd.rs
+++ b/compiler/rustc_target/src/spec/x86_64_unknown_netbsd.rs
@@ -11,6 +11,7 @@ pub fn target() -> Target {
         | SanitizerSet::LEAK
         | SanitizerSet::MEMORY
         | SanitizerSet::THREAD;
+    base.supports_xray = true;
 
     Target {
         llvm_target: "x86_64-unknown-netbsd".into(),
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_none.rs b/compiler/rustc_target/src/spec/x86_64_unknown_none.rs
index 32060c35c..43c5ce78c 100644
--- a/compiler/rustc_target/src/spec/x86_64_unknown_none.rs
+++ b/compiler/rustc_target/src/spec/x86_64_unknown_none.rs
@@ -20,7 +20,7 @@ pub fn target() -> Target {
         features:
             "-mmx,-sse,-sse2,-sse3,-ssse3,-sse4.1,-sse4.2,-3dnow,-3dnowa,-avx,-avx2,+soft-float"
                 .into(),
-        supported_sanitizers: SanitizerSet::KCFI,
+        supported_sanitizers: SanitizerSet::KCFI | SanitizerSet::KERNELADDRESS,
         disable_redzone: true,
         panic_strategy: PanicStrategy::Abort,
         code_model: Some(CodeModel::Kernel),
diff --git a/compiler/rustc_target/src/spec/x86_64_unknown_openbsd.rs b/compiler/rustc_target/src/spec/x86_64_unknown_openbsd.rs
index 66b8e2022..8e4d42a0a 100644
--- a/compiler/rustc_target/src/spec/x86_64_unknown_openbsd.rs
+++ b/compiler/rustc_target/src/spec/x86_64_unknown_openbsd.rs
@@ -6,6 +6,7 @@ pub fn target() -> Target {
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m64"]);
     base.stack_probes = StackProbeType::X86;
+    base.supports_xray = true;
 
     Target {
         llvm_target: "x86_64-unknown-openbsd".into(),
diff --git a/compiler/rustc_trait_selection/Cargo.toml b/compiler/rustc_trait_selection/Cargo.toml
index 90d879976..d3eba43b4 100644
--- a/compiler/rustc_trait_selection/Cargo.toml
+++ b/compiler/rustc_trait_selection/Cargo.toml
@@ -16,7 +16,6 @@ rustc_errors = { path = "../rustc_errors" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
 rustc_infer = { path = "../rustc_infer" }
-rustc_lint_defs = { path = "../rustc_lint_defs" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_query_system = { path = "../rustc_query_system" }
 rustc_serialize = { path = "../rustc_serialize" }
@@ -25,3 +24,4 @@ rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 rustc_transmute = { path = "../rustc_transmute", features = ["rustc"] }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
+itertools = "0.10.1"
diff --git a/compiler/rustc_error_messages/locales/en-US/trait_selection.ftl b/compiler/rustc_trait_selection/locales/en-US.ftl
index 14eb4a550..14eb4a550 100644
--- a/compiler/rustc_error_messages/locales/en-US/trait_selection.ftl
+++ b/compiler/rustc_trait_selection/locales/en-US.ftl
diff --git a/compiler/rustc_trait_selection/src/errors.rs b/compiler/rustc_trait_selection/src/errors.rs
index 4405537c6..df7c4df18 100644
--- a/compiler/rustc_trait_selection/src/errors.rs
+++ b/compiler/rustc_trait_selection/src/errors.rs
@@ -1,4 +1,5 @@
-use rustc_errors::{fluent, ErrorGuaranteed, Handler, IntoDiagnostic};
+use crate::fluent_generated as fluent;
+use rustc_errors::{ErrorGuaranteed, Handler, IntoDiagnostic};
 use rustc_macros::Diagnostic;
 use rustc_middle::ty::{self, PolyTraitRef, Ty};
 use rustc_span::{Span, Symbol};
@@ -69,19 +70,19 @@ impl IntoDiagnostic<'_> for NegativePositiveConflict<'_> {
         diag.code(rustc_errors::error_code!(E0751));
         match self.negative_impl_span {
             Ok(span) => {
-                diag.span_label(span, fluent::negative_implementation_here);
+                diag.span_label(span, fluent::trait_selection_negative_implementation_here);
             }
             Err(cname) => {
-                diag.note(fluent::negative_implementation_in_crate);
+                diag.note(fluent::trait_selection_negative_implementation_in_crate);
                 diag.set_arg("negative_impl_cname", cname.to_string());
             }
         }
         match self.positive_impl_span {
             Ok(span) => {
-                diag.span_label(span, fluent::positive_implementation_here);
+                diag.span_label(span, fluent::trait_selection_positive_implementation_here);
             }
             Err(cname) => {
-                diag.note(fluent::positive_implementation_in_crate);
+                diag.note(fluent::trait_selection_positive_implementation_in_crate);
                 diag.set_arg("positive_impl_cname", cname.to_string());
             }
         }
diff --git a/compiler/rustc_trait_selection/src/infer.rs b/compiler/rustc_trait_selection/src/infer.rs
index 50c1787ef..9b47c7299 100644
--- a/compiler/rustc_trait_selection/src/infer.rs
+++ b/compiler/rustc_trait_selection/src/infer.rs
@@ -6,7 +6,7 @@ use rustc_hir::lang_items::LangItem;
 use rustc_middle::arena::ArenaAllocatable;
 use rustc_middle::infer::canonical::{Canonical, CanonicalQueryResponse, QueryResponse};
 use rustc_middle::traits::query::Fallible;
-use rustc_middle::ty::{self, Ty, TypeFoldable, TypeVisitable};
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable, TypeVisitableExt};
 use rustc_middle::ty::{GenericArg, ToPredicate};
 use rustc_span::{Span, DUMMY_SP};
 
@@ -42,7 +42,7 @@ pub trait InferCtxtExt<'tcx> {
     fn type_implements_trait(
         &self,
         trait_def_id: DefId,
-        params: impl IntoIterator<Item = impl Into<GenericArg<'tcx>>>,
+        params: impl IntoIterator<Item: Into<GenericArg<'tcx>>>,
         param_env: ty::ParamEnv<'tcx>,
     ) -> traits::EvaluationResult;
 }
@@ -82,7 +82,7 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
     fn type_implements_trait(
         &self,
         trait_def_id: DefId,
-        params: impl IntoIterator<Item = impl Into<GenericArg<'tcx>>>,
+        params: impl IntoIterator<Item: Into<GenericArg<'tcx>>>,
         param_env: ty::ParamEnv<'tcx>,
     ) -> traits::EvaluationResult {
         let trait_ref = self.tcx.mk_trait_ref(trait_def_id, params);
@@ -104,8 +104,8 @@ pub trait InferCtxtBuilderExt<'tcx> {
         operation: impl FnOnce(&ObligationCtxt<'_, 'tcx>, K) -> Fallible<R>,
     ) -> Fallible<CanonicalQueryResponse<'tcx, R>>
     where
-        K: TypeFoldable<'tcx>,
-        R: Debug + TypeFoldable<'tcx>,
+        K: TypeFoldable<TyCtxt<'tcx>>,
+        R: Debug + TypeFoldable<TyCtxt<'tcx>>,
         Canonical<'tcx, QueryResponse<'tcx, R>>: ArenaAllocatable<'tcx>;
 }
 
@@ -125,15 +125,15 @@ impl<'tcx> InferCtxtBuilderExt<'tcx> for InferCtxtBuilder<'tcx> {
     /// In part because we would need a `for<'tcx>` sort of
     /// bound for the closure and in part because it is convenient to
     /// have `'tcx` be free on this function so that we can talk about
-    /// `K: TypeFoldable<'tcx>`.)
+    /// `K: TypeFoldable<TyCtxt<'tcx>>`.)
     fn enter_canonical_trait_query<K, R>(
         &mut self,
         canonical_key: &Canonical<'tcx, K>,
         operation: impl FnOnce(&ObligationCtxt<'_, 'tcx>, K) -> Fallible<R>,
     ) -> Fallible<CanonicalQueryResponse<'tcx, R>>
     where
-        K: TypeFoldable<'tcx>,
-        R: Debug + TypeFoldable<'tcx>,
+        K: TypeFoldable<TyCtxt<'tcx>>,
+        R: Debug + TypeFoldable<TyCtxt<'tcx>>,
         Canonical<'tcx, QueryResponse<'tcx, R>>: ArenaAllocatable<'tcx>,
     {
         let (infcx, key, canonical_inference_vars) =
diff --git a/compiler/rustc_trait_selection/src/lib.rs b/compiler/rustc_trait_selection/src/lib.rs
index 6fa094103..548b42cef 100644
--- a/compiler/rustc_trait_selection/src/lib.rs
+++ b/compiler/rustc_trait_selection/src/lib.rs
@@ -36,7 +36,12 @@ extern crate rustc_middle;
 #[macro_use]
 extern crate smallvec;
 
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
+use rustc_macros::fluent_messages;
+
 pub mod errors;
 pub mod infer;
 pub mod solve;
 pub mod traits;
+
+fluent_messages! { "../locales/en-US.ftl" }
diff --git a/compiler/rustc_trait_selection/src/solve/assembly.rs b/compiler/rustc_trait_selection/src/solve/assembly.rs
index 31c1bc9ec..dec9f8016 100644
--- a/compiler/rustc_trait_selection/src/solve/assembly.rs
+++ b/compiler/rustc_trait_selection/src/solve/assembly.rs
@@ -1,7 +1,9 @@
 //! Code shared by trait and projection goals for candidate assembly.
 
-use super::infcx_ext::InferCtxtExt;
+#[cfg(doc)]
+use super::trait_goals::structural_traits::*;
 use super::{CanonicalResponse, Certainty, EvalCtxt, Goal, MaybeCause, QueryResult};
+use itertools::Itertools;
 use rustc_hir::def_id::DefId;
 use rustc_infer::traits::query::NoSolution;
 use rustc_infer::traits::util::elaborate_predicates;
@@ -76,63 +78,135 @@ pub(super) enum CandidateSource {
     ///     let _y = x.clone();
     /// }
     /// ```
-    AliasBound(usize),
+    AliasBound,
 }
 
-pub(super) trait GoalKind<'tcx>: TypeFoldable<'tcx> + Copy + Eq {
+/// Methods used to assemble candidates for either trait or projection goals.
+pub(super) trait GoalKind<'tcx>: TypeFoldable<TyCtxt<'tcx>> + Copy + Eq {
     fn self_ty(self) -> Ty<'tcx>;
 
     fn with_self_ty(self, tcx: TyCtxt<'tcx>, self_ty: Ty<'tcx>) -> Self;
 
     fn trait_def_id(self, tcx: TyCtxt<'tcx>) -> DefId;
 
-    fn consider_impl_candidate(
+    // Consider a clause, which consists of a "assumption" and some "requirements",
+    // to satisfy a goal. If the requirements hold, then attempt to satisfy our
+    // goal by equating it with the assumption.
+    fn consider_implied_clause(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
-        impl_def_id: DefId,
+        assumption: ty::Predicate<'tcx>,
+        requirements: impl IntoIterator<Item = Goal<'tcx, ty::Predicate<'tcx>>>,
     ) -> QueryResult<'tcx>;
 
-    fn consider_assumption(
+    // Consider a clause specifically for a `dyn Trait` self type. This requires
+    // additionally checking all of the supertraits and object bounds to hold,
+    // since they're not implied by the well-formedness of the object type.
+    fn consider_object_bound_candidate(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
         assumption: ty::Predicate<'tcx>,
     ) -> QueryResult<'tcx>;
 
+    fn consider_impl_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+        impl_def_id: DefId,
+    ) -> QueryResult<'tcx>;
+
+    // A type implements an `auto trait` if its components do as well. These components
+    // are given by built-in rules from [`instantiate_constituent_tys_for_auto_trait`].
     fn consider_auto_trait_candidate(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx>;
 
+    // A trait alias holds if the RHS traits and `where` clauses hold.
     fn consider_trait_alias_candidate(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx>;
 
+    // A type is `Copy` or `Clone` if its components are `Sized`. These components
+    // are given by built-in rules from [`instantiate_constituent_tys_for_sized_trait`].
     fn consider_builtin_sized_candidate(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx>;
 
+    // A type is `Copy` or `Clone` if its components are `Copy` or `Clone`. These
+    // components are given by built-in rules from [`instantiate_constituent_tys_for_copy_clone_trait`].
     fn consider_builtin_copy_clone_candidate(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx>;
 
-    fn consider_builtin_pointer_sized_candidate(
+    // A type is `PointerLike` if we can compute its layout, and that layout
+    // matches the layout of `usize`.
+    fn consider_builtin_pointer_like_candidate(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx>;
 
+    // A callable type (a closure, fn def, or fn ptr) is known to implement the `Fn<A>`
+    // family of traits where `A` is given by the signature of the type.
     fn consider_builtin_fn_trait_candidates(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
         kind: ty::ClosureKind,
     ) -> QueryResult<'tcx>;
 
+    // `Tuple` is implemented if the `Self` type is a tuple.
     fn consider_builtin_tuple_candidate(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx>;
+
+    // `Pointee` is always implemented.
+    //
+    // See the projection implementation for the `Metadata` types for all of
+    // the built-in types. For structs, the metadata type is given by the struct
+    // tail.
+    fn consider_builtin_pointee_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx>;
+
+    // A generator (that comes from an `async` desugaring) is known to implement
+    // `Future<Output = O>`, where `O` is given by the generator's return type
+    // that was computed during type-checking.
+    fn consider_builtin_future_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx>;
+
+    // A generator (that doesn't come from an `async` desugaring) is known to
+    // implement `Generator<R, Yield = Y, Return = O>`, given the resume, yield,
+    // and return types of the generator computed during type-checking.
+    fn consider_builtin_generator_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx>;
+
+    // The most common forms of unsizing are array to slice, and concrete (Sized)
+    // type into a `dyn Trait`. ADTs and Tuples can also have their final field
+    // unsized if it's generic.
+    fn consider_builtin_unsize_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx>;
+
+    // `dyn Trait1` can be unsized to `dyn Trait2` if they are the same trait, or
+    // if `Trait2` is a (transitive) supertrait of `Trait2`.
+    fn consider_builtin_dyn_upcast_candidates(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> Vec<CanonicalResponse<'tcx>>;
+
+    fn consider_builtin_discriminant_kind_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx>;
 }
 
 impl<'tcx> EvalCtxt<'_, 'tcx> {
@@ -140,7 +214,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         &mut self,
         goal: Goal<'tcx, G>,
     ) -> Vec<Candidate<'tcx>> {
-        debug_assert_eq!(goal, self.infcx.resolve_vars_if_possible(goal));
+        debug_assert_eq!(goal, self.resolve_vars_if_possible(goal));
 
         // HACK: `_: Trait` is ambiguous, because it may be satisfied via a builtin rule,
         // object bound, alias bound, etc. We are unable to determine this until we can at
@@ -148,9 +222,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         if goal.predicate.self_ty().is_ty_var() {
             return vec![Candidate {
                 source: CandidateSource::BuiltinImpl,
-                result: self
-                    .make_canonical_response(Certainty::Maybe(MaybeCause::Ambiguity))
-                    .unwrap(),
+                result: self.make_canonical_response(Certainty::AMBIGUOUS).unwrap(),
             }];
         }
 
@@ -186,8 +258,8 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         let &ty::Alias(ty::Projection, projection_ty) = goal.predicate.self_ty().kind() else {
             return
         };
-        self.infcx.probe(|_| {
-            let normalized_ty = self.infcx.next_ty_infer();
+        self.probe(|this| {
+            let normalized_ty = this.next_ty_infer();
             let normalizes_to_goal = goal.with(
                 tcx,
                 ty::Binder::dummy(ty::ProjectionPredicate {
@@ -195,18 +267,16 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
                     term: normalized_ty.into(),
                 }),
             );
-            let normalization_certainty = match self.evaluate_goal(normalizes_to_goal) {
+            let normalization_certainty = match this.evaluate_goal(normalizes_to_goal) {
                 Ok((_, certainty)) => certainty,
                 Err(NoSolution) => return,
             };
-            let normalized_ty = self.infcx.resolve_vars_if_possible(normalized_ty);
+            let normalized_ty = this.resolve_vars_if_possible(normalized_ty);
 
             // NOTE: Alternatively we could call `evaluate_goal` here and only have a `Normalized` candidate.
             // This doesn't work as long as we use `CandidateSource` in winnowing.
             let goal = goal.with(tcx, goal.predicate.with_self_ty(tcx, normalized_ty));
-            // FIXME: This is broken if we care about the `usize` of `AliasBound` because the self type
-            // could be normalized to yet another projection with different item bounds.
-            let normalized_candidates = self.assemble_and_evaluate_candidates(goal);
+            let normalized_candidates = this.assemble_and_evaluate_candidates(goal);
             for mut normalized_candidate in normalized_candidates {
                 normalized_candidate.result =
                     normalized_candidate.result.unchecked_map(|mut response| {
@@ -255,12 +325,22 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             || lang_items.clone_trait() == Some(trait_def_id)
         {
             G::consider_builtin_copy_clone_candidate(self, goal)
-        } else if lang_items.pointer_sized() == Some(trait_def_id) {
-            G::consider_builtin_pointer_sized_candidate(self, goal)
+        } else if lang_items.pointer_like() == Some(trait_def_id) {
+            G::consider_builtin_pointer_like_candidate(self, goal)
         } else if let Some(kind) = self.tcx().fn_trait_kind_from_def_id(trait_def_id) {
             G::consider_builtin_fn_trait_candidates(self, goal, kind)
         } else if lang_items.tuple_trait() == Some(trait_def_id) {
             G::consider_builtin_tuple_candidate(self, goal)
+        } else if lang_items.pointee_trait() == Some(trait_def_id) {
+            G::consider_builtin_pointee_candidate(self, goal)
+        } else if lang_items.future_trait() == Some(trait_def_id) {
+            G::consider_builtin_future_candidate(self, goal)
+        } else if lang_items.gen_trait() == Some(trait_def_id) {
+            G::consider_builtin_generator_candidate(self, goal)
+        } else if lang_items.unsize_trait() == Some(trait_def_id) {
+            G::consider_builtin_unsize_candidate(self, goal)
+        } else if lang_items.discriminant_kind_trait() == Some(trait_def_id) {
+            G::consider_builtin_discriminant_kind_candidate(self, goal)
         } else {
             Err(NoSolution)
         };
@@ -271,6 +351,14 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             }
             Err(NoSolution) => (),
         }
+
+        // There may be multiple unsize candidates for a trait with several supertraits:
+        // `trait Foo: Bar<A> + Bar<B>` and `dyn Foo: Unsize<dyn Bar<_>>`
+        if lang_items.unsize_trait() == Some(trait_def_id) {
+            for result in G::consider_builtin_dyn_upcast_candidates(self, goal) {
+                candidates.push(Candidate { source: CandidateSource::BuiltinImpl, result });
+            }
+        }
     }
 
     fn assemble_param_env_candidates<G: GoalKind<'tcx>>(
@@ -279,7 +367,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         candidates: &mut Vec<Candidate<'tcx>>,
     ) {
         for (i, assumption) in goal.param_env.caller_bounds().iter().enumerate() {
-            match G::consider_assumption(self, goal, assumption) {
+            match G::consider_implied_clause(self, goal, assumption, []) {
                 Ok(result) => {
                     candidates.push(Candidate { source: CandidateSource::ParamEnv(i), result })
                 }
@@ -312,25 +400,23 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             | ty::Closure(..)
             | ty::Generator(..)
             | ty::GeneratorWitness(_)
+            | ty::GeneratorWitnessMIR(..)
             | ty::Never
             | ty::Tuple(_)
             | ty::Param(_)
             | ty::Placeholder(..)
-            | ty::Infer(_)
+            | ty::Infer(ty::IntVar(_) | ty::FloatVar(_))
             | ty::Error(_) => return,
-            ty::Bound(..) => bug!("unexpected bound type: {goal:?}"),
+            ty::Infer(ty::TyVar(_) | ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_))
+            | ty::Bound(..) => bug!("unexpected self type for `{goal:?}`"),
             ty::Alias(_, alias_ty) => alias_ty,
         };
 
-        for (i, (assumption, _)) in self
-            .tcx()
-            .bound_explicit_item_bounds(alias_ty.def_id)
-            .subst_iter_copied(self.tcx(), alias_ty.substs)
-            .enumerate()
+        for assumption in self.tcx().item_bounds(alias_ty.def_id).subst(self.tcx(), alias_ty.substs)
         {
-            match G::consider_assumption(self, goal, assumption) {
+            match G::consider_implied_clause(self, goal, assumption, []) {
                 Ok(result) => {
-                    candidates.push(Candidate { source: CandidateSource::AliasBound(i), result })
+                    candidates.push(Candidate { source: CandidateSource::AliasBound, result })
                 }
                 Err(NoSolution) => (),
             }
@@ -362,13 +448,15 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             | ty::Closure(..)
             | ty::Generator(..)
             | ty::GeneratorWitness(_)
+            | ty::GeneratorWitnessMIR(..)
             | ty::Never
             | ty::Tuple(_)
             | ty::Param(_)
             | ty::Placeholder(..)
-            | ty::Infer(_)
+            | ty::Infer(ty::IntVar(_) | ty::FloatVar(_))
             | ty::Error(_) => return,
-            ty::Bound(..) => bug!("unexpected bound type: {goal:?}"),
+            ty::Infer(ty::TyVar(_) | ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_))
+            | ty::Bound(..) => bug!("unexpected self type for `{goal:?}`"),
             ty::Dynamic(bounds, ..) => bounds,
         };
 
@@ -376,7 +464,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         for assumption in
             elaborate_predicates(tcx, bounds.iter().map(|bound| bound.with_self_ty(tcx, self_ty)))
         {
-            match G::consider_assumption(self, goal, assumption.predicate) {
+            match G::consider_object_bound_candidate(self, goal, assumption.predicate) {
                 Ok(result) => {
                     candidates.push(Candidate { source: CandidateSource::BuiltinImpl, result })
                 }
@@ -384,4 +472,79 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             }
         }
     }
+
+    #[instrument(level = "debug", skip(self), ret)]
+    pub(super) fn merge_candidates_and_discard_reservation_impls(
+        &mut self,
+        mut candidates: Vec<Candidate<'tcx>>,
+    ) -> QueryResult<'tcx> {
+        match candidates.len() {
+            0 => return Err(NoSolution),
+            1 => return Ok(self.discard_reservation_impl(candidates.pop().unwrap()).result),
+            _ => {}
+        }
+
+        if candidates.len() > 1 {
+            let mut i = 0;
+            'outer: while i < candidates.len() {
+                for j in (0..candidates.len()).filter(|&j| i != j) {
+                    if self.trait_candidate_should_be_dropped_in_favor_of(
+                        &candidates[i],
+                        &candidates[j],
+                    ) {
+                        debug!(candidate = ?candidates[i], "Dropping candidate #{}/{}", i, candidates.len());
+                        candidates.swap_remove(i);
+                        continue 'outer;
+                    }
+                }
+
+                debug!(candidate = ?candidates[i], "Retaining candidate #{}/{}", i, candidates.len());
+                i += 1;
+            }
+
+            // If there are *STILL* multiple candidates that have *different* response
+            // results, give up and report ambiguity.
+            if candidates.len() > 1 && !candidates.iter().map(|cand| cand.result).all_equal() {
+                let certainty = if candidates.iter().all(|x| {
+                    matches!(x.result.value.certainty, Certainty::Maybe(MaybeCause::Overflow))
+                }) {
+                    Certainty::Maybe(MaybeCause::Overflow)
+                } else {
+                    Certainty::AMBIGUOUS
+                };
+                return self.make_canonical_response(certainty);
+            }
+        }
+
+        // FIXME: What if there are >1 candidates left with the same response, and one is a reservation impl?
+        Ok(self.discard_reservation_impl(candidates.pop().unwrap()).result)
+    }
+
+    fn trait_candidate_should_be_dropped_in_favor_of(
+        &self,
+        candidate: &Candidate<'tcx>,
+        other: &Candidate<'tcx>,
+    ) -> bool {
+        // FIXME: implement this
+        match (candidate.source, other.source) {
+            (CandidateSource::Impl(_), _)
+            | (CandidateSource::ParamEnv(_), _)
+            | (CandidateSource::AliasBound, _)
+            | (CandidateSource::BuiltinImpl, _) => false,
+        }
+    }
+
+    fn discard_reservation_impl(&self, mut candidate: Candidate<'tcx>) -> Candidate<'tcx> {
+        if let CandidateSource::Impl(def_id) = candidate.source {
+            if let ty::ImplPolarity::Reservation = self.tcx().impl_polarity(def_id) {
+                debug!("Selected reservation impl");
+                // We assemble all candidates inside of a probe so by
+                // making a new canonical response here our result will
+                // have no constraints.
+                candidate.result = self.make_canonical_response(Certainty::AMBIGUOUS).unwrap();
+            }
+        }
+
+        candidate
+    }
 }
diff --git a/compiler/rustc_trait_selection/src/solve/canonical/canonicalize.rs b/compiler/rustc_trait_selection/src/solve/canonical/canonicalize.rs
new file mode 100644
index 000000000..c048d4a2a
--- /dev/null
+++ b/compiler/rustc_trait_selection/src/solve/canonical/canonicalize.rs
@@ -0,0 +1,390 @@
+use std::cmp::Ordering;
+
+use crate::infer::InferCtxt;
+use rustc_middle::infer::canonical::Canonical;
+use rustc_middle::infer::canonical::CanonicalTyVarKind;
+use rustc_middle::infer::canonical::CanonicalVarInfo;
+use rustc_middle::infer::canonical::CanonicalVarInfos;
+use rustc_middle::infer::canonical::CanonicalVarKind;
+use rustc_middle::ty::BoundRegionKind::BrAnon;
+use rustc_middle::ty::BoundTyKind;
+use rustc_middle::ty::TyCtxt;
+use rustc_middle::ty::TypeVisitableExt;
+use rustc_middle::ty::{self, Ty};
+use rustc_middle::ty::{TypeFoldable, TypeFolder, TypeSuperFoldable};
+
+/// Whether we're canonicalizing a query input or the query reponse.
+///
+/// When canonicalizing an input we're in the context of the caller
+/// while canonicalizing the response happens in the context of the
+/// query.
+#[derive(Debug, Clone, Copy)]
+pub enum CanonicalizeMode {
+    Input,
+    /// FIXME: We currently return region constraints refering to
+    /// placeholders and inference variables from a binder instantiated
+    /// inside of the query.
+    ///
+    /// In the long term we should eagerly deal with these constraints
+    /// inside of the query and only propagate constraints which are
+    /// actually nameable by the caller.
+    Response {
+        /// The highest universe nameable by the caller.
+        ///
+        /// All variables in a universe nameable by the caller get mapped
+        /// to the root universe in the response and then mapped back to
+        /// their correct universe when applying the query response in the
+        /// context of the caller.
+        ///
+        /// This doesn't work for universes created inside of the query so
+        /// we do remember their universe in the response.
+        max_input_universe: ty::UniverseIndex,
+    },
+}
+
+pub struct Canonicalizer<'a, 'tcx> {
+    infcx: &'a InferCtxt<'tcx>,
+    canonicalize_mode: CanonicalizeMode,
+
+    variables: &'a mut Vec<ty::GenericArg<'tcx>>,
+    primitive_var_infos: Vec<CanonicalVarInfo<'tcx>>,
+    binder_index: ty::DebruijnIndex,
+}
+
+impl<'a, 'tcx> Canonicalizer<'a, 'tcx> {
+    #[instrument(level = "debug", skip(infcx), ret)]
+    pub fn canonicalize<T: TypeFoldable<TyCtxt<'tcx>>>(
+        infcx: &'a InferCtxt<'tcx>,
+        canonicalize_mode: CanonicalizeMode,
+        variables: &'a mut Vec<ty::GenericArg<'tcx>>,
+        value: T,
+    ) -> Canonical<'tcx, T> {
+        let mut canonicalizer = Canonicalizer {
+            infcx,
+            canonicalize_mode,
+
+            variables,
+            primitive_var_infos: Vec::new(),
+            binder_index: ty::INNERMOST,
+        };
+
+        let value = value.fold_with(&mut canonicalizer);
+        assert!(!value.needs_infer());
+        assert!(!value.has_placeholders());
+
+        let (max_universe, variables) = canonicalizer.finalize();
+
+        Canonical { max_universe, variables, value }
+    }
+
+    fn finalize(self) -> (ty::UniverseIndex, CanonicalVarInfos<'tcx>) {
+        let mut var_infos = self.primitive_var_infos;
+        // See the rustc-dev-guide section about how we deal with universes
+        // during canonicalization in the new solver.
+        match self.canonicalize_mode {
+            // We try to deduplicate as many query calls as possible and hide
+            // all information which should not matter for the solver.
+            //
+            // For this we compress universes as much as possible.
+            CanonicalizeMode::Input => {}
+            // When canonicalizing a response we map a universes already entered
+            // by the caller to the root universe and only return useful universe
+            // information for placeholders and inference variables created inside
+            // of the query.
+            CanonicalizeMode::Response { max_input_universe } => {
+                for var in var_infos.iter_mut() {
+                    let uv = var.universe();
+                    let new_uv = ty::UniverseIndex::from(
+                        uv.index().saturating_sub(max_input_universe.index()),
+                    );
+                    *var = var.with_updated_universe(new_uv);
+                }
+                let max_universe = var_infos
+                    .iter()
+                    .map(|info| info.universe())
+                    .max()
+                    .unwrap_or(ty::UniverseIndex::ROOT);
+
+                let var_infos = self.infcx.tcx.mk_canonical_var_infos(&var_infos);
+                return (max_universe, var_infos);
+            }
+        }
+
+        // Given a `var_infos` with existentials `En` and universals `Un` in
+        // universes `n`, this algorithm compresses them in place so that:
+        //
+        // - the new universe indices are as small as possible
+        // - we only create a new universe if we would otherwise put a placeholder in
+        //   the same compressed universe as an existential which cannot name it
+        //
+        // Let's walk through an example:
+        // - var_infos: [E0, U1, E5, U2, E2, E6, U6], curr_compressed_uv: 0, next_orig_uv: 0
+        // - var_infos: [E0, U1, E5, U2, E2, E6, U6], curr_compressed_uv: 0, next_orig_uv: 1
+        // - var_infos: [E0, U1, E5, U2, E2, E6, U6], curr_compressed_uv: 1, next_orig_uv: 2
+        // - var_infos: [E0, U1, E5, U1, E1, E6, U6], curr_compressed_uv: 1, next_orig_uv: 5
+        // - var_infos: [E0, U1, E1, U1, E1, E6, U6], curr_compressed_uv: 1, next_orig_uv: 6
+        // - var_infos: [E0, U1, E1, U1, E1, E2, U2], curr_compressed_uv: 2, next_orig_uv: -
+        //
+        // This algorithm runs in `O(n²)` where `n` is the number of different universe
+        // indices in the input. This should be fine as `n` is expected to be small.
+        let mut curr_compressed_uv = ty::UniverseIndex::ROOT;
+        let mut existential_in_new_uv = false;
+        let mut next_orig_uv = Some(ty::UniverseIndex::ROOT);
+        while let Some(orig_uv) = next_orig_uv.take() {
+            let mut update_uv = |var: &mut CanonicalVarInfo<'tcx>, orig_uv, is_existential| {
+                let uv = var.universe();
+                match uv.cmp(&orig_uv) {
+                    Ordering::Less => (), // Already updated
+                    Ordering::Equal => {
+                        if is_existential {
+                            existential_in_new_uv = true;
+                        } else if existential_in_new_uv {
+                            //  `var` is a placeholder from a universe which is not nameable
+                            // by an existential which we already put into the compressed
+                            // universe `curr_compressed_uv`. We therefore have to create a
+                            // new universe for `var`.
+                            curr_compressed_uv = curr_compressed_uv.next_universe();
+                            existential_in_new_uv = false;
+                        }
+
+                        *var = var.with_updated_universe(curr_compressed_uv);
+                    }
+                    Ordering::Greater => {
+                        // We can ignore this variable in this iteration. We only look at
+                        // universes which actually occur in the input for performance.
+                        //
+                        // For this we set `next_orig_uv` to the next smallest, not yet compressed,
+                        // universe of the input.
+                        if next_orig_uv.map_or(true, |curr_next_uv| uv.cannot_name(curr_next_uv)) {
+                            next_orig_uv = Some(uv);
+                        }
+                    }
+                }
+            };
+
+            // For each universe which occurs in the input, we first iterate over all
+            // placeholders and then over all inference variables.
+            //
+            // Whenever we compress the universe of a placeholder, no existential with
+            // an already compressed universe can name that placeholder.
+            for is_existential in [false, true] {
+                for var in var_infos.iter_mut() {
+                    // We simply put all regions from the input into the highest
+                    // compressed universe, so we only deal with them at the end.
+                    if !var.is_region() {
+                        if is_existential == var.is_existential() {
+                            update_uv(var, orig_uv, is_existential)
+                        }
+                    }
+                }
+            }
+        }
+
+        for var in var_infos.iter_mut() {
+            if var.is_region() {
+                assert!(var.is_existential());
+                *var = var.with_updated_universe(curr_compressed_uv);
+            }
+        }
+
+        let var_infos = self.infcx.tcx.mk_canonical_var_infos(&var_infos);
+        (curr_compressed_uv, var_infos)
+    }
+}
+
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'_, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
+        self.infcx.tcx
+    }
+
+    fn fold_binder<T>(&mut self, t: ty::Binder<'tcx, T>) -> ty::Binder<'tcx, T>
+    where
+        T: TypeFoldable<TyCtxt<'tcx>>,
+    {
+        self.binder_index.shift_in(1);
+        let t = t.super_fold_with(self);
+        self.binder_index.shift_out(1);
+        t
+    }
+
+    fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
+        let r = self.infcx.shallow_resolve(r);
+        let kind = match *r {
+            ty::ReLateBound(..) => return r,
+
+            ty::ReStatic => match self.canonicalize_mode {
+                CanonicalizeMode::Input => CanonicalVarKind::Region(ty::UniverseIndex::ROOT),
+                CanonicalizeMode::Response { .. } => return r,
+            },
+
+            ty::ReErased | ty::ReFree(_) | ty::ReEarlyBound(_) => match self.canonicalize_mode {
+                CanonicalizeMode::Input => CanonicalVarKind::Region(ty::UniverseIndex::ROOT),
+                CanonicalizeMode::Response { .. } => bug!("unexpected region in response: {r:?}"),
+            },
+
+            ty::RePlaceholder(placeholder) => match self.canonicalize_mode {
+                // We canonicalize placeholder regions as existentials in query inputs.
+                CanonicalizeMode::Input => CanonicalVarKind::Region(ty::UniverseIndex::ROOT),
+                CanonicalizeMode::Response { max_input_universe } => {
+                    // If we have a placeholder region inside of a query, it must be from
+                    // a new universe.
+                    if max_input_universe.can_name(placeholder.universe) {
+                        bug!("new placeholder in universe {max_input_universe:?}: {r:?}");
+                    }
+                    CanonicalVarKind::PlaceholderRegion(placeholder)
+                }
+            },
+
+            ty::ReVar(_) => match self.canonicalize_mode {
+                CanonicalizeMode::Input => CanonicalVarKind::Region(ty::UniverseIndex::ROOT),
+                CanonicalizeMode::Response { .. } => {
+                    CanonicalVarKind::Region(self.infcx.universe_of_region(r))
+                }
+            },
+
+            ty::ReError(_) => return r,
+        };
+
+        let existing_bound_var = match self.canonicalize_mode {
+            CanonicalizeMode::Input => None,
+            CanonicalizeMode::Response { .. } => {
+                self.variables.iter().position(|&v| v == r.into()).map(ty::BoundVar::from)
+            }
+        };
+        let var = existing_bound_var.unwrap_or_else(|| {
+            let var = ty::BoundVar::from(self.variables.len());
+            self.variables.push(r.into());
+            self.primitive_var_infos.push(CanonicalVarInfo { kind });
+            var
+        });
+        let br = ty::BoundRegion { var, kind: BrAnon(var.as_u32(), None) };
+        self.interner().mk_re_late_bound(self.binder_index, br)
+    }
+
+    fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
+        let kind = match *t.kind() {
+            ty::Infer(ty::TyVar(vid)) => match self.infcx.probe_ty_var(vid) {
+                Ok(t) => return self.fold_ty(t),
+                Err(ui) => CanonicalVarKind::Ty(CanonicalTyVarKind::General(ui)),
+            },
+            ty::Infer(ty::IntVar(_)) => {
+                let nt = self.infcx.shallow_resolve(t);
+                if nt != t {
+                    return self.fold_ty(nt);
+                } else {
+                    CanonicalVarKind::Ty(CanonicalTyVarKind::Int)
+                }
+            }
+            ty::Infer(ty::FloatVar(_)) => {
+                let nt = self.infcx.shallow_resolve(t);
+                if nt != t {
+                    return self.fold_ty(nt);
+                } else {
+                    CanonicalVarKind::Ty(CanonicalTyVarKind::Int)
+                }
+            }
+            ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => {
+                bug!("fresh var during canonicalization: {t:?}")
+            }
+            ty::Placeholder(placeholder) => match self.canonicalize_mode {
+                CanonicalizeMode::Input => CanonicalVarKind::PlaceholderTy(ty::Placeholder {
+                    universe: placeholder.universe,
+                    name: BoundTyKind::Anon(self.variables.len() as u32),
+                }),
+                CanonicalizeMode::Response { .. } => CanonicalVarKind::PlaceholderTy(placeholder),
+            },
+            ty::Param(_) => match self.canonicalize_mode {
+                CanonicalizeMode::Input => CanonicalVarKind::PlaceholderTy(ty::Placeholder {
+                    universe: ty::UniverseIndex::ROOT,
+                    name: ty::BoundTyKind::Anon(self.variables.len() as u32),
+                }),
+                CanonicalizeMode::Response { .. } => bug!("param ty in response: {t:?}"),
+            },
+            ty::Bool
+            | ty::Char
+            | ty::Int(_)
+            | ty::Uint(_)
+            | ty::Float(_)
+            | ty::Adt(_, _)
+            | ty::Foreign(_)
+            | ty::Str
+            | ty::Array(_, _)
+            | ty::Slice(_)
+            | ty::RawPtr(_)
+            | ty::Ref(_, _, _)
+            | ty::FnDef(_, _)
+            | ty::FnPtr(_)
+            | ty::Dynamic(_, _, _)
+            | ty::Closure(_, _)
+            | ty::Generator(_, _, _)
+            | ty::GeneratorWitness(_)
+            | ty::GeneratorWitnessMIR(..)
+            | ty::Never
+            | ty::Tuple(_)
+            | ty::Alias(_, _)
+            | ty::Bound(_, _)
+            | ty::Error(_) => return t.super_fold_with(self),
+        };
+
+        let var = ty::BoundVar::from(
+            self.variables.iter().position(|&v| v == t.into()).unwrap_or_else(|| {
+                let var = self.variables.len();
+                self.variables.push(t.into());
+                self.primitive_var_infos.push(CanonicalVarInfo { kind });
+                var
+            }),
+        );
+        let bt = ty::BoundTy { var, kind: BoundTyKind::Anon(var.index() as u32) };
+        self.interner().mk_bound(self.binder_index, bt)
+    }
+
+    fn fold_const(&mut self, c: ty::Const<'tcx>) -> ty::Const<'tcx> {
+        let kind = match c.kind() {
+            ty::ConstKind::Infer(ty::InferConst::Var(vid)) => match self.infcx.probe_const_var(vid)
+            {
+                Ok(c) => return self.fold_const(c),
+                Err(universe) => CanonicalVarKind::Const(universe, c.ty()),
+            },
+            ty::ConstKind::Infer(ty::InferConst::Fresh(_)) => {
+                bug!("fresh var during canonicalization: {c:?}")
+            }
+            ty::ConstKind::Placeholder(placeholder) => match self.canonicalize_mode {
+                CanonicalizeMode::Input => CanonicalVarKind::PlaceholderConst(
+                    ty::Placeholder {
+                        universe: placeholder.universe,
+                        name: ty::BoundVar::from(self.variables.len()),
+                    },
+                    c.ty(),
+                ),
+                CanonicalizeMode::Response { .. } => {
+                    CanonicalVarKind::PlaceholderConst(placeholder, c.ty())
+                }
+            },
+            ty::ConstKind::Param(_) => match self.canonicalize_mode {
+                CanonicalizeMode::Input => CanonicalVarKind::PlaceholderConst(
+                    ty::Placeholder {
+                        universe: ty::UniverseIndex::ROOT,
+                        name: ty::BoundVar::from(self.variables.len()),
+                    },
+                    c.ty(),
+                ),
+                CanonicalizeMode::Response { .. } => bug!("param ty in response: {c:?}"),
+            },
+            ty::ConstKind::Bound(_, _)
+            | ty::ConstKind::Unevaluated(_)
+            | ty::ConstKind::Value(_)
+            | ty::ConstKind::Error(_)
+            | ty::ConstKind::Expr(_) => return c.super_fold_with(self),
+        };
+
+        let var = ty::BoundVar::from(
+            self.variables.iter().position(|&v| v == c.into()).unwrap_or_else(|| {
+                let var = self.variables.len();
+                self.variables.push(c.into());
+                self.primitive_var_infos.push(CanonicalVarInfo { kind });
+                var
+            }),
+        );
+        self.interner().mk_const(ty::ConstKind::Bound(self.binder_index, var), c.ty())
+    }
+}
diff --git a/compiler/rustc_trait_selection/src/solve/canonical/mod.rs b/compiler/rustc_trait_selection/src/solve/canonical/mod.rs
new file mode 100644
index 000000000..8c3be8da1
--- /dev/null
+++ b/compiler/rustc_trait_selection/src/solve/canonical/mod.rs
@@ -0,0 +1,240 @@
+/// Canonicalization is used to separate some goal from its context,
+/// throwing away unnecessary information in the process.
+///
+/// This is necessary to cache goals containing inference variables
+/// and placeholders without restricting them to the current `InferCtxt`.
+///
+/// Canonicalization is fairly involved, for more details see the relevant
+/// section of the [rustc-dev-guide][c].
+///
+/// [c]: https://rustc-dev-guide.rust-lang.org/solve/canonicalization.html
+use self::canonicalize::{CanonicalizeMode, Canonicalizer};
+use super::{CanonicalGoal, Certainty, EvalCtxt, Goal};
+use super::{CanonicalResponse, ExternalConstraints, QueryResult, Response};
+use rustc_infer::infer::canonical::query_response::make_query_region_constraints;
+use rustc_infer::infer::canonical::CanonicalVarValues;
+use rustc_infer::infer::canonical::{CanonicalExt, QueryRegionConstraints};
+use rustc_infer::traits::query::NoSolution;
+use rustc_infer::traits::solve::ExternalConstraintsData;
+use rustc_infer::traits::ObligationCause;
+use rustc_middle::ty::{self, GenericArgKind};
+use rustc_span::DUMMY_SP;
+use std::iter;
+use std::ops::Deref;
+
+mod canonicalize;
+
+impl<'tcx> EvalCtxt<'_, 'tcx> {
+    /// Canonicalizes the goal remembering the original values
+    /// for each bound variable.
+    pub(super) fn canonicalize_goal(
+        &self,
+        goal: Goal<'tcx, ty::Predicate<'tcx>>,
+    ) -> (Vec<ty::GenericArg<'tcx>>, CanonicalGoal<'tcx>) {
+        let mut orig_values = Default::default();
+        let canonical_goal = Canonicalizer::canonicalize(
+            self.infcx,
+            CanonicalizeMode::Input,
+            &mut orig_values,
+            goal,
+        );
+        (orig_values, canonical_goal)
+    }
+
+    /// To return the constraints of a canonical query to the caller, we canonicalize:
+    ///
+    /// - `var_values`: a map from bound variables in the canonical goal to
+    ///   the values inferred while solving the instantiated goal.
+    /// - `external_constraints`: additional constraints which aren't expressable
+    ///   using simple unification of inference variables.
+    #[instrument(level = "debug", skip(self))]
+    pub(super) fn make_canonical_response(&self, certainty: Certainty) -> QueryResult<'tcx> {
+        let external_constraints = self.compute_external_query_constraints()?;
+
+        let response = Response { var_values: self.var_values, external_constraints, certainty };
+        let canonical = Canonicalizer::canonicalize(
+            self.infcx,
+            CanonicalizeMode::Response { max_input_universe: self.max_input_universe },
+            &mut Default::default(),
+            response,
+        );
+        Ok(canonical)
+    }
+
+    #[instrument(level = "debug", skip(self), ret)]
+    fn compute_external_query_constraints(&self) -> Result<ExternalConstraints<'tcx>, NoSolution> {
+        // Cannot use `take_registered_region_obligations` as we may compute the response
+        // inside of a `probe` whenever we have multiple choices inside of the solver.
+        let region_obligations = self.infcx.inner.borrow().region_obligations().to_owned();
+        let region_constraints = self.infcx.with_region_constraints(|region_constraints| {
+            make_query_region_constraints(
+                self.tcx(),
+                region_obligations
+                    .iter()
+                    .map(|r_o| (r_o.sup_type, r_o.sub_region, r_o.origin.to_constraint_category())),
+                region_constraints,
+            )
+        });
+        let opaque_types = self.infcx.clone_opaque_types_for_query_response();
+        Ok(self
+            .tcx()
+            .mk_external_constraints(ExternalConstraintsData { region_constraints, opaque_types }))
+    }
+
+    /// After calling a canonical query, we apply the constraints returned
+    /// by the query using this function.
+    ///
+    /// This happens in three steps:
+    /// - we instantiate the bound variables of the query response
+    /// - we unify the `var_values` of the response with the `original_values`
+    /// - we apply the `external_constraints` returned by the query
+    pub(super) fn instantiate_and_apply_query_response(
+        &mut self,
+        param_env: ty::ParamEnv<'tcx>,
+        original_values: Vec<ty::GenericArg<'tcx>>,
+        response: CanonicalResponse<'tcx>,
+    ) -> Result<Certainty, NoSolution> {
+        let substitution = self.compute_query_response_substitution(&original_values, &response);
+
+        let Response { var_values, external_constraints, certainty } =
+            response.substitute(self.tcx(), &substitution);
+
+        self.unify_query_var_values(param_env, &original_values, var_values)?;
+
+        // FIXME: implement external constraints.
+        let ExternalConstraintsData { region_constraints, opaque_types: _ } =
+            external_constraints.deref();
+        self.register_region_constraints(region_constraints);
+
+        Ok(certainty)
+    }
+
+    /// This returns the substitutions to instantiate the bound variables of
+    /// the canonical reponse. This depends on the `original_values` for the
+    /// bound variables.
+    fn compute_query_response_substitution(
+        &self,
+        original_values: &[ty::GenericArg<'tcx>],
+        response: &CanonicalResponse<'tcx>,
+    ) -> CanonicalVarValues<'tcx> {
+        // FIXME: Longterm canonical queries should deal with all placeholders
+        // created inside of the query directly instead of returning them to the
+        // caller.
+        let prev_universe = self.infcx.universe();
+        let universes_created_in_query = response.max_universe.index() + 1;
+        for _ in 0..universes_created_in_query {
+            self.infcx.create_next_universe();
+        }
+
+        let var_values = response.value.var_values;
+        assert_eq!(original_values.len(), var_values.len());
+
+        // If the query did not make progress with constraining inference variables,
+        // we would normally create a new inference variables for bound existential variables
+        // only then unify this new inference variable with the inference variable from
+        // the input.
+        //
+        // We therefore instantiate the existential variable in the canonical response with the
+        // inference variable of the input right away, which is more performant.
+        let mut opt_values = vec![None; response.variables.len()];
+        for (original_value, result_value) in iter::zip(original_values, var_values.var_values) {
+            match result_value.unpack() {
+                GenericArgKind::Type(t) => {
+                    if let &ty::Bound(debruijn, b) = t.kind() {
+                        assert_eq!(debruijn, ty::INNERMOST);
+                        opt_values[b.var.index()] = Some(*original_value);
+                    }
+                }
+                GenericArgKind::Lifetime(r) => {
+                    if let ty::ReLateBound(debruijn, br) = *r {
+                        assert_eq!(debruijn, ty::INNERMOST);
+                        opt_values[br.var.index()] = Some(*original_value);
+                    }
+                }
+                GenericArgKind::Const(c) => {
+                    if let ty::ConstKind::Bound(debrujin, b) = c.kind() {
+                        assert_eq!(debrujin, ty::INNERMOST);
+                        opt_values[b.index()] = Some(*original_value);
+                    }
+                }
+            }
+        }
+
+        let var_values = self.tcx().mk_substs_from_iter(response.variables.iter().enumerate().map(
+            |(index, info)| {
+                if info.universe() != ty::UniverseIndex::ROOT {
+                    // A variable from inside a binder of the query. While ideally these shouldn't
+                    // exist at all (see the FIXME at the start of this method), we have to deal with
+                    // them for now.
+                    self.infcx.instantiate_canonical_var(DUMMY_SP, info, |idx| {
+                        ty::UniverseIndex::from(prev_universe.index() + idx.index())
+                    })
+                } else if info.is_existential() {
+                    // As an optimization we sometimes avoid creating a new inference variable here.
+                    //
+                    // All new inference variables we create start out in the current universe of the caller.
+                    // This is conceptionally wrong as these inference variables would be able to name
+                    // more placeholders then they should be able to. However the inference variables have
+                    // to "come from somewhere", so by equating them with the original values of the caller
+                    // later on, we pull them down into their correct universe again.
+                    if let Some(v) = opt_values[index] {
+                        v
+                    } else {
+                        self.infcx.instantiate_canonical_var(DUMMY_SP, info, |_| prev_universe)
+                    }
+                } else {
+                    // For placeholders which were already part of the input, we simply map this
+                    // universal bound variable back the placeholder of the input.
+                    original_values[info.expect_anon_placeholder() as usize]
+                }
+            },
+        ));
+
+        CanonicalVarValues { var_values }
+    }
+
+    #[instrument(level = "debug", skip(self, param_env), ret)]
+    fn unify_query_var_values(
+        &self,
+        param_env: ty::ParamEnv<'tcx>,
+        original_values: &[ty::GenericArg<'tcx>],
+        var_values: CanonicalVarValues<'tcx>,
+    ) -> Result<(), NoSolution> {
+        assert_eq!(original_values.len(), var_values.len());
+        for (&orig, response) in iter::zip(original_values, var_values.var_values) {
+            // This can fail due to the occurs check, see
+            // `tests/ui/typeck/lazy-norm/equating-projection-cyclically.rs` for an example
+            // where that can happen.
+            //
+            // FIXME: To deal with #105787 I also expect us to emit nested obligations here at
+            // some point. We can figure out how to deal with this once we actually have
+            // an ICE.
+            let nested_goals = self.eq(param_env, orig, response)?;
+            assert!(nested_goals.is_empty(), "{nested_goals:?}");
+        }
+
+        Ok(())
+    }
+
+    fn register_region_constraints(&mut self, region_constraints: &QueryRegionConstraints<'tcx>) {
+        for &(ty::OutlivesPredicate(lhs, rhs), _) in &region_constraints.outlives {
+            match lhs.unpack() {
+                GenericArgKind::Lifetime(lhs) => self.infcx.region_outlives_predicate(
+                    &ObligationCause::dummy(),
+                    ty::Binder::dummy(ty::OutlivesPredicate(lhs, rhs)),
+                ),
+                GenericArgKind::Type(lhs) => self.infcx.register_region_obligation_with_cause(
+                    lhs,
+                    rhs,
+                    &ObligationCause::dummy(),
+                ),
+                GenericArgKind::Const(_) => bug!("const outlives: {lhs:?}: {rhs:?}"),
+            }
+        }
+
+        for member_constraint in &region_constraints.member_constraints {
+            // FIXME: Deal with member constraints :<
+            let _ = member_constraint;
+        }
+    }
+}
diff --git a/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs b/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs
new file mode 100644
index 000000000..95612674e
--- /dev/null
+++ b/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs
@@ -0,0 +1,184 @@
+use rustc_hir::def_id::DefId;
+use rustc_infer::infer::at::ToTrace;
+use rustc_infer::infer::canonical::CanonicalVarValues;
+use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
+use rustc_infer::infer::{InferCtxt, InferOk, LateBoundRegionConversionTime};
+use rustc_infer::traits::query::NoSolution;
+use rustc_infer::traits::ObligationCause;
+use rustc_middle::infer::unify_key::{ConstVariableOrigin, ConstVariableOriginKind};
+use rustc_middle::ty::{
+    self, Ty, TyCtxt, TypeFoldable, TypeSuperVisitable, TypeVisitable, TypeVisitableExt,
+    TypeVisitor,
+};
+use rustc_span::DUMMY_SP;
+use std::ops::ControlFlow;
+
+use super::search_graph::SearchGraph;
+use super::Goal;
+
+pub struct EvalCtxt<'a, 'tcx> {
+    // FIXME: should be private.
+    pub(super) infcx: &'a InferCtxt<'tcx>,
+    pub(super) var_values: CanonicalVarValues<'tcx>,
+    /// The highest universe index nameable by the caller.
+    ///
+    /// When we enter a new binder inside of the query we create new universes
+    /// which the caller cannot name. We have to be careful with variables from
+    /// these new universes when creating the query response.
+    ///
+    /// Both because these new universes can prevent us from reaching a fixpoint
+    /// if we have a coinductive cycle and because that's the only way we can return
+    /// new placeholders to the caller.
+    pub(super) max_input_universe: ty::UniverseIndex,
+
+    pub(super) search_graph: &'a mut SearchGraph<'tcx>,
+
+    /// This field is used by a debug assertion in [`EvalCtxt::evaluate_goal`],
+    /// see the comment in that method for more details.
+    pub in_projection_eq_hack: bool,
+}
+
+impl<'tcx> EvalCtxt<'_, 'tcx> {
+    pub(super) fn probe<T>(&mut self, f: impl FnOnce(&mut EvalCtxt<'_, 'tcx>) -> T) -> T {
+        self.infcx.probe(|_| f(self))
+    }
+
+    pub(super) fn tcx(&self) -> TyCtxt<'tcx> {
+        self.infcx.tcx
+    }
+
+    pub(super) fn next_ty_infer(&self) -> Ty<'tcx> {
+        self.infcx.next_ty_var(TypeVariableOrigin {
+            kind: TypeVariableOriginKind::MiscVariable,
+            span: DUMMY_SP,
+        })
+    }
+
+    pub(super) fn next_const_infer(&self, ty: Ty<'tcx>) -> ty::Const<'tcx> {
+        self.infcx.next_const_var(
+            ty,
+            ConstVariableOrigin { kind: ConstVariableOriginKind::MiscVariable, span: DUMMY_SP },
+        )
+    }
+
+    /// Is the projection predicate is of the form `exists<T> <Ty as Trait>::Assoc = T`.
+    ///
+    /// This is the case if the `term` is an inference variable in the innermost universe
+    /// and does not occur in any other part of the predicate.
+    pub(super) fn term_is_fully_unconstrained(
+        &self,
+        goal: Goal<'tcx, ty::ProjectionPredicate<'tcx>>,
+    ) -> bool {
+        let term_is_infer = match goal.predicate.term.unpack() {
+            ty::TermKind::Ty(ty) => {
+                if let &ty::Infer(ty::TyVar(vid)) = ty.kind() {
+                    match self.infcx.probe_ty_var(vid) {
+                        Ok(value) => bug!("resolved var in query: {goal:?} {value:?}"),
+                        Err(universe) => universe == self.universe(),
+                    }
+                } else {
+                    false
+                }
+            }
+            ty::TermKind::Const(ct) => {
+                if let ty::ConstKind::Infer(ty::InferConst::Var(vid)) = ct.kind() {
+                    match self.infcx.probe_const_var(vid) {
+                        Ok(value) => bug!("resolved var in query: {goal:?} {value:?}"),
+                        Err(universe) => universe == self.universe(),
+                    }
+                } else {
+                    false
+                }
+            }
+        };
+
+        // Guard against `<T as Trait<?0>>::Assoc = ?0>`.
+        struct ContainsTerm<'tcx> {
+            term: ty::Term<'tcx>,
+        }
+        impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for ContainsTerm<'tcx> {
+            type BreakTy = ();
+            fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
+                if t.needs_infer() {
+                    if ty::Term::from(t) == self.term {
+                        ControlFlow::Break(())
+                    } else {
+                        t.super_visit_with(self)
+                    }
+                } else {
+                    ControlFlow::Continue(())
+                }
+            }
+
+            fn visit_const(&mut self, c: ty::Const<'tcx>) -> ControlFlow<Self::BreakTy> {
+                if c.needs_infer() {
+                    if ty::Term::from(c) == self.term {
+                        ControlFlow::Break(())
+                    } else {
+                        c.super_visit_with(self)
+                    }
+                } else {
+                    ControlFlow::Continue(())
+                }
+            }
+        }
+
+        let mut visitor = ContainsTerm { term: goal.predicate.term };
+
+        term_is_infer
+            && goal.predicate.projection_ty.visit_with(&mut visitor).is_continue()
+            && goal.param_env.visit_with(&mut visitor).is_continue()
+    }
+
+    #[instrument(level = "debug", skip(self, param_env), ret)]
+    pub(super) fn eq<T: ToTrace<'tcx>>(
+        &self,
+        param_env: ty::ParamEnv<'tcx>,
+        lhs: T,
+        rhs: T,
+    ) -> Result<Vec<Goal<'tcx, ty::Predicate<'tcx>>>, NoSolution> {
+        self.infcx
+            .at(&ObligationCause::dummy(), param_env)
+            .eq(lhs, rhs)
+            .map(|InferOk { value: (), obligations }| {
+                obligations.into_iter().map(|o| o.into()).collect()
+            })
+            .map_err(|e| {
+                debug!(?e, "failed to equate");
+                NoSolution
+            })
+    }
+
+    pub(super) fn instantiate_binder_with_infer<T: TypeFoldable<TyCtxt<'tcx>> + Copy>(
+        &self,
+        value: ty::Binder<'tcx, T>,
+    ) -> T {
+        self.infcx.instantiate_binder_with_fresh_vars(
+            DUMMY_SP,
+            LateBoundRegionConversionTime::HigherRankedType,
+            value,
+        )
+    }
+
+    pub(super) fn instantiate_binder_with_placeholders<T: TypeFoldable<TyCtxt<'tcx>> + Copy>(
+        &self,
+        value: ty::Binder<'tcx, T>,
+    ) -> T {
+        self.infcx.instantiate_binder_with_placeholders(value)
+    }
+
+    pub(super) fn resolve_vars_if_possible<T>(&self, value: T) -> T
+    where
+        T: TypeFoldable<TyCtxt<'tcx>>,
+    {
+        self.infcx.resolve_vars_if_possible(value)
+    }
+
+    pub(super) fn fresh_substs_for_item(&self, def_id: DefId) -> ty::SubstsRef<'tcx> {
+        self.infcx.fresh_substs_for_item(DUMMY_SP, def_id)
+    }
+
+    pub(super) fn universe(&self) -> ty::UniverseIndex {
+        self.infcx.universe()
+    }
+}
diff --git a/compiler/rustc_trait_selection/src/solve/fulfill.rs b/compiler/rustc_trait_selection/src/solve/fulfill.rs
index a6240666e..a55b984fd 100644
--- a/compiler/rustc_trait_selection/src/solve/fulfill.rs
+++ b/compiler/rustc_trait_selection/src/solve/fulfill.rs
@@ -1,16 +1,14 @@
 use std::mem;
 
-use rustc_data_structures::fx::FxHashMap;
-use rustc_infer::{
-    infer::InferCtxt,
-    traits::{
-        query::NoSolution, FulfillmentError, FulfillmentErrorCode, PredicateObligation,
-        SelectionError, TraitEngine,
-    },
+use rustc_infer::infer::InferCtxt;
+use rustc_infer::traits::{
+    query::NoSolution, FulfillmentError, FulfillmentErrorCode, MismatchedProjectionTypes,
+    PredicateObligation, SelectionError, TraitEngine,
 };
 use rustc_middle::ty;
+use rustc_middle::ty::error::{ExpectedFound, TypeError};
 
-use super::{search_graph, Certainty, EvalCtxt};
+use super::{Certainty, InferCtxtEvalExt};
 
 /// A trait engine using the new trait solver.
 ///
@@ -42,12 +40,7 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentCtxt<'tcx> {
         self.obligations.push(obligation);
     }
 
-    fn select_all_or_error(&mut self, infcx: &InferCtxt<'tcx>) -> Vec<FulfillmentError<'tcx>> {
-        let errors = self.select_where_possible(infcx);
-        if !errors.is_empty() {
-            return errors;
-        }
-
+    fn collect_remaining_errors(&mut self) -> Vec<FulfillmentError<'tcx>> {
         self.obligations
             .drain(..)
             .map(|obligation| FulfillmentError {
@@ -68,16 +61,65 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentCtxt<'tcx> {
             let mut has_changed = false;
             for obligation in mem::take(&mut self.obligations) {
                 let goal = obligation.clone().into();
-                let search_graph = &mut search_graph::SearchGraph::new(infcx.tcx);
-                let mut ecx = EvalCtxt::new_outside_solver(infcx, search_graph);
-                let (changed, certainty) = match ecx.evaluate_goal(goal) {
+                let (changed, certainty) = match infcx.evaluate_root_goal(goal) {
                     Ok(result) => result,
                     Err(NoSolution) => {
                         errors.push(FulfillmentError {
                             obligation: obligation.clone(),
-                            code: FulfillmentErrorCode::CodeSelectionError(
-                                SelectionError::Unimplemented,
-                            ),
+                            code: match goal.predicate.kind().skip_binder() {
+                                ty::PredicateKind::Clause(ty::Clause::Projection(_)) => {
+                                    FulfillmentErrorCode::CodeProjectionError(
+                                        // FIXME: This could be a `Sorts` if the term is a type
+                                        MismatchedProjectionTypes { err: TypeError::Mismatch },
+                                    )
+                                }
+                                ty::PredicateKind::AliasEq(_, _) => {
+                                    FulfillmentErrorCode::CodeProjectionError(
+                                        MismatchedProjectionTypes { err: TypeError::Mismatch },
+                                    )
+                                }
+                                ty::PredicateKind::Subtype(pred) => {
+                                    let (a, b) = infcx.instantiate_binder_with_placeholders(
+                                        goal.predicate.kind().rebind((pred.a, pred.b)),
+                                    );
+                                    let expected_found = ExpectedFound::new(true, a, b);
+                                    FulfillmentErrorCode::CodeSubtypeError(
+                                        expected_found,
+                                        TypeError::Sorts(expected_found),
+                                    )
+                                }
+                                ty::PredicateKind::Coerce(pred) => {
+                                    let (a, b) = infcx.instantiate_binder_with_placeholders(
+                                        goal.predicate.kind().rebind((pred.a, pred.b)),
+                                    );
+                                    let expected_found = ExpectedFound::new(false, a, b);
+                                    FulfillmentErrorCode::CodeSubtypeError(
+                                        expected_found,
+                                        TypeError::Sorts(expected_found),
+                                    )
+                                }
+                                ty::PredicateKind::ConstEquate(a, b) => {
+                                    let (a, b) = infcx.instantiate_binder_with_placeholders(
+                                        goal.predicate.kind().rebind((a, b)),
+                                    );
+                                    let expected_found = ExpectedFound::new(true, a, b);
+                                    FulfillmentErrorCode::CodeConstEquateError(
+                                        expected_found,
+                                        TypeError::ConstMismatch(expected_found),
+                                    )
+                                }
+                                ty::PredicateKind::Clause(_)
+                                | ty::PredicateKind::WellFormed(_)
+                                | ty::PredicateKind::ObjectSafe(_)
+                                | ty::PredicateKind::ClosureKind(_, _, _)
+                                | ty::PredicateKind::ConstEvaluatable(_)
+                                | ty::PredicateKind::TypeWellFormedFromEnv(_)
+                                | ty::PredicateKind::Ambiguous => {
+                                    FulfillmentErrorCode::CodeSelectionError(
+                                        SelectionError::Unimplemented,
+                                    )
+                                }
+                            },
                             root_obligation: obligation,
                         });
                         continue;
@@ -103,7 +145,10 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentCtxt<'tcx> {
         self.obligations.clone()
     }
 
-    fn relationships(&mut self) -> &mut FxHashMap<ty::TyVid, ty::FoundRelationships> {
-        unimplemented!("Should be moved out of `TraitEngine`")
+    fn drain_unstalled_obligations(
+        &mut self,
+        _: &InferCtxt<'tcx>,
+    ) -> Vec<PredicateObligation<'tcx>> {
+        unimplemented!()
     }
 }
diff --git a/compiler/rustc_trait_selection/src/solve/infcx_ext.rs b/compiler/rustc_trait_selection/src/solve/infcx_ext.rs
deleted file mode 100644
index 42f597c78..000000000
--- a/compiler/rustc_trait_selection/src/solve/infcx_ext.rs
+++ /dev/null
@@ -1,78 +0,0 @@
-use rustc_infer::infer::at::ToTrace;
-use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
-use rustc_infer::infer::{InferCtxt, InferOk, LateBoundRegionConversionTime};
-use rustc_infer::traits::query::NoSolution;
-use rustc_infer::traits::ObligationCause;
-use rustc_middle::infer::unify_key::{ConstVariableOrigin, ConstVariableOriginKind};
-use rustc_middle::ty::{self, Ty, TypeFoldable};
-use rustc_span::DUMMY_SP;
-
-use super::Goal;
-
-/// Methods used inside of the canonical queries of the solver.
-///
-/// Most notably these do not care about diagnostics information.
-/// If you find this while looking for methods to use outside of the
-/// solver, you may look at the implementation of these method for
-/// help.
-pub(super) trait InferCtxtExt<'tcx> {
-    fn next_ty_infer(&self) -> Ty<'tcx>;
-    fn next_const_infer(&self, ty: Ty<'tcx>) -> ty::Const<'tcx>;
-
-    fn eq<T: ToTrace<'tcx>>(
-        &self,
-        param_env: ty::ParamEnv<'tcx>,
-        lhs: T,
-        rhs: T,
-    ) -> Result<Vec<Goal<'tcx, ty::Predicate<'tcx>>>, NoSolution>;
-
-    fn instantiate_bound_vars_with_infer<T: TypeFoldable<'tcx> + Copy>(
-        &self,
-        value: ty::Binder<'tcx, T>,
-    ) -> T;
-}
-
-impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
-    fn next_ty_infer(&self) -> Ty<'tcx> {
-        self.next_ty_var(TypeVariableOrigin {
-            kind: TypeVariableOriginKind::MiscVariable,
-            span: DUMMY_SP,
-        })
-    }
-    fn next_const_infer(&self, ty: Ty<'tcx>) -> ty::Const<'tcx> {
-        self.next_const_var(
-            ty,
-            ConstVariableOrigin { kind: ConstVariableOriginKind::MiscVariable, span: DUMMY_SP },
-        )
-    }
-
-    #[instrument(level = "debug", skip(self, param_env), ret)]
-    fn eq<T: ToTrace<'tcx>>(
-        &self,
-        param_env: ty::ParamEnv<'tcx>,
-        lhs: T,
-        rhs: T,
-    ) -> Result<Vec<Goal<'tcx, ty::Predicate<'tcx>>>, NoSolution> {
-        self.at(&ObligationCause::dummy(), param_env)
-            .define_opaque_types(false)
-            .eq(lhs, rhs)
-            .map(|InferOk { value: (), obligations }| {
-                obligations.into_iter().map(|o| o.into()).collect()
-            })
-            .map_err(|e| {
-                debug!(?e, "failed to equate");
-                NoSolution
-            })
-    }
-
-    fn instantiate_bound_vars_with_infer<T: TypeFoldable<'tcx> + Copy>(
-        &self,
-        value: ty::Binder<'tcx, T>,
-    ) -> T {
-        self.replace_bound_vars_with_fresh_vars(
-            DUMMY_SP,
-            LateBoundRegionConversionTime::HigherRankedType,
-            value,
-        )
-    }
-}
diff --git a/compiler/rustc_trait_selection/src/solve/mod.rs b/compiler/rustc_trait_selection/src/solve/mod.rs
index 32eb84635..57b6a4527 100644
--- a/compiler/rustc_trait_selection/src/solve/mod.rs
+++ b/compiler/rustc_trait_selection/src/solve/mod.rs
@@ -13,31 +13,34 @@
 // preserves universes and creates a unique var (in the highest universe) for each
 // appearance of a region.
 
-// FIXME: `CanonicalVarValues` should be interned and `Copy`.
-
 // FIXME: uses of `infcx.at` need to enable deferred projection equality once that's implemented.
 
 use std::mem;
 
-use rustc_infer::infer::canonical::{Canonical, CanonicalVarKind, CanonicalVarValues};
-use rustc_infer::infer::canonical::{OriginalQueryValues, QueryRegionConstraints, QueryResponse};
+use rustc_hir::def_id::DefId;
+use rustc_infer::infer::canonical::{Canonical, CanonicalVarValues};
 use rustc_infer::infer::{InferCtxt, InferOk, TyCtxtInferExt};
 use rustc_infer::traits::query::NoSolution;
 use rustc_infer::traits::Obligation;
-use rustc_middle::infer::canonical::Certainty as OldCertainty;
+use rustc_middle::traits::solve::{ExternalConstraints, ExternalConstraintsData};
 use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_middle::ty::{RegionOutlivesPredicate, ToPredicate, TypeOutlivesPredicate};
+use rustc_middle::ty::{
+    CoercePredicate, RegionOutlivesPredicate, SubtypePredicate, ToPredicate, TypeOutlivesPredicate,
+};
 use rustc_span::DUMMY_SP;
 
+use crate::solve::search_graph::OverflowHandler;
 use crate::traits::ObligationCause;
 
 mod assembly;
+mod canonical;
+mod eval_ctxt;
 mod fulfill;
-mod infcx_ext;
 mod project_goals;
 mod search_graph;
 mod trait_goals;
 
+pub use eval_ctxt::EvalCtxt;
 pub use fulfill::FulfillmentCtxt;
 
 /// A goal is a statement, i.e. `predicate`, we want to prove
@@ -71,8 +74,7 @@ impl<'tcx, P> From<Obligation<'tcx, P>> for Goal<'tcx, P> {
         Goal { param_env: obligation.param_env, predicate: obligation.predicate }
     }
 }
-
-#[derive(Debug, PartialEq, Eq, Clone, Hash, TypeFoldable, TypeVisitable)]
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, TypeFoldable, TypeVisitable)]
 pub struct Response<'tcx> {
     pub var_values: CanonicalVarValues<'tcx>,
     /// Additional constraints returned by this query.
@@ -80,6 +82,18 @@ pub struct Response<'tcx> {
     pub certainty: Certainty,
 }
 
+trait CanonicalResponseExt {
+    fn has_no_inference_or_external_constraints(&self) -> bool;
+}
+
+impl<'tcx> CanonicalResponseExt for Canonical<'tcx, Response<'tcx>> {
+    fn has_no_inference_or_external_constraints(&self) -> bool {
+        self.value.external_constraints.region_constraints.is_empty()
+            && self.value.var_values.is_identity()
+            && self.value.external_constraints.opaque_types.is_empty()
+    }
+}
+
 #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, TypeFoldable, TypeVisitable)]
 pub enum Certainty {
     Yes,
@@ -87,6 +101,8 @@ pub enum Certainty {
 }
 
 impl Certainty {
+    pub const AMBIGUOUS: Certainty = Certainty::Maybe(MaybeCause::Ambiguity);
+
     /// When proving multiple goals using **AND**, e.g. nested obligations for an impl,
     /// use this function to unify the certainty of these goals
     pub fn unify_and(self, other: Certainty) -> Certainty {
@@ -118,14 +134,6 @@ pub enum MaybeCause {
     Overflow,
 }
 
-/// Additional constraints returned on success.
-#[derive(Debug, PartialEq, Eq, Clone, Hash, TypeFoldable, TypeVisitable, Default)]
-pub struct ExternalConstraints<'tcx> {
-    // FIXME: implement this.
-    regions: (),
-    opaque_types: Vec<(Ty<'tcx>, Ty<'tcx>)>,
-}
-
 type CanonicalGoal<'tcx, T = ty::Predicate<'tcx>> = Canonical<'tcx, Goal<'tcx, T>>;
 type CanonicalResponse<'tcx> = Canonical<'tcx, Response<'tcx>>;
 /// The result of evaluating a canonical query.
@@ -136,78 +144,70 @@ type CanonicalResponse<'tcx> = Canonical<'tcx, Response<'tcx>>;
 /// solver, merge the two responses again.
 pub type QueryResult<'tcx> = Result<CanonicalResponse<'tcx>, NoSolution>;
 
-pub trait TyCtxtExt<'tcx> {
-    fn evaluate_goal(self, goal: CanonicalGoal<'tcx>) -> QueryResult<'tcx>;
-}
-
-impl<'tcx> TyCtxtExt<'tcx> for TyCtxt<'tcx> {
-    fn evaluate_goal(self, goal: CanonicalGoal<'tcx>) -> QueryResult<'tcx> {
-        let mut search_graph = search_graph::SearchGraph::new(self);
-        EvalCtxt::evaluate_canonical_goal(self, &mut search_graph, goal)
-    }
+pub trait InferCtxtEvalExt<'tcx> {
+    /// Evaluates a goal from **outside** of the trait solver.
+    ///
+    /// Using this while inside of the solver is wrong as it uses a new
+    /// search graph which would break cycle detection.
+    fn evaluate_root_goal(
+        &self,
+        goal: Goal<'tcx, ty::Predicate<'tcx>>,
+    ) -> Result<(bool, Certainty), NoSolution>;
 }
 
-struct EvalCtxt<'a, 'tcx> {
-    infcx: &'a InferCtxt<'tcx>,
-    var_values: CanonicalVarValues<'tcx>,
+impl<'tcx> InferCtxtEvalExt<'tcx> for InferCtxt<'tcx> {
+    fn evaluate_root_goal(
+        &self,
+        goal: Goal<'tcx, ty::Predicate<'tcx>>,
+    ) -> Result<(bool, Certainty), NoSolution> {
+        let mut search_graph = search_graph::SearchGraph::new(self.tcx);
+
+        let result = EvalCtxt {
+            search_graph: &mut search_graph,
+            infcx: self,
+            // Only relevant when canonicalizing the response.
+            max_input_universe: ty::UniverseIndex::ROOT,
+            var_values: CanonicalVarValues::dummy(),
+            in_projection_eq_hack: false,
+        }
+        .evaluate_goal(goal);
 
-    search_graph: &'a mut search_graph::SearchGraph<'tcx>,
+        assert!(search_graph.is_empty());
+        result
+    }
 }
 
 impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
-    fn tcx(&self) -> TyCtxt<'tcx> {
-        self.infcx.tcx
-    }
-
-    /// Creates a new evaluation context outside of the trait solver.
+    /// The entry point of the solver.
     ///
-    /// With this solver making a canonical response doesn't make much sense.
-    /// The `search_graph` for this solver has to be completely empty.
-    fn new_outside_solver(
-        infcx: &'a InferCtxt<'tcx>,
-        search_graph: &'a mut search_graph::SearchGraph<'tcx>,
-    ) -> EvalCtxt<'a, 'tcx> {
-        assert!(search_graph.is_empty());
-        EvalCtxt { infcx, var_values: CanonicalVarValues::dummy(), search_graph }
-    }
-
+    /// This function deals with (coinductive) cycles, overflow, and caching
+    /// and then calls [`EvalCtxt::compute_goal`] which contains the actual
+    /// logic of the solver.
+    ///
+    /// Instead of calling this function directly, use either [EvalCtxt::evaluate_goal]
+    /// if you're inside of the solver or [InferCtxtEvalExt::evaluate_root_goal] if you're
+    /// outside of it.
     #[instrument(level = "debug", skip(tcx, search_graph), ret)]
     fn evaluate_canonical_goal(
         tcx: TyCtxt<'tcx>,
         search_graph: &'a mut search_graph::SearchGraph<'tcx>,
         canonical_goal: CanonicalGoal<'tcx>,
     ) -> QueryResult<'tcx> {
-        match search_graph.try_push_stack(tcx, canonical_goal) {
-            Ok(()) => {}
-            // Our goal is already on the stack, eager return.
-            Err(response) => return response,
-        }
-
-        // We may have to repeatedly recompute the goal in case of coinductive cycles,
-        // check out the `cache` module for more information.
+        // Deal with overflow, caching, and coinduction.
         //
-        // FIXME: Similar to `evaluate_all`, this has to check for overflow.
-        loop {
+        // The actual solver logic happens in `ecx.compute_goal`.
+        search_graph.with_new_goal(tcx, canonical_goal, |search_graph| {
             let (ref infcx, goal, var_values) =
                 tcx.infer_ctxt().build_with_canonical(DUMMY_SP, &canonical_goal);
-            let mut ecx = EvalCtxt { infcx, var_values, search_graph };
-            let result = ecx.compute_goal(goal);
-
-            // FIXME: `Response` should be `Copy`
-            if search_graph.try_finalize_goal(tcx, canonical_goal, result.clone()) {
-                return result;
-            }
-        }
-    }
-
-    fn make_canonical_response(&self, certainty: Certainty) -> QueryResult<'tcx> {
-        let external_constraints = take_external_constraints(self.infcx)?;
-
-        Ok(self.infcx.canonicalize_response(Response {
-            var_values: self.var_values.clone(),
-            external_constraints,
-            certainty,
-        }))
+            let mut ecx = EvalCtxt {
+                infcx,
+                var_values,
+                max_input_universe: canonical_goal.max_universe,
+                search_graph,
+                in_projection_eq_hack: false,
+            };
+            ecx.compute_goal(goal)
+        })
     }
 
     /// Recursively evaluates `goal`, returning whether any inference vars have
@@ -216,14 +216,39 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
         &mut self,
         goal: Goal<'tcx, ty::Predicate<'tcx>>,
     ) -> Result<(bool, Certainty), NoSolution> {
-        let mut orig_values = OriginalQueryValues::default();
-        let canonical_goal = self.infcx.canonicalize_query(goal, &mut orig_values);
+        let (orig_values, canonical_goal) = self.canonicalize_goal(goal);
         let canonical_response =
             EvalCtxt::evaluate_canonical_goal(self.tcx(), self.search_graph, canonical_goal)?;
-        Ok((
-            !canonical_response.value.var_values.is_identity(),
-            instantiate_canonical_query_response(self.infcx, &orig_values, canonical_response),
-        ))
+
+        let has_changed = !canonical_response.value.var_values.is_identity();
+        let certainty = self.instantiate_and_apply_query_response(
+            goal.param_env,
+            orig_values,
+            canonical_response,
+        )?;
+
+        // Check that rerunning this query with its inference constraints applied
+        // doesn't result in new inference constraints and has the same result.
+        //
+        // If we have projection goals like `<T as Trait>::Assoc == u32` we recursively
+        // call `exists<U> <T as Trait>::Assoc == U` to enable better caching. This goal
+        // could constrain `U` to `u32` which would cause this check to result in a
+        // solver cycle.
+        if cfg!(debug_assertions)
+            && has_changed
+            && !self.in_projection_eq_hack
+            && !self.search_graph.in_cycle()
+        {
+            let (_orig_values, canonical_goal) = self.canonicalize_goal(goal);
+            let canonical_response =
+                EvalCtxt::evaluate_canonical_goal(self.tcx(), self.search_graph, canonical_goal)?;
+            if !canonical_response.value.var_values.is_identity() {
+                bug!("unstable result: {goal:?} {canonical_goal:?} {canonical_response:?}");
+            }
+            assert_eq!(certainty, canonical_response.value.certainty);
+        }
+
+        Ok((has_changed, certainty))
     }
 
     fn compute_goal(&mut self, goal: Goal<'tcx, ty::Predicate<'tcx>>) -> QueryResult<'tcx> {
@@ -243,19 +268,40 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
                 ty::PredicateKind::Clause(ty::Clause::RegionOutlives(predicate)) => {
                     self.compute_region_outlives_goal(Goal { param_env, predicate })
                 }
+                ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(ct, ty)) => {
+                    self.compute_const_arg_has_type_goal(Goal { param_env, predicate: (ct, ty) })
+                }
+                ty::PredicateKind::Subtype(predicate) => {
+                    self.compute_subtype_goal(Goal { param_env, predicate })
+                }
+                ty::PredicateKind::Coerce(predicate) => {
+                    self.compute_coerce_goal(Goal { param_env, predicate })
+                }
+                ty::PredicateKind::ClosureKind(def_id, substs, kind) => self
+                    .compute_closure_kind_goal(Goal {
+                        param_env,
+                        predicate: (def_id, substs, kind),
+                    }),
+                ty::PredicateKind::ObjectSafe(trait_def_id) => {
+                    self.compute_object_safe_goal(trait_def_id)
+                }
+                ty::PredicateKind::WellFormed(arg) => {
+                    self.compute_well_formed_goal(Goal { param_env, predicate: arg })
+                }
+                ty::PredicateKind::Ambiguous => self.make_canonical_response(Certainty::AMBIGUOUS),
                 // FIXME: implement these predicates :)
-                ty::PredicateKind::WellFormed(_)
-                | ty::PredicateKind::ObjectSafe(_)
-                | ty::PredicateKind::ClosureKind(_, _, _)
-                | ty::PredicateKind::Subtype(_)
-                | ty::PredicateKind::Coerce(_)
-                | ty::PredicateKind::ConstEvaluatable(_)
-                | ty::PredicateKind::ConstEquate(_, _)
-                | ty::PredicateKind::TypeWellFormedFromEnv(_)
-                | ty::PredicateKind::Ambiguous => self.make_canonical_response(Certainty::Yes),
+                ty::PredicateKind::ConstEvaluatable(_) | ty::PredicateKind::ConstEquate(_, _) => {
+                    self.make_canonical_response(Certainty::Yes)
+                }
+                ty::PredicateKind::TypeWellFormedFromEnv(..) => {
+                    bug!("TypeWellFormedFromEnv is only used for Chalk")
+                }
+                ty::PredicateKind::AliasEq(lhs, rhs) => {
+                    self.compute_alias_eq_goal(Goal { param_env, predicate: (lhs, rhs) })
+                }
             }
         } else {
-            let kind = self.infcx.replace_bound_vars_with_placeholders(kind);
+            let kind = self.infcx.instantiate_binder_with_placeholders(kind);
             let goal = goal.with(self.tcx(), ty::Binder::dummy(kind));
             let (_, certainty) = self.evaluate_goal(goal)?;
             self.make_canonical_response(certainty)
@@ -264,102 +310,255 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
 
     fn compute_type_outlives_goal(
         &mut self,
-        _goal: Goal<'tcx, TypeOutlivesPredicate<'tcx>>,
+        goal: Goal<'tcx, TypeOutlivesPredicate<'tcx>>,
     ) -> QueryResult<'tcx> {
+        let ty::OutlivesPredicate(ty, lt) = goal.predicate;
+        self.infcx.register_region_obligation_with_cause(ty, lt, &ObligationCause::dummy());
         self.make_canonical_response(Certainty::Yes)
     }
 
     fn compute_region_outlives_goal(
         &mut self,
-        _goal: Goal<'tcx, RegionOutlivesPredicate<'tcx>>,
+        goal: Goal<'tcx, RegionOutlivesPredicate<'tcx>>,
     ) -> QueryResult<'tcx> {
+        self.infcx.region_outlives_predicate(
+            &ObligationCause::dummy(),
+            ty::Binder::dummy(goal.predicate),
+        );
         self.make_canonical_response(Certainty::Yes)
     }
+
+    fn compute_coerce_goal(
+        &mut self,
+        goal: Goal<'tcx, CoercePredicate<'tcx>>,
+    ) -> QueryResult<'tcx> {
+        self.compute_subtype_goal(Goal {
+            param_env: goal.param_env,
+            predicate: SubtypePredicate {
+                a_is_expected: false,
+                a: goal.predicate.a,
+                b: goal.predicate.b,
+            },
+        })
+    }
+
+    fn compute_subtype_goal(
+        &mut self,
+        goal: Goal<'tcx, SubtypePredicate<'tcx>>,
+    ) -> QueryResult<'tcx> {
+        if goal.predicate.a.is_ty_var() && goal.predicate.b.is_ty_var() {
+            // FIXME: Do we want to register a subtype relation between these vars?
+            // That won't actually reflect in the query response, so it seems moot.
+            self.make_canonical_response(Certainty::AMBIGUOUS)
+        } else {
+            let InferOk { value: (), obligations } = self
+                .infcx
+                .at(&ObligationCause::dummy(), goal.param_env)
+                .sub(goal.predicate.a, goal.predicate.b)?;
+            self.evaluate_all_and_make_canonical_response(
+                obligations.into_iter().map(|pred| pred.into()).collect(),
+            )
+        }
+    }
+
+    fn compute_closure_kind_goal(
+        &mut self,
+        goal: Goal<'tcx, (DefId, ty::SubstsRef<'tcx>, ty::ClosureKind)>,
+    ) -> QueryResult<'tcx> {
+        let (_, substs, expected_kind) = goal.predicate;
+        let found_kind = substs.as_closure().kind_ty().to_opt_closure_kind();
+
+        let Some(found_kind) = found_kind else {
+            return self.make_canonical_response(Certainty::AMBIGUOUS);
+        };
+        if found_kind.extends(expected_kind) {
+            self.make_canonical_response(Certainty::Yes)
+        } else {
+            Err(NoSolution)
+        }
+    }
+
+    fn compute_object_safe_goal(&mut self, trait_def_id: DefId) -> QueryResult<'tcx> {
+        if self.tcx().check_is_object_safe(trait_def_id) {
+            self.make_canonical_response(Certainty::Yes)
+        } else {
+            Err(NoSolution)
+        }
+    }
+
+    fn compute_well_formed_goal(
+        &mut self,
+        goal: Goal<'tcx, ty::GenericArg<'tcx>>,
+    ) -> QueryResult<'tcx> {
+        match crate::traits::wf::unnormalized_obligations(
+            self.infcx,
+            goal.param_env,
+            goal.predicate,
+        ) {
+            Some(obligations) => self.evaluate_all_and_make_canonical_response(
+                obligations.into_iter().map(|o| o.into()).collect(),
+            ),
+            None => self.make_canonical_response(Certainty::AMBIGUOUS),
+        }
+    }
+
+    #[instrument(level = "debug", skip(self), ret)]
+    fn compute_alias_eq_goal(
+        &mut self,
+        goal: Goal<'tcx, (ty::Term<'tcx>, ty::Term<'tcx>)>,
+    ) -> QueryResult<'tcx> {
+        let tcx = self.tcx();
+
+        let evaluate_normalizes_to = |ecx: &mut EvalCtxt<'_, 'tcx>, alias, other| {
+            debug!("evaluate_normalizes_to(alias={:?}, other={:?})", alias, other);
+            let r = ecx.probe(|ecx| {
+                let (_, certainty) = ecx.evaluate_goal(goal.with(
+                    tcx,
+                    ty::Binder::dummy(ty::ProjectionPredicate {
+                        projection_ty: alias,
+                        term: other,
+                    }),
+                ))?;
+                ecx.make_canonical_response(certainty)
+            });
+            debug!("evaluate_normalizes_to(..) -> {:?}", r);
+            r
+        };
+
+        if goal.predicate.0.is_infer() || goal.predicate.1.is_infer() {
+            bug!(
+                "`AliasEq` goal with an infer var on lhs or rhs which should have been instantiated"
+            );
+        }
+
+        match (
+            goal.predicate.0.to_alias_term_no_opaque(tcx),
+            goal.predicate.1.to_alias_term_no_opaque(tcx),
+        ) {
+            (None, None) => bug!("`AliasEq` goal without an alias on either lhs or rhs"),
+            (Some(alias), None) => evaluate_normalizes_to(self, alias, goal.predicate.1),
+            (None, Some(alias)) => evaluate_normalizes_to(self, alias, goal.predicate.0),
+            (Some(alias_lhs), Some(alias_rhs)) => {
+                debug!("compute_alias_eq_goal: both sides are aliases");
+
+                let mut candidates = Vec::with_capacity(3);
+
+                // Evaluate all 3 potential candidates for the alias' being equal
+                candidates.push(evaluate_normalizes_to(self, alias_lhs, goal.predicate.1));
+                candidates.push(evaluate_normalizes_to(self, alias_rhs, goal.predicate.0));
+                candidates.push(self.probe(|this| {
+                    debug!("compute_alias_eq_goal: alias defids are equal, equating substs");
+                    let nested_goals = this.eq(goal.param_env, alias_lhs, alias_rhs)?;
+                    this.evaluate_all_and_make_canonical_response(nested_goals)
+                }));
+
+                debug!(?candidates);
+
+                self.try_merge_responses(candidates.into_iter())
+            }
+        }
+    }
+
+    #[instrument(level = "debug", skip(self), ret)]
+    fn compute_const_arg_has_type_goal(
+        &mut self,
+        goal: Goal<'tcx, (ty::Const<'tcx>, Ty<'tcx>)>,
+    ) -> QueryResult<'tcx> {
+        let (ct, ty) = goal.predicate;
+        let nested_goals = self.eq(goal.param_env, ct.ty(), ty)?;
+        self.evaluate_all_and_make_canonical_response(nested_goals)
+    }
 }
 
 impl<'tcx> EvalCtxt<'_, 'tcx> {
+    // Recursively evaluates a list of goals to completion, returning the certainty
+    // of all of the goals.
     fn evaluate_all(
         &mut self,
         mut goals: Vec<Goal<'tcx, ty::Predicate<'tcx>>>,
     ) -> Result<Certainty, NoSolution> {
         let mut new_goals = Vec::new();
-        self.repeat_while_none(|this| {
-            let mut has_changed = Err(Certainty::Yes);
-            for goal in goals.drain(..) {
-                let (changed, certainty) = match this.evaluate_goal(goal) {
-                    Ok(result) => result,
-                    Err(NoSolution) => return Some(Err(NoSolution)),
-                };
-
-                if changed {
-                    has_changed = Ok(());
-                }
+        self.repeat_while_none(
+            |_| Ok(Certainty::Maybe(MaybeCause::Overflow)),
+            |this| {
+                let mut has_changed = Err(Certainty::Yes);
+                for goal in goals.drain(..) {
+                    let (changed, certainty) = match this.evaluate_goal(goal) {
+                        Ok(result) => result,
+                        Err(NoSolution) => return Some(Err(NoSolution)),
+                    };
+
+                    if changed {
+                        has_changed = Ok(());
+                    }
 
-                match certainty {
-                    Certainty::Yes => {}
-                    Certainty::Maybe(_) => {
-                        new_goals.push(goal);
-                        has_changed = has_changed.map_err(|c| c.unify_and(certainty));
+                    match certainty {
+                        Certainty::Yes => {}
+                        Certainty::Maybe(_) => {
+                            new_goals.push(goal);
+                            has_changed = has_changed.map_err(|c| c.unify_and(certainty));
+                        }
                     }
                 }
-            }
 
-            match has_changed {
-                Ok(()) => {
-                    mem::swap(&mut new_goals, &mut goals);
-                    None
+                match has_changed {
+                    Ok(()) => {
+                        mem::swap(&mut new_goals, &mut goals);
+                        None
+                    }
+                    Err(certainty) => Some(Ok(certainty)),
                 }
-                Err(certainty) => Some(Ok(certainty)),
-            }
-        })
+            },
+        )
     }
 
+    // Recursively evaluates a list of goals to completion, making a query response.
+    //
+    // This is just a convenient way of calling [`EvalCtxt::evaluate_all`],
+    // then [`EvalCtxt::make_canonical_response`].
     fn evaluate_all_and_make_canonical_response(
         &mut self,
         goals: Vec<Goal<'tcx, ty::Predicate<'tcx>>>,
     ) -> QueryResult<'tcx> {
         self.evaluate_all(goals).and_then(|certainty| self.make_canonical_response(certainty))
     }
-}
 
-#[instrument(level = "debug", skip(infcx), ret)]
-fn take_external_constraints<'tcx>(
-    infcx: &InferCtxt<'tcx>,
-) -> Result<ExternalConstraints<'tcx>, NoSolution> {
-    let region_obligations = infcx.take_registered_region_obligations();
-    let opaque_types = infcx.take_opaque_types_for_query_response();
-    Ok(ExternalConstraints {
-        // FIXME: Now that's definitely wrong :)
-        //
-        // Should also do the leak check here I think
-        regions: drop(region_obligations),
-        opaque_types,
-    })
-}
+    fn try_merge_responses(
+        &mut self,
+        responses: impl Iterator<Item = QueryResult<'tcx>>,
+    ) -> QueryResult<'tcx> {
+        let candidates = responses.into_iter().flatten().collect::<Box<[_]>>();
 
-fn instantiate_canonical_query_response<'tcx>(
-    infcx: &InferCtxt<'tcx>,
-    original_values: &OriginalQueryValues<'tcx>,
-    response: CanonicalResponse<'tcx>,
-) -> Certainty {
-    let Ok(InferOk { value, obligations }) = infcx
-        .instantiate_query_response_and_region_obligations(
-            &ObligationCause::dummy(),
-            ty::ParamEnv::empty(),
-            original_values,
-            &response.unchecked_map(|resp| QueryResponse {
-                var_values: resp.var_values,
-                region_constraints: QueryRegionConstraints::default(),
-                certainty: match resp.certainty {
-                    Certainty::Yes => OldCertainty::Proven,
-                    Certainty::Maybe(_) => OldCertainty::Ambiguous,
-                },
-                opaque_types: resp.external_constraints.opaque_types,
-                value: resp.certainty,
-            }),
-        ) else { bug!(); };
-    assert!(obligations.is_empty());
-    value
+        if candidates.is_empty() {
+            return Err(NoSolution);
+        }
+
+        // FIXME(-Ztreat-solver=next): We should instead try to find a `Certainty::Yes` response with
+        // a subset of the constraints that all the other responses have.
+        let one = candidates[0];
+        if candidates[1..].iter().all(|resp| resp == &one) {
+            return Ok(one);
+        }
+
+        if let Some(response) = candidates.iter().find(|response| {
+            response.value.certainty == Certainty::Yes
+                && response.has_no_inference_or_external_constraints()
+        }) {
+            return Ok(*response);
+        }
+
+        let certainty = candidates.iter().fold(Certainty::AMBIGUOUS, |certainty, response| {
+            certainty.unify_and(response.value.certainty)
+        });
+        // FIXME(-Ztrait-solver=next): We should take the intersection of the constraints on all the
+        // responses and use that for the constraints of this ambiguous response.
+        let response = self.make_canonical_response(certainty);
+        if let Ok(response) = &response {
+            assert!(response.has_no_inference_or_external_constraints());
+        }
+
+        response
+    }
 }
 
 pub(super) fn response_no_constraints<'tcx>(
@@ -367,33 +566,14 @@ pub(super) fn response_no_constraints<'tcx>(
     goal: Canonical<'tcx, impl Sized>,
     certainty: Certainty,
 ) -> QueryResult<'tcx> {
-    let var_values = goal
-        .variables
-        .iter()
-        .enumerate()
-        .map(|(i, info)| match info.kind {
-            CanonicalVarKind::Ty(_) | CanonicalVarKind::PlaceholderTy(_) => {
-                tcx.mk_ty(ty::Bound(ty::INNERMOST, ty::BoundVar::from_usize(i).into())).into()
-            }
-            CanonicalVarKind::Region(_) | CanonicalVarKind::PlaceholderRegion(_) => {
-                let br = ty::BoundRegion {
-                    var: ty::BoundVar::from_usize(i),
-                    kind: ty::BrAnon(i as u32, None),
-                };
-                tcx.mk_region(ty::ReLateBound(ty::INNERMOST, br)).into()
-            }
-            CanonicalVarKind::Const(_, ty) | CanonicalVarKind::PlaceholderConst(_, ty) => tcx
-                .mk_const(ty::ConstKind::Bound(ty::INNERMOST, ty::BoundVar::from_usize(i)), ty)
-                .into(),
-        })
-        .collect();
-
     Ok(Canonical {
         max_universe: goal.max_universe,
         variables: goal.variables,
         value: Response {
-            var_values: CanonicalVarValues { var_values },
-            external_constraints: Default::default(),
+            var_values: CanonicalVarValues::make_identity(tcx, goal.variables),
+            // FIXME: maybe we should store the "no response" version in tcx, like
+            // we do for tcx.types and stuff.
+            external_constraints: tcx.mk_external_constraints(ExternalConstraintsData::default()),
             certainty,
         },
     })
diff --git a/compiler/rustc_trait_selection/src/solve/project_goals.rs b/compiler/rustc_trait_selection/src/solve/project_goals.rs
index e39fa0533..33c66d072 100644
--- a/compiler/rustc_trait_selection/src/solve/project_goals.rs
+++ b/compiler/rustc_trait_selection/src/solve/project_goals.rs
@@ -1,23 +1,22 @@
 use crate::traits::{specialization_graph, translate_substs};
 
-use super::assembly::{self, Candidate, CandidateSource};
-use super::infcx_ext::InferCtxtExt;
+use super::assembly;
 use super::trait_goals::structural_traits;
-use super::{Certainty, EvalCtxt, Goal, MaybeCause, QueryResult};
+use super::{Certainty, EvalCtxt, Goal, QueryResult};
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
+use rustc_hir::LangItem;
 use rustc_infer::infer::InferCtxt;
 use rustc_infer::traits::query::NoSolution;
 use rustc_infer::traits::specialization_graph::LeafDef;
 use rustc_infer::traits::Reveal;
 use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams};
+use rustc_middle::ty::ProjectionPredicate;
 use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_middle::ty::{ProjectionPredicate, TypeSuperVisitable, TypeVisitor};
-use rustc_middle::ty::{ToPredicate, TypeVisitable};
-use rustc_span::DUMMY_SP;
+use rustc_middle::ty::{ToPredicate, TypeVisitableExt};
+use rustc_span::{sym, DUMMY_SP};
 use std::iter;
-use std::ops::ControlFlow;
 
 impl<'tcx> EvalCtxt<'_, 'tcx> {
     pub(super) fn compute_projection_goal(
@@ -27,151 +26,62 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         // To only compute normalization once for each projection we only
         // normalize if the expected term is an unconstrained inference variable.
         //
-        // E.g. for `<T as Trait>::Assoc = u32` we recursively compute the goal
-        // `exists<U> <T as Trait>::Assoc = U` and then take the resulting type for
+        // E.g. for `<T as Trait>::Assoc == u32` we recursively compute the goal
+        // `exists<U> <T as Trait>::Assoc == U` and then take the resulting type for
         // `U` and equate it with `u32`. This means that we don't need a separate
         // projection cache in the solver.
         if self.term_is_fully_unconstrained(goal) {
             let candidates = self.assemble_and_evaluate_candidates(goal);
-            self.merge_project_candidates(candidates)
+            self.merge_candidates_and_discard_reservation_impls(candidates)
         } else {
             let predicate = goal.predicate;
             let unconstrained_rhs = match predicate.term.unpack() {
-                ty::TermKind::Ty(_) => self.infcx.next_ty_infer().into(),
-                ty::TermKind::Const(ct) => self.infcx.next_const_infer(ct.ty()).into(),
+                ty::TermKind::Ty(_) => self.next_ty_infer().into(),
+                ty::TermKind::Const(ct) => self.next_const_infer(ct.ty()).into(),
             };
             let unconstrained_predicate = ty::Clause::Projection(ProjectionPredicate {
                 projection_ty: goal.predicate.projection_ty,
                 term: unconstrained_rhs,
             });
-            let (_has_changed, normalize_certainty) =
-                self.evaluate_goal(goal.with(self.tcx(), unconstrained_predicate))?;
+            let (_has_changed, normalize_certainty) = self.in_projection_eq_hack(|this| {
+                this.evaluate_goal(goal.with(this.tcx(), unconstrained_predicate))
+            })?;
 
-            let nested_eq_goals =
-                self.infcx.eq(goal.param_env, unconstrained_rhs, predicate.term)?;
+            let nested_eq_goals = self.eq(goal.param_env, unconstrained_rhs, predicate.term)?;
             let eval_certainty = self.evaluate_all(nested_eq_goals)?;
             self.make_canonical_response(normalize_certainty.unify_and(eval_certainty))
         }
     }
 
-    /// Is the projection predicate is of the form `exists<T> <Ty as Trait>::Assoc = T`.
-    ///
-    /// This is the case if the `term` is an inference variable in the innermost universe
-    /// and does not occur in any other part of the predicate.
-    fn term_is_fully_unconstrained(&self, goal: Goal<'tcx, ProjectionPredicate<'tcx>>) -> bool {
-        let infcx = self.infcx;
-        let term_is_infer = match goal.predicate.term.unpack() {
-            ty::TermKind::Ty(ty) => {
-                if let &ty::Infer(ty::TyVar(vid)) = ty.kind() {
-                    match infcx.probe_ty_var(vid) {
-                        Ok(value) => bug!("resolved var in query: {goal:?} {value:?}"),
-                        Err(universe) => universe == infcx.universe(),
-                    }
-                } else {
-                    false
-                }
-            }
-            ty::TermKind::Const(ct) => {
-                if let ty::ConstKind::Infer(ty::InferConst::Var(vid)) = ct.kind() {
-                    match self.infcx.probe_const_var(vid) {
-                        Ok(value) => bug!("resolved var in query: {goal:?} {value:?}"),
-                        Err(universe) => universe == infcx.universe(),
-                    }
-                } else {
-                    false
-                }
-            }
-        };
-
-        // Guard against `<T as Trait<?0>>::Assoc = ?0>`.
-        struct ContainsTerm<'tcx> {
-            term: ty::Term<'tcx>,
-        }
-        impl<'tcx> TypeVisitor<'tcx> for ContainsTerm<'tcx> {
-            type BreakTy = ();
-            fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
-                if t.needs_infer() {
-                    if ty::Term::from(t) == self.term {
-                        ControlFlow::BREAK
-                    } else {
-                        t.super_visit_with(self)
-                    }
-                } else {
-                    ControlFlow::CONTINUE
-                }
-            }
-
-            fn visit_const(&mut self, c: ty::Const<'tcx>) -> ControlFlow<Self::BreakTy> {
-                if c.needs_infer() {
-                    if ty::Term::from(c) == self.term {
-                        ControlFlow::BREAK
-                    } else {
-                        c.super_visit_with(self)
-                    }
-                } else {
-                    ControlFlow::CONTINUE
-                }
-            }
-        }
-
-        let mut visitor = ContainsTerm { term: goal.predicate.term };
-
-        term_is_infer
-            && goal.predicate.projection_ty.visit_with(&mut visitor).is_continue()
-            && goal.param_env.visit_with(&mut visitor).is_continue()
+    /// This sets a flag used by a debug assert in [`EvalCtxt::evaluate_goal`],
+    /// see the comment in that method for more details.
+    fn in_projection_eq_hack<T>(&mut self, f: impl FnOnce(&mut Self) -> T) -> T {
+        self.in_projection_eq_hack = true;
+        let result = f(self);
+        self.in_projection_eq_hack = false;
+        result
     }
 
-    fn merge_project_candidates(
+    /// After normalizing the projection to `normalized_alias` with the given
+    /// `normalization_certainty`, constrain the inference variable `term` to it
+    /// and return a query response.
+    fn eq_term_and_make_canonical_response(
         &mut self,
-        mut candidates: Vec<Candidate<'tcx>>,
+        goal: Goal<'tcx, ProjectionPredicate<'tcx>>,
+        normalization_certainty: Certainty,
+        normalized_alias: impl Into<ty::Term<'tcx>>,
     ) -> QueryResult<'tcx> {
-        match candidates.len() {
-            0 => return Err(NoSolution),
-            1 => return Ok(candidates.pop().unwrap().result),
-            _ => {}
-        }
-
-        if candidates.len() > 1 {
-            let mut i = 0;
-            'outer: while i < candidates.len() {
-                for j in (0..candidates.len()).filter(|&j| i != j) {
-                    if self.project_candidate_should_be_dropped_in_favor_of(
-                        &candidates[i],
-                        &candidates[j],
-                    ) {
-                        debug!(candidate = ?candidates[i], "Dropping candidate #{}/{}", i, candidates.len());
-                        candidates.swap_remove(i);
-                        continue 'outer;
-                    }
-                }
+        // The term of our goal should be fully unconstrained, so this should never fail.
+        //
+        // It can however be ambiguous when the `normalized_alias` contains a projection.
+        let nested_goals = self
+            .eq(goal.param_env, goal.predicate.term, normalized_alias.into())
+            .expect("failed to unify with unconstrained term");
 
-                debug!(candidate = ?candidates[i], "Retaining candidate #{}/{}", i, candidates.len());
-                // If there are *STILL* multiple candidates, give up
-                // and report ambiguity.
-                i += 1;
-                if i > 1 {
-                    debug!("multiple matches, ambig");
-                    // FIXME: return overflow if all candidates overflow, otherwise return ambiguity.
-                    unimplemented!();
-                }
-            }
-        }
+        let unify_certainty =
+            self.evaluate_all(nested_goals).expect("failed to unify with unconstrained term");
 
-        Ok(candidates.pop().unwrap().result)
-    }
-
-    fn project_candidate_should_be_dropped_in_favor_of(
-        &self,
-        candidate: &Candidate<'tcx>,
-        other: &Candidate<'tcx>,
-    ) -> bool {
-        // FIXME: implement this
-        match (candidate.source, other.source) {
-            (CandidateSource::Impl(_), _)
-            | (CandidateSource::ParamEnv(_), _)
-            | (CandidateSource::BuiltinImpl, _)
-            | (CandidateSource::AliasBound(_), _) => unimplemented!(),
-        }
+        self.make_canonical_response(normalization_certainty.unify_and(unify_certainty))
     }
 }
 
@@ -188,6 +98,82 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
         self.trait_def_id(tcx)
     }
 
+    fn consider_implied_clause(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+        assumption: ty::Predicate<'tcx>,
+        requirements: impl IntoIterator<Item = Goal<'tcx, ty::Predicate<'tcx>>>,
+    ) -> QueryResult<'tcx> {
+        if let Some(poly_projection_pred) = assumption.to_opt_poly_projection_pred()
+            && poly_projection_pred.projection_def_id() == goal.predicate.def_id()
+        {
+            ecx.probe(|ecx| {
+                let assumption_projection_pred =
+                    ecx.instantiate_binder_with_infer(poly_projection_pred);
+                let mut nested_goals = ecx.eq(
+                    goal.param_env,
+                    goal.predicate.projection_ty,
+                    assumption_projection_pred.projection_ty,
+                )?;
+                nested_goals.extend(requirements);
+                let subst_certainty = ecx.evaluate_all(nested_goals)?;
+
+                ecx.eq_term_and_make_canonical_response(
+                    goal,
+                    subst_certainty,
+                    assumption_projection_pred.term,
+                )
+            })
+        } else {
+            Err(NoSolution)
+        }
+    }
+
+    fn consider_object_bound_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+        assumption: ty::Predicate<'tcx>,
+    ) -> QueryResult<'tcx> {
+        if let Some(poly_projection_pred) = assumption.to_opt_poly_projection_pred()
+            && poly_projection_pred.projection_def_id() == goal.predicate.def_id()
+        {
+            ecx.probe(|ecx| {
+                let assumption_projection_pred =
+                    ecx.instantiate_binder_with_infer(poly_projection_pred);
+                let mut nested_goals = ecx.eq(
+                    goal.param_env,
+                    goal.predicate.projection_ty,
+                    assumption_projection_pred.projection_ty,
+                )?;
+
+                let tcx = ecx.tcx();
+                let ty::Dynamic(bounds, _, _) = *goal.predicate.self_ty().kind() else {
+                    bug!("expected object type in `consider_object_bound_candidate`");
+                };
+                nested_goals.extend(
+                    structural_traits::predicates_for_object_candidate(
+                        ecx,
+                        goal.param_env,
+                        goal.predicate.projection_ty.trait_ref(tcx),
+                        bounds,
+                    )
+                    .into_iter()
+                    .map(|pred| goal.with(tcx, pred)),
+                );
+
+                let subst_certainty = ecx.evaluate_all(nested_goals)?;
+
+                ecx.eq_term_and_make_canonical_response(
+                    goal,
+                    subst_certainty,
+                    assumption_projection_pred.term,
+                )
+            })
+        } else {
+            Err(NoSolution)
+        }
+    }
+
     fn consider_impl_candidate(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, ProjectionPredicate<'tcx>>,
@@ -204,11 +190,11 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
             return Err(NoSolution);
         }
 
-        ecx.infcx.probe(|_| {
-            let impl_substs = ecx.infcx.fresh_substs_for_item(DUMMY_SP, impl_def_id);
+        ecx.probe(|ecx| {
+            let impl_substs = ecx.fresh_substs_for_item(impl_def_id);
             let impl_trait_ref = impl_trait_ref.subst(tcx, impl_substs);
 
-            let mut nested_goals = ecx.infcx.eq(goal.param_env, goal_trait_ref, impl_trait_ref)?;
+            let mut nested_goals = ecx.eq(goal.param_env, goal_trait_ref, impl_trait_ref)?;
             let where_clause_bounds = tcx
                 .predicates_of(impl_def_id)
                 .instantiate(tcx, impl_substs)
@@ -217,7 +203,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
                 .map(|pred| goal.with(tcx, pred));
 
             nested_goals.extend(where_clause_bounds);
-            let trait_ref_certainty = ecx.evaluate_all(nested_goals)?;
+            let match_impl_certainty = ecx.evaluate_all(nested_goals)?;
 
             // In case the associated item is hidden due to specialization, we have to
             // return ambiguity this would otherwise be incomplete, resulting in
@@ -229,8 +215,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
                 goal.predicate.def_id(),
                 impl_def_id
             )? else {
-                let certainty = Certainty::Maybe(MaybeCause::Ambiguity);
-                return ecx.make_canonical_response(trait_ref_certainty.unify_and(certainty));
+                return ecx.make_canonical_response(match_impl_certainty.unify_and(Certainty::AMBIGUOUS));
             };
 
             if !assoc_def.item.defaultness(tcx).has_value() {
@@ -265,7 +250,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
 
             // Finally we construct the actual value of the associated type.
             let is_const = matches!(tcx.def_kind(assoc_def.item.def_id), DefKind::AssocConst);
-            let ty = tcx.bound_type_of(assoc_def.item.def_id);
+            let ty = tcx.type_of(assoc_def.item.def_id);
             let term: ty::EarlyBinder<ty::Term<'tcx>> = if is_const {
                 let identity_substs =
                     ty::InternalSubsts::identity_for_item(tcx, assoc_def.item.def_id);
@@ -277,54 +262,10 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
                 ty.map_bound(|ty| ty.into())
             };
 
-            // The term of our goal should be fully unconstrained, so this should never fail.
-            //
-            // It can however be ambiguous when the resolved type is a projection.
-            let nested_goals = ecx
-                .infcx
-                .eq(goal.param_env, goal.predicate.term, term.subst(tcx, substs))
-                .expect("failed to unify with unconstrained term");
-            let rhs_certainty =
-                ecx.evaluate_all(nested_goals).expect("failed to unify with unconstrained term");
-
-            ecx.make_canonical_response(trait_ref_certainty.unify_and(rhs_certainty))
+            ecx.eq_term_and_make_canonical_response(goal, match_impl_certainty, term.subst(tcx, substs))
         })
     }
 
-    fn consider_assumption(
-        ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, Self>,
-        assumption: ty::Predicate<'tcx>,
-    ) -> QueryResult<'tcx> {
-        if let Some(poly_projection_pred) = assumption.to_opt_poly_projection_pred() {
-            ecx.infcx.probe(|_| {
-                let assumption_projection_pred =
-                    ecx.infcx.instantiate_bound_vars_with_infer(poly_projection_pred);
-                let nested_goals = ecx.infcx.eq(
-                    goal.param_env,
-                    goal.predicate.projection_ty,
-                    assumption_projection_pred.projection_ty,
-                )?;
-                let subst_certainty = ecx.evaluate_all(nested_goals)?;
-
-                // The term of our goal should be fully unconstrained, so this should never fail.
-                //
-                // It can however be ambiguous when the resolved type is a projection.
-                let nested_goals = ecx
-                    .infcx
-                    .eq(goal.param_env, goal.predicate.term, assumption_projection_pred.term)
-                    .expect("failed to unify with unconstrained term");
-                let rhs_certainty = ecx
-                    .evaluate_all(nested_goals)
-                    .expect("failed to unify with unconstrained term");
-
-                ecx.make_canonical_response(subst_certainty.unify_and(rhs_certainty))
-            })
-        } else {
-            Err(NoSolution)
-        }
-    }
-
     fn consider_auto_trait_candidate(
         _ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
@@ -353,11 +294,11 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
         bug!("`Copy`/`Clone` does not have an associated type: {:?}", goal);
     }
 
-    fn consider_builtin_pointer_sized_candidate(
+    fn consider_builtin_pointer_like_candidate(
         _ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx> {
-        bug!("`PointerSized` does not have an associated type: {:?}", goal);
+        bug!("`PointerLike` does not have an associated type: {:?}", goal);
     }
 
     fn consider_builtin_fn_trait_candidates(
@@ -365,25 +306,28 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
         goal: Goal<'tcx, Self>,
         goal_kind: ty::ClosureKind,
     ) -> QueryResult<'tcx> {
-        if let Some(tupled_inputs_and_output) =
-            structural_traits::extract_tupled_inputs_and_output_from_callable(
-                ecx.tcx(),
-                goal.predicate.self_ty(),
-                goal_kind,
-            )?
-        {
-            let pred = tupled_inputs_and_output
-                .map_bound(|(inputs, output)| ty::ProjectionPredicate {
-                    projection_ty: ecx
-                        .tcx()
-                        .mk_alias_ty(goal.predicate.def_id(), [goal.predicate.self_ty(), inputs]),
-                    term: output.into(),
-                })
-                .to_predicate(ecx.tcx());
-            Self::consider_assumption(ecx, goal, pred)
-        } else {
-            ecx.make_canonical_response(Certainty::Maybe(MaybeCause::Ambiguity))
-        }
+        let tcx = ecx.tcx();
+        let Some(tupled_inputs_and_output) =
+        structural_traits::extract_tupled_inputs_and_output_from_callable(
+            tcx,
+            goal.predicate.self_ty(),
+            goal_kind,
+        )? else {
+        return ecx.make_canonical_response(Certainty::AMBIGUOUS);
+    };
+        let output_is_sized_pred = tupled_inputs_and_output
+            .map_bound(|(_, output)| tcx.at(DUMMY_SP).mk_trait_ref(LangItem::Sized, [output]));
+
+        let pred = tupled_inputs_and_output
+            .map_bound(|(inputs, output)| ty::ProjectionPredicate {
+                projection_ty: tcx
+                    .mk_alias_ty(goal.predicate.def_id(), [goal.predicate.self_ty(), inputs]),
+                term: output.into(),
+            })
+            .to_predicate(tcx);
+        // A built-in `Fn` impl only holds if the output is sized.
+        // (FIXME: technically we only need to check this if the type is a fn ptr...)
+        Self::consider_implied_clause(ecx, goal, pred, [goal.with(tcx, output_is_sized_pred)])
     }
 
     fn consider_builtin_tuple_candidate(
@@ -392,6 +336,193 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
     ) -> QueryResult<'tcx> {
         bug!("`Tuple` does not have an associated type: {:?}", goal);
     }
+
+    fn consider_builtin_pointee_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        let tcx = ecx.tcx();
+        ecx.probe(|ecx| {
+            let metadata_ty = match goal.predicate.self_ty().kind() {
+                ty::Bool
+                | ty::Char
+                | ty::Int(..)
+                | ty::Uint(..)
+                | ty::Float(..)
+                | ty::Array(..)
+                | ty::RawPtr(..)
+                | ty::Ref(..)
+                | ty::FnDef(..)
+                | ty::FnPtr(..)
+                | ty::Closure(..)
+                | ty::Infer(ty::IntVar(..) | ty::FloatVar(..))
+                | ty::Generator(..)
+                | ty::GeneratorWitness(..)
+                | ty::GeneratorWitnessMIR(..)
+                | ty::Never
+                | ty::Foreign(..) => tcx.types.unit,
+
+                ty::Error(e) => tcx.ty_error(*e),
+
+                ty::Str | ty::Slice(_) => tcx.types.usize,
+
+                ty::Dynamic(_, _, _) => {
+                    let dyn_metadata = tcx.require_lang_item(LangItem::DynMetadata, None);
+                    tcx.type_of(dyn_metadata)
+                        .subst(tcx, &[ty::GenericArg::from(goal.predicate.self_ty())])
+                }
+
+                ty::Alias(_, _) | ty::Param(_) | ty::Placeholder(..) => {
+                    // FIXME(ptr_metadata): It would also be possible to return a `Ok(Ambig)` with no constraints.
+                    let sized_predicate = ty::Binder::dummy(tcx.at(DUMMY_SP).mk_trait_ref(
+                        LangItem::Sized,
+                        [ty::GenericArg::from(goal.predicate.self_ty())],
+                    ));
+
+                    let (_, is_sized_certainty) =
+                        ecx.evaluate_goal(goal.with(tcx, sized_predicate))?;
+                    return ecx.eq_term_and_make_canonical_response(
+                        goal,
+                        is_sized_certainty,
+                        tcx.types.unit,
+                    );
+                }
+
+                ty::Adt(def, substs) if def.is_struct() => {
+                    match def.non_enum_variant().fields.last() {
+                        None => tcx.types.unit,
+                        Some(field_def) => {
+                            let self_ty = field_def.ty(tcx, substs);
+                            let new_goal = goal.with(
+                                tcx,
+                                ty::Binder::dummy(goal.predicate.with_self_ty(tcx, self_ty)),
+                            );
+                            let (_, certainty) = ecx.evaluate_goal(new_goal)?;
+                            return ecx.make_canonical_response(certainty);
+                        }
+                    }
+                }
+                ty::Adt(_, _) => tcx.types.unit,
+
+                ty::Tuple(elements) => match elements.last() {
+                    None => tcx.types.unit,
+                    Some(&self_ty) => {
+                        let new_goal = goal.with(
+                            tcx,
+                            ty::Binder::dummy(goal.predicate.with_self_ty(tcx, self_ty)),
+                        );
+                        let (_, certainty) = ecx.evaluate_goal(new_goal)?;
+                        return ecx.make_canonical_response(certainty);
+                    }
+                },
+
+                ty::Infer(
+                    ty::TyVar(_) | ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_),
+                )
+                | ty::Bound(..) => bug!(
+                    "unexpected self ty `{:?}` when normalizing `<T as Pointee>::Metadata`",
+                    goal.predicate.self_ty()
+                ),
+            };
+
+            ecx.eq_term_and_make_canonical_response(goal, Certainty::Yes, metadata_ty)
+        })
+    }
+
+    fn consider_builtin_future_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        let self_ty = goal.predicate.self_ty();
+        let ty::Generator(def_id, substs, _) = *self_ty.kind() else {
+            return Err(NoSolution);
+        };
+
+        // Generators are not futures unless they come from `async` desugaring
+        let tcx = ecx.tcx();
+        if !tcx.generator_is_async(def_id) {
+            return Err(NoSolution);
+        }
+
+        let term = substs.as_generator().return_ty().into();
+
+        Self::consider_implied_clause(
+            ecx,
+            goal,
+            ty::Binder::dummy(ty::ProjectionPredicate {
+                projection_ty: ecx.tcx().mk_alias_ty(goal.predicate.def_id(), [self_ty]),
+                term,
+            })
+            .to_predicate(tcx),
+            // Technically, we need to check that the future type is Sized,
+            // but that's already proven by the generator being WF.
+            [],
+        )
+    }
+
+    fn consider_builtin_generator_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        let self_ty = goal.predicate.self_ty();
+        let ty::Generator(def_id, substs, _) = *self_ty.kind() else {
+            return Err(NoSolution);
+        };
+
+        // `async`-desugared generators do not implement the generator trait
+        let tcx = ecx.tcx();
+        if tcx.generator_is_async(def_id) {
+            return Err(NoSolution);
+        }
+
+        let generator = substs.as_generator();
+
+        let name = tcx.associated_item(goal.predicate.def_id()).name;
+        let term = if name == sym::Return {
+            generator.return_ty().into()
+        } else if name == sym::Yield {
+            generator.yield_ty().into()
+        } else {
+            bug!("unexpected associated item `<{self_ty} as Generator>::{name}`")
+        };
+
+        Self::consider_implied_clause(
+            ecx,
+            goal,
+            ty::Binder::dummy(ty::ProjectionPredicate {
+                projection_ty: ecx
+                    .tcx()
+                    .mk_alias_ty(goal.predicate.def_id(), [self_ty, generator.resume_ty()]),
+                term,
+            })
+            .to_predicate(tcx),
+            // Technically, we need to check that the future type is Sized,
+            // but that's already proven by the generator being WF.
+            [],
+        )
+    }
+
+    fn consider_builtin_unsize_candidate(
+        _ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        bug!("`Unsize` does not have an associated type: {:?}", goal);
+    }
+
+    fn consider_builtin_dyn_upcast_candidates(
+        _ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> Vec<super::CanonicalResponse<'tcx>> {
+        bug!("`Unsize` does not have an associated type: {:?}", goal);
+    }
+
+    fn consider_builtin_discriminant_kind_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        let discriminant = goal.predicate.self_ty().discriminant_ty(ecx.tcx());
+        ecx.probe(|ecx| ecx.eq_term_and_make_canonical_response(goal, Certainty::Yes, discriminant))
+    }
 }
 
 /// This behavior is also implemented in `rustc_ty_utils` and in the old `project` code.
diff --git a/compiler/rustc_trait_selection/src/solve/search_graph/cache.rs b/compiler/rustc_trait_selection/src/solve/search_graph/cache.rs
index 730a8e612..86b13c05f 100644
--- a/compiler/rustc_trait_selection/src/solve/search_graph/cache.rs
+++ b/compiler/rustc_trait_selection/src/solve/search_graph/cache.rs
@@ -95,7 +95,7 @@ impl<'tcx> ProvisionalCache<'tcx> {
     }
 
     pub(super) fn provisional_result(&self, entry_index: EntryIndex) -> QueryResult<'tcx> {
-        self.entries[entry_index].response.clone()
+        self.entries[entry_index].response
     }
 }
 
diff --git a/compiler/rustc_trait_selection/src/solve/search_graph/mod.rs b/compiler/rustc_trait_selection/src/solve/search_graph/mod.rs
index 0030e9aa3..c7eb8de65 100644
--- a/compiler/rustc_trait_selection/src/solve/search_graph/mod.rs
+++ b/compiler/rustc_trait_selection/src/solve/search_graph/mod.rs
@@ -3,11 +3,12 @@ mod overflow;
 
 use self::cache::ProvisionalEntry;
 use super::{CanonicalGoal, Certainty, MaybeCause, QueryResult};
+pub(super) use crate::solve::search_graph::overflow::OverflowHandler;
 use cache::ProvisionalCache;
 use overflow::OverflowData;
 use rustc_index::vec::IndexVec;
 use rustc_middle::ty::TyCtxt;
-use std::collections::hash_map::Entry;
+use std::{collections::hash_map::Entry, mem};
 
 rustc_index::newtype_index! {
     pub struct StackDepth {}
@@ -42,10 +43,29 @@ impl<'tcx> SearchGraph<'tcx> {
             && !self.overflow_data.did_overflow()
     }
 
+    /// Whether we're currently in a cycle. This should only be used
+    /// for debug assertions.
+    pub(super) fn in_cycle(&self) -> bool {
+        if let Some(stack_depth) = self.stack.last() {
+            // Either the current goal on the stack is the root of a cycle...
+            if self.stack[stack_depth].has_been_used {
+                return true;
+            }
+
+            // ...or it depends on a goal with a lower depth.
+            let current_goal = self.stack[stack_depth].goal;
+            let entry_index = self.provisional_cache.lookup_table[&current_goal];
+            self.provisional_cache.entries[entry_index].depth != stack_depth
+        } else {
+            false
+        }
+    }
+
     /// Tries putting the new goal on the stack, returning an error if it is already cached.
     ///
     /// This correctly updates the provisional cache if there is a cycle.
-    pub(super) fn try_push_stack(
+    #[instrument(level = "debug", skip(self, tcx), ret)]
+    fn try_push_stack(
         &mut self,
         tcx: TyCtxt<'tcx>,
         goal: CanonicalGoal<'tcx>,
@@ -79,8 +99,10 @@ impl<'tcx> SearchGraph<'tcx> {
             Entry::Occupied(entry_index) => {
                 let entry_index = *entry_index.get();
 
-                cache.add_dependency_of_leaf_on(entry_index);
                 let stack_depth = cache.depth(entry_index);
+                debug!("encountered cycle with depth {stack_depth:?}");
+
+                cache.add_dependency_of_leaf_on(entry_index);
 
                 self.stack[stack_depth].has_been_used = true;
                 // NOTE: The goals on the stack aren't the only goals involved in this cycle.
@@ -117,25 +139,29 @@ impl<'tcx> SearchGraph<'tcx> {
     /// updated the provisional cache and we have to recompute the current goal.
     ///
     /// FIXME: Refer to the rustc-dev-guide entry once it exists.
-    pub(super) fn try_finalize_goal(
+    #[instrument(level = "debug", skip(self, tcx, actual_goal), ret)]
+    fn try_finalize_goal(
         &mut self,
         tcx: TyCtxt<'tcx>,
         actual_goal: CanonicalGoal<'tcx>,
         response: QueryResult<'tcx>,
     ) -> bool {
-        let StackElem { goal, has_been_used } = self.stack.pop().unwrap();
+        let stack_elem = self.stack.pop().unwrap();
+        let StackElem { goal, has_been_used } = stack_elem;
         assert_eq!(goal, actual_goal);
 
         let cache = &mut self.provisional_cache;
         let provisional_entry_index = *cache.lookup_table.get(&goal).unwrap();
         let provisional_entry = &mut cache.entries[provisional_entry_index];
-        let depth = provisional_entry.depth;
+        // We eagerly update the response in the cache here. If we have to reevaluate
+        // this goal we use the new response when hitting a cycle, and we definitely
+        // want to access the final response whenever we look at the cache.
+        let prev_response = mem::replace(&mut provisional_entry.response, response);
+
         // Was the current goal the root of a cycle and was the provisional response
         // different from the final one.
-        if has_been_used && provisional_entry.response != response {
-            // If so, update the provisional reponse for this goal...
-            provisional_entry.response = response;
-            // ...remove all entries whose result depends on this goal
+        if has_been_used && prev_response != response {
+            // If so, remove all entries whose result depends on this goal
             // from the provisional cache...
             //
             // That's not completely correct, as a nested goal can also
@@ -150,29 +176,72 @@ impl<'tcx> SearchGraph<'tcx> {
             self.stack.push(StackElem { goal, has_been_used: false });
             false
         } else {
-            // If not, we're done with this goal.
-            //
-            // Check whether that this goal doesn't depend on a goal deeper on the stack
-            // and if so, move it and all nested goals to the global cache.
-            //
-            // Note that if any nested goal were to depend on something deeper on the stack,
-            // this would have also updated the depth of the current goal.
-            if depth == self.stack.next_index() {
-                for (i, entry) in cache.entries.drain_enumerated(provisional_entry_index.index()..)
-                {
-                    let actual_index = cache.lookup_table.remove(&entry.goal);
-                    debug_assert_eq!(Some(i), actual_index);
-                    debug_assert!(entry.depth == depth);
-                    cache::try_move_finished_goal_to_global_cache(
-                        tcx,
-                        &mut self.overflow_data,
-                        &self.stack,
-                        entry.goal,
-                        entry.response,
-                    );
-                }
-            }
+            self.try_move_finished_goal_to_global_cache(tcx, stack_elem);
             true
         }
     }
+
+    fn try_move_finished_goal_to_global_cache(
+        &mut self,
+        tcx: TyCtxt<'tcx>,
+        stack_elem: StackElem<'tcx>,
+    ) {
+        let StackElem { goal, .. } = stack_elem;
+        let cache = &mut self.provisional_cache;
+        let provisional_entry_index = *cache.lookup_table.get(&goal).unwrap();
+        let provisional_entry = &mut cache.entries[provisional_entry_index];
+        let depth = provisional_entry.depth;
+
+        // If not, we're done with this goal.
+        //
+        // Check whether that this goal doesn't depend on a goal deeper on the stack
+        // and if so, move it and all nested goals to the global cache.
+        //
+        // Note that if any nested goal were to depend on something deeper on the stack,
+        // this would have also updated the depth of the current goal.
+        if depth == self.stack.next_index() {
+            for (i, entry) in cache.entries.drain_enumerated(provisional_entry_index.index()..) {
+                let actual_index = cache.lookup_table.remove(&entry.goal);
+                debug_assert_eq!(Some(i), actual_index);
+                debug_assert!(entry.depth == depth);
+                cache::try_move_finished_goal_to_global_cache(
+                    tcx,
+                    &mut self.overflow_data,
+                    &self.stack,
+                    entry.goal,
+                    entry.response,
+                );
+            }
+        }
+    }
+
+    pub(super) fn with_new_goal(
+        &mut self,
+        tcx: TyCtxt<'tcx>,
+        canonical_goal: CanonicalGoal<'tcx>,
+        mut loop_body: impl FnMut(&mut Self) -> QueryResult<'tcx>,
+    ) -> QueryResult<'tcx> {
+        match self.try_push_stack(tcx, canonical_goal) {
+            Ok(()) => {}
+            // Our goal is already on the stack, eager return.
+            Err(response) => return response,
+        }
+
+        self.repeat_while_none(
+            |this| {
+                let result = this.deal_with_overflow(tcx, canonical_goal);
+                let stack_elem = this.stack.pop().unwrap();
+                this.try_move_finished_goal_to_global_cache(tcx, stack_elem);
+                result
+            },
+            |this| {
+                let result = loop_body(this);
+                if this.try_finalize_goal(tcx, canonical_goal, result) {
+                    Some(result)
+                } else {
+                    None
+                }
+            },
+        )
+    }
 }
diff --git a/compiler/rustc_trait_selection/src/solve/search_graph/overflow.rs b/compiler/rustc_trait_selection/src/solve/search_graph/overflow.rs
index 1dd3894c9..56409b060 100644
--- a/compiler/rustc_trait_selection/src/solve/search_graph/overflow.rs
+++ b/compiler/rustc_trait_selection/src/solve/search_graph/overflow.rs
@@ -50,6 +50,42 @@ impl OverflowData {
     }
 }
 
+pub(in crate::solve) trait OverflowHandler<'tcx> {
+    fn search_graph(&mut self) -> &mut SearchGraph<'tcx>;
+
+    fn repeat_while_none<T>(
+        &mut self,
+        on_overflow: impl FnOnce(&mut Self) -> Result<T, NoSolution>,
+        mut loop_body: impl FnMut(&mut Self) -> Option<Result<T, NoSolution>>,
+    ) -> Result<T, NoSolution> {
+        let start_depth = self.search_graph().overflow_data.additional_depth;
+        let depth = self.search_graph().stack.len();
+        while !self.search_graph().overflow_data.has_overflow(depth) {
+            if let Some(result) = loop_body(self) {
+                self.search_graph().overflow_data.additional_depth = start_depth;
+                return result;
+            }
+
+            self.search_graph().overflow_data.additional_depth += 1;
+        }
+        self.search_graph().overflow_data.additional_depth = start_depth;
+        self.search_graph().overflow_data.deal_with_overflow();
+        on_overflow(self)
+    }
+}
+
+impl<'tcx> OverflowHandler<'tcx> for EvalCtxt<'_, 'tcx> {
+    fn search_graph(&mut self) -> &mut SearchGraph<'tcx> {
+        &mut self.search_graph
+    }
+}
+
+impl<'tcx> OverflowHandler<'tcx> for SearchGraph<'tcx> {
+    fn search_graph(&mut self) -> &mut SearchGraph<'tcx> {
+        self
+    }
+}
+
 impl<'tcx> SearchGraph<'tcx> {
     pub fn deal_with_overflow(
         &mut self,
@@ -60,25 +96,3 @@ impl<'tcx> SearchGraph<'tcx> {
         response_no_constraints(tcx, goal, Certainty::Maybe(MaybeCause::Overflow))
     }
 }
-
-impl<'tcx> EvalCtxt<'_, 'tcx> {
-    /// A `while`-loop which tracks overflow.
-    pub fn repeat_while_none(
-        &mut self,
-        mut loop_body: impl FnMut(&mut Self) -> Option<Result<Certainty, NoSolution>>,
-    ) -> Result<Certainty, NoSolution> {
-        let start_depth = self.search_graph.overflow_data.additional_depth;
-        let depth = self.search_graph.stack.len();
-        while !self.search_graph.overflow_data.has_overflow(depth) {
-            if let Some(result) = loop_body(self) {
-                self.search_graph.overflow_data.additional_depth = start_depth;
-                return result;
-            }
-
-            self.search_graph.overflow_data.additional_depth += 1;
-        }
-        self.search_graph.overflow_data.additional_depth = start_depth;
-        self.search_graph.overflow_data.deal_with_overflow();
-        Ok(Certainty::Maybe(MaybeCause::Overflow))
-    }
-}
diff --git a/compiler/rustc_trait_selection/src/solve/trait_goals.rs b/compiler/rustc_trait_selection/src/solve/trait_goals.rs
index 9985d7181..5c499c36e 100644
--- a/compiler/rustc_trait_selection/src/solve/trait_goals.rs
+++ b/compiler/rustc_trait_selection/src/solve/trait_goals.rs
@@ -2,15 +2,15 @@
 
 use std::iter;
 
-use super::assembly::{self, Candidate, CandidateSource};
-use super::infcx_ext::InferCtxtExt;
-use super::{Certainty, EvalCtxt, Goal, MaybeCause, QueryResult};
+use super::assembly;
+use super::{CanonicalResponse, Certainty, EvalCtxt, Goal, QueryResult};
 use rustc_hir::def_id::DefId;
-use rustc_infer::infer::InferCtxt;
+use rustc_hir::LangItem;
 use rustc_infer::traits::query::NoSolution;
+use rustc_infer::traits::util::supertraits;
 use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams};
 use rustc_middle::ty::{self, ToPredicate, Ty, TyCtxt};
-use rustc_middle::ty::{TraitPredicate, TypeVisitable};
+use rustc_middle::ty::{TraitPredicate, TypeVisitableExt};
 use rustc_span::DUMMY_SP;
 
 pub mod structural_traits;
@@ -43,12 +43,12 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
             return Err(NoSolution);
         }
 
-        ecx.infcx.probe(|_| {
-            let impl_substs = ecx.infcx.fresh_substs_for_item(DUMMY_SP, impl_def_id);
+        ecx.probe(|ecx| {
+            let impl_substs = ecx.fresh_substs_for_item(impl_def_id);
             let impl_trait_ref = impl_trait_ref.subst(tcx, impl_substs);
 
             let mut nested_goals =
-                ecx.infcx.eq(goal.param_env, goal.predicate.trait_ref, impl_trait_ref)?;
+                ecx.eq(goal.param_env, goal.predicate.trait_ref, impl_trait_ref)?;
             let where_clause_bounds = tcx
                 .predicates_of(impl_def_id)
                 .instantiate(tcx, impl_substs)
@@ -60,21 +60,65 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
         })
     }
 
-    fn consider_assumption(
+    fn consider_implied_clause(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
         assumption: ty::Predicate<'tcx>,
+        requirements: impl IntoIterator<Item = Goal<'tcx, ty::Predicate<'tcx>>>,
     ) -> QueryResult<'tcx> {
-        if let Some(poly_trait_pred) = assumption.to_opt_poly_trait_pred() {
+        if let Some(poly_trait_pred) = assumption.to_opt_poly_trait_pred()
+            && poly_trait_pred.def_id() == goal.predicate.def_id()
+        {
             // FIXME: Constness and polarity
-            ecx.infcx.probe(|_| {
+            ecx.probe(|ecx| {
                 let assumption_trait_pred =
-                    ecx.infcx.instantiate_bound_vars_with_infer(poly_trait_pred);
-                let nested_goals = ecx.infcx.eq(
+                    ecx.instantiate_binder_with_infer(poly_trait_pred);
+                let mut nested_goals = ecx.eq(
                     goal.param_env,
                     goal.predicate.trait_ref,
                     assumption_trait_pred.trait_ref,
                 )?;
+                nested_goals.extend(requirements);
+                ecx.evaluate_all_and_make_canonical_response(nested_goals)
+            })
+        } else {
+            Err(NoSolution)
+        }
+    }
+
+    fn consider_object_bound_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+        assumption: ty::Predicate<'tcx>,
+    ) -> QueryResult<'tcx> {
+        if let Some(poly_trait_pred) = assumption.to_opt_poly_trait_pred()
+            && poly_trait_pred.def_id() == goal.predicate.def_id()
+        {
+            // FIXME: Constness and polarity
+            ecx.probe(|ecx| {
+                let assumption_trait_pred =
+                    ecx.instantiate_binder_with_infer(poly_trait_pred);
+                let mut nested_goals = ecx.eq(
+                    goal.param_env,
+                    goal.predicate.trait_ref,
+                    assumption_trait_pred.trait_ref,
+                )?;
+
+                let tcx = ecx.tcx();
+                let ty::Dynamic(bounds, _, _) = *goal.predicate.self_ty().kind() else {
+                    bug!("expected object type in `consider_object_bound_candidate`");
+                };
+                nested_goals.extend(
+                    structural_traits::predicates_for_object_candidate(
+                        ecx,
+                        goal.param_env,
+                        goal.predicate.trait_ref,
+                        bounds,
+                    )
+                    .into_iter()
+                    .map(|pred| goal.with(tcx, pred)),
+                );
+
                 ecx.evaluate_all_and_make_canonical_response(nested_goals)
             })
         } else {
@@ -86,6 +130,20 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx> {
+        // This differs from the current stable behavior and
+        // fixes #84857. Due to breakage found via crater, we
+        // currently instead lint patterns which can be used to
+        // exploit this unsoundness on stable, see #93367 for
+        // more details.
+        if let Some(def_id) = ecx.tcx().find_map_relevant_impl(
+            goal.predicate.def_id(),
+            goal.predicate.self_ty(),
+            Some,
+        ) {
+            debug!(?def_id, ?goal, "disqualified auto-trait implementation");
+            return Err(NoSolution);
+        }
+
         ecx.probe_and_evaluate_goal_for_constituent_tys(
             goal,
             structural_traits::instantiate_constituent_tys_for_auto_trait,
@@ -98,7 +156,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
     ) -> QueryResult<'tcx> {
         let tcx = ecx.tcx();
 
-        ecx.infcx.probe(|_| {
+        ecx.probe(|ecx| {
             let nested_obligations = tcx
                 .predicates_of(goal.predicate.def_id())
                 .instantiate(tcx, goal.predicate.trait_ref.substs);
@@ -128,12 +186,12 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
         )
     }
 
-    fn consider_builtin_pointer_sized_candidate(
+    fn consider_builtin_pointer_like_candidate(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx> {
         if goal.predicate.self_ty().has_non_region_infer() {
-            return ecx.make_canonical_response(Certainty::Maybe(MaybeCause::Ambiguity));
+            return ecx.make_canonical_response(Certainty::AMBIGUOUS);
         }
 
         let tcx = ecx.tcx();
@@ -156,23 +214,26 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
         goal: Goal<'tcx, Self>,
         goal_kind: ty::ClosureKind,
     ) -> QueryResult<'tcx> {
-        if let Some(tupled_inputs_and_output) =
+        let tcx = ecx.tcx();
+        let Some(tupled_inputs_and_output) =
             structural_traits::extract_tupled_inputs_and_output_from_callable(
-                ecx.tcx(),
+                tcx,
                 goal.predicate.self_ty(),
                 goal_kind,
-            )?
-        {
-            let pred = tupled_inputs_and_output
-                .map_bound(|(inputs, _)| {
-                    ecx.tcx()
-                        .mk_trait_ref(goal.predicate.def_id(), [goal.predicate.self_ty(), inputs])
-                })
-                .to_predicate(ecx.tcx());
-            Self::consider_assumption(ecx, goal, pred)
-        } else {
-            ecx.make_canonical_response(Certainty::Maybe(MaybeCause::Ambiguity))
-        }
+            )? else {
+            return ecx.make_canonical_response(Certainty::AMBIGUOUS);
+        };
+        let output_is_sized_pred = tupled_inputs_and_output
+            .map_bound(|(_, output)| tcx.at(DUMMY_SP).mk_trait_ref(LangItem::Sized, [output]));
+
+        let pred = tupled_inputs_and_output
+            .map_bound(|(inputs, _)| {
+                tcx.mk_trait_ref(goal.predicate.def_id(), [goal.predicate.self_ty(), inputs])
+            })
+            .to_predicate(tcx);
+        // A built-in `Fn` impl only holds if the output is sized.
+        // (FIXME: technically we only need to check this if the type is a fn ptr...)
+        Self::consider_implied_clause(ecx, goal, pred, [goal.with(tcx, output_is_sized_pred)])
     }
 
     fn consider_builtin_tuple_candidate(
@@ -185,6 +246,272 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
             Err(NoSolution)
         }
     }
+
+    fn consider_builtin_pointee_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        _goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        ecx.make_canonical_response(Certainty::Yes)
+    }
+
+    fn consider_builtin_future_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        let ty::Generator(def_id, _, _) = *goal.predicate.self_ty().kind() else {
+            return Err(NoSolution);
+        };
+
+        // Generators are not futures unless they come from `async` desugaring
+        let tcx = ecx.tcx();
+        if !tcx.generator_is_async(def_id) {
+            return Err(NoSolution);
+        }
+
+        // Async generator unconditionally implement `Future`
+        // Technically, we need to check that the future output type is Sized,
+        // but that's already proven by the generator being WF.
+        ecx.make_canonical_response(Certainty::Yes)
+    }
+
+    fn consider_builtin_generator_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        let self_ty = goal.predicate.self_ty();
+        let ty::Generator(def_id, substs, _) = *self_ty.kind() else {
+            return Err(NoSolution);
+        };
+
+        // `async`-desugared generators do not implement the generator trait
+        let tcx = ecx.tcx();
+        if tcx.generator_is_async(def_id) {
+            return Err(NoSolution);
+        }
+
+        let generator = substs.as_generator();
+        Self::consider_implied_clause(
+            ecx,
+            goal,
+            ty::Binder::dummy(
+                tcx.mk_trait_ref(goal.predicate.def_id(), [self_ty, generator.resume_ty()]),
+            )
+            .to_predicate(tcx),
+            // Technically, we need to check that the generator types are Sized,
+            // but that's already proven by the generator being WF.
+            [],
+        )
+    }
+
+    fn consider_builtin_unsize_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        let tcx = ecx.tcx();
+        let a_ty = goal.predicate.self_ty();
+        let b_ty = goal.predicate.trait_ref.substs.type_at(1);
+        if b_ty.is_ty_var() {
+            return ecx.make_canonical_response(Certainty::AMBIGUOUS);
+        }
+        ecx.probe(|ecx| {
+            match (a_ty.kind(), b_ty.kind()) {
+                // Trait upcasting, or `dyn Trait + Auto + 'a` -> `dyn Trait + 'b`
+                (&ty::Dynamic(_, _, ty::Dyn), &ty::Dynamic(_, _, ty::Dyn)) => {
+                    // Dyn upcasting is handled separately, since due to upcasting,
+                    // when there are two supertraits that differ by substs, we
+                    // may return more than one query response.
+                    return Err(NoSolution);
+                }
+                // `T` -> `dyn Trait` unsizing
+                (_, &ty::Dynamic(data, region, ty::Dyn)) => {
+                    // Can only unsize to an object-safe type
+                    if data
+                        .principal_def_id()
+                        .map_or(false, |def_id| !tcx.check_is_object_safe(def_id))
+                    {
+                        return Err(NoSolution);
+                    }
+
+                    let Some(sized_def_id) = tcx.lang_items().sized_trait() else {
+                        return Err(NoSolution);
+                    };
+                    let nested_goals: Vec<_> = data
+                        .iter()
+                        // Check that the type implements all of the predicates of the def-id.
+                        // (i.e. the principal, all of the associated types match, and any auto traits)
+                        .map(|pred| goal.with(tcx, pred.with_self_ty(tcx, a_ty)))
+                        .chain([
+                            // The type must be Sized to be unsized.
+                            goal.with(
+                                tcx,
+                                ty::Binder::dummy(tcx.mk_trait_ref(sized_def_id, [a_ty])),
+                            ),
+                            // The type must outlive the lifetime of the `dyn` we're unsizing into.
+                            goal.with(tcx, ty::Binder::dummy(ty::OutlivesPredicate(a_ty, region))),
+                        ])
+                        .collect();
+
+                    ecx.evaluate_all_and_make_canonical_response(nested_goals)
+                }
+                // `[T; n]` -> `[T]` unsizing
+                (&ty::Array(a_elem_ty, ..), &ty::Slice(b_elem_ty)) => {
+                    // We just require that the element type stays the same
+                    let nested_goals = ecx.eq(goal.param_env, a_elem_ty, b_elem_ty)?;
+                    ecx.evaluate_all_and_make_canonical_response(nested_goals)
+                }
+                // Struct unsizing `Struct<T>` -> `Struct<U>` where `T: Unsize<U>`
+                (&ty::Adt(a_def, a_substs), &ty::Adt(b_def, b_substs))
+                    if a_def.is_struct() && a_def.did() == b_def.did() =>
+                {
+                    let unsizing_params = tcx.unsizing_params_for_adt(a_def.did());
+                    // We must be unsizing some type parameters. This also implies
+                    // that the struct has a tail field.
+                    if unsizing_params.is_empty() {
+                        return Err(NoSolution);
+                    }
+
+                    let tail_field = a_def
+                        .non_enum_variant()
+                        .fields
+                        .last()
+                        .expect("expected unsized ADT to have a tail field");
+                    let tail_field_ty = tcx.type_of(tail_field.did);
+
+                    let a_tail_ty = tail_field_ty.subst(tcx, a_substs);
+                    let b_tail_ty = tail_field_ty.subst(tcx, b_substs);
+
+                    // Substitute just the unsizing params from B into A. The type after
+                    // this substitution must be equal to B. This is so we don't unsize
+                    // unrelated type parameters.
+                    let new_a_substs =
+                        tcx.mk_substs_from_iter(a_substs.iter().enumerate().map(|(i, a)| {
+                            if unsizing_params.contains(i as u32) { b_substs[i] } else { a }
+                        }));
+                    let unsized_a_ty = tcx.mk_adt(a_def, new_a_substs);
+
+                    // Finally, we require that `TailA: Unsize<TailB>` for the tail field
+                    // types.
+                    let mut nested_goals = ecx.eq(goal.param_env, unsized_a_ty, b_ty)?;
+                    nested_goals.push(goal.with(
+                        tcx,
+                        ty::Binder::dummy(
+                            tcx.mk_trait_ref(goal.predicate.def_id(), [a_tail_ty, b_tail_ty]),
+                        ),
+                    ));
+
+                    ecx.evaluate_all_and_make_canonical_response(nested_goals)
+                }
+                // Tuple unsizing `(.., T)` -> `(.., U)` where `T: Unsize<U>`
+                (&ty::Tuple(a_tys), &ty::Tuple(b_tys))
+                    if a_tys.len() == b_tys.len() && !a_tys.is_empty() =>
+                {
+                    let (a_last_ty, a_rest_tys) = a_tys.split_last().unwrap();
+                    let b_last_ty = b_tys.last().unwrap();
+
+                    // Substitute just the tail field of B., and require that they're equal.
+                    let unsized_a_ty =
+                        tcx.mk_tup_from_iter(a_rest_tys.iter().chain([b_last_ty]).copied());
+                    let mut nested_goals = ecx.eq(goal.param_env, unsized_a_ty, b_ty)?;
+
+                    // Similar to ADTs, require that the rest of the fields are equal.
+                    nested_goals.push(goal.with(
+                        tcx,
+                        ty::Binder::dummy(
+                            tcx.mk_trait_ref(goal.predicate.def_id(), [*a_last_ty, *b_last_ty]),
+                        ),
+                    ));
+
+                    ecx.evaluate_all_and_make_canonical_response(nested_goals)
+                }
+                _ => Err(NoSolution),
+            }
+        })
+    }
+
+    fn consider_builtin_dyn_upcast_candidates(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> Vec<CanonicalResponse<'tcx>> {
+        let tcx = ecx.tcx();
+
+        let a_ty = goal.predicate.self_ty();
+        let b_ty = goal.predicate.trait_ref.substs.type_at(1);
+        let ty::Dynamic(a_data, a_region, ty::Dyn) = *a_ty.kind() else {
+            return vec![];
+        };
+        let ty::Dynamic(b_data, b_region, ty::Dyn) = *b_ty.kind() else {
+            return vec![];
+        };
+
+        // All of a's auto traits need to be in b's auto traits.
+        let auto_traits_compatible =
+            b_data.auto_traits().all(|b| a_data.auto_traits().any(|a| a == b));
+        if !auto_traits_compatible {
+            return vec![];
+        }
+
+        let mut unsize_dyn_to_principal = |principal: Option<ty::PolyExistentialTraitRef<'tcx>>| {
+            ecx.probe(|ecx| -> Result<_, NoSolution> {
+                // Require that all of the trait predicates from A match B, except for
+                // the auto traits. We do this by constructing a new A type with B's
+                // auto traits, and equating these types.
+                let new_a_data = principal
+                    .into_iter()
+                    .map(|trait_ref| trait_ref.map_bound(ty::ExistentialPredicate::Trait))
+                    .chain(a_data.iter().filter(|a| {
+                        matches!(a.skip_binder(), ty::ExistentialPredicate::Projection(_))
+                    }))
+                    .chain(
+                        b_data
+                            .auto_traits()
+                            .map(ty::ExistentialPredicate::AutoTrait)
+                            .map(ty::Binder::dummy),
+                    );
+                let new_a_data = tcx.mk_poly_existential_predicates_from_iter(new_a_data);
+                let new_a_ty = tcx.mk_dynamic(new_a_data, b_region, ty::Dyn);
+
+                // We also require that A's lifetime outlives B's lifetime.
+                let mut nested_obligations = ecx.eq(goal.param_env, new_a_ty, b_ty)?;
+                nested_obligations.push(
+                    goal.with(tcx, ty::Binder::dummy(ty::OutlivesPredicate(a_region, b_region))),
+                );
+
+                ecx.evaluate_all_and_make_canonical_response(nested_obligations)
+            })
+        };
+
+        let mut responses = vec![];
+        // If the principal def ids match (or are both none), then we're not doing
+        // trait upcasting. We're just removing auto traits (or shortening the lifetime).
+        if a_data.principal_def_id() == b_data.principal_def_id() {
+            if let Ok(response) = unsize_dyn_to_principal(a_data.principal()) {
+                responses.push(response);
+            }
+        } else if let Some(a_principal) = a_data.principal()
+            && let Some(b_principal) = b_data.principal()
+        {
+            for super_trait_ref in supertraits(tcx, a_principal.with_self_ty(tcx, a_ty)) {
+                if super_trait_ref.def_id() != b_principal.def_id() {
+                    continue;
+                }
+                let erased_trait_ref = super_trait_ref
+                    .map_bound(|trait_ref| ty::ExistentialTraitRef::erase_self_ty(tcx, trait_ref));
+                if let Ok(response) = unsize_dyn_to_principal(Some(erased_trait_ref)) {
+                    responses.push(response);
+                }
+            }
+        }
+
+        responses
+    }
+
+    fn consider_builtin_discriminant_kind_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        _goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        // `DiscriminantKind` is automatically implemented for every type.
+        ecx.make_canonical_response(Certainty::Yes)
+    }
 }
 
 impl<'tcx> EvalCtxt<'_, 'tcx> {
@@ -195,16 +522,16 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
     fn probe_and_evaluate_goal_for_constituent_tys(
         &mut self,
         goal: Goal<'tcx, TraitPredicate<'tcx>>,
-        constituent_tys: impl Fn(&InferCtxt<'tcx>, Ty<'tcx>) -> Result<Vec<Ty<'tcx>>, NoSolution>,
+        constituent_tys: impl Fn(&EvalCtxt<'_, 'tcx>, Ty<'tcx>) -> Result<Vec<Ty<'tcx>>, NoSolution>,
     ) -> QueryResult<'tcx> {
-        self.infcx.probe(|_| {
-            self.evaluate_all_and_make_canonical_response(
-                constituent_tys(self.infcx, goal.predicate.self_ty())?
+        self.probe(|this| {
+            this.evaluate_all_and_make_canonical_response(
+                constituent_tys(this, goal.predicate.self_ty())?
                     .into_iter()
                     .map(|ty| {
                         goal.with(
-                            self.tcx(),
-                            ty::Binder::dummy(goal.predicate.with_self_ty(self.tcx(), ty)),
+                            this.tcx(),
+                            ty::Binder::dummy(goal.predicate.with_self_ty(this.tcx(), ty)),
                         )
                     })
                     .collect(),
@@ -217,73 +544,6 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         goal: Goal<'tcx, TraitPredicate<'tcx>>,
     ) -> QueryResult<'tcx> {
         let candidates = self.assemble_and_evaluate_candidates(goal);
-        self.merge_trait_candidates_discard_reservation_impls(candidates)
-    }
-
-    #[instrument(level = "debug", skip(self), ret)]
-    pub(super) fn merge_trait_candidates_discard_reservation_impls(
-        &mut self,
-        mut candidates: Vec<Candidate<'tcx>>,
-    ) -> QueryResult<'tcx> {
-        match candidates.len() {
-            0 => return Err(NoSolution),
-            1 => return Ok(self.discard_reservation_impl(candidates.pop().unwrap()).result),
-            _ => {}
-        }
-
-        if candidates.len() > 1 {
-            let mut i = 0;
-            'outer: while i < candidates.len() {
-                for j in (0..candidates.len()).filter(|&j| i != j) {
-                    if self.trait_candidate_should_be_dropped_in_favor_of(
-                        &candidates[i],
-                        &candidates[j],
-                    ) {
-                        debug!(candidate = ?candidates[i], "Dropping candidate #{}/{}", i, candidates.len());
-                        candidates.swap_remove(i);
-                        continue 'outer;
-                    }
-                }
-
-                debug!(candidate = ?candidates[i], "Retaining candidate #{}/{}", i, candidates.len());
-                // If there are *STILL* multiple candidates, give up
-                // and report ambiguity.
-                i += 1;
-                if i > 1 {
-                    debug!("multiple matches, ambig");
-                    // FIXME: return overflow if all candidates overflow, otherwise return ambiguity.
-                    unimplemented!();
-                }
-            }
-        }
-
-        Ok(self.discard_reservation_impl(candidates.pop().unwrap()).result)
-    }
-
-    fn trait_candidate_should_be_dropped_in_favor_of(
-        &self,
-        candidate: &Candidate<'tcx>,
-        other: &Candidate<'tcx>,
-    ) -> bool {
-        // FIXME: implement this
-        match (candidate.source, other.source) {
-            (CandidateSource::Impl(_), _)
-            | (CandidateSource::ParamEnv(_), _)
-            | (CandidateSource::AliasBound(_), _)
-            | (CandidateSource::BuiltinImpl, _) => unimplemented!(),
-        }
-    }
-
-    fn discard_reservation_impl(&self, candidate: Candidate<'tcx>) -> Candidate<'tcx> {
-        if let CandidateSource::Impl(def_id) = candidate.source {
-            if let ty::ImplPolarity::Reservation = self.tcx().impl_polarity(def_id) {
-                debug!("Selected reservation impl");
-                // FIXME: reduce candidate to ambiguous
-                // FIXME: replace `var_values` with identity, yeet external constraints.
-                unimplemented!()
-            }
-        }
-
-        candidate
+        self.merge_candidates_and_discard_reservation_impls(candidates)
     }
 }
diff --git a/compiler/rustc_trait_selection/src/solve/trait_goals/structural_traits.rs b/compiler/rustc_trait_selection/src/solve/trait_goals/structural_traits.rs
index a11cd13cb..d7d93377c 100644
--- a/compiler/rustc_trait_selection/src/solve/trait_goals/structural_traits.rs
+++ b/compiler/rustc_trait_selection/src/solve/trait_goals/structural_traits.rs
@@ -1,16 +1,19 @@
-use rustc_hir::{Movability, Mutability};
-use rustc_infer::{infer::InferCtxt, traits::query::NoSolution};
-use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_data_structures::fx::FxHashMap;
+use rustc_hir::{def_id::DefId, Movability, Mutability};
+use rustc_infer::traits::query::NoSolution;
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeSuperFoldable};
+
+use crate::solve::EvalCtxt;
 
 // Calculates the constituent types of a type for `auto trait` purposes.
 //
 // For types with an "existential" binder, i.e. generator witnesses, we also
 // instantiate the binder with placeholders eagerly.
 pub(super) fn instantiate_constituent_tys_for_auto_trait<'tcx>(
-    infcx: &InferCtxt<'tcx>,
+    ecx: &EvalCtxt<'_, 'tcx>,
     ty: Ty<'tcx>,
 ) -> Result<Vec<Ty<'tcx>>, NoSolution> {
-    let tcx = infcx.tcx;
+    let tcx = ecx.tcx();
     match *ty.kind() {
         ty::Uint(_)
         | ty::Int(_)
@@ -18,21 +21,24 @@ pub(super) fn instantiate_constituent_tys_for_auto_trait<'tcx>(
         | ty::Float(_)
         | ty::FnDef(..)
         | ty::FnPtr(_)
-        | ty::Str
         | ty::Error(_)
         | ty::Infer(ty::IntVar(_) | ty::FloatVar(_))
         | ty::Never
         | ty::Char => Ok(vec![]),
 
-        ty::Placeholder(..)
-        | ty::Dynamic(..)
+        // Treat this like `struct str([u8]);`
+        ty::Str => Ok(vec![tcx.mk_slice(tcx.types.u8)]),
+
+        ty::Dynamic(..)
         | ty::Param(..)
         | ty::Foreign(..)
         | ty::Alias(ty::Projection, ..)
-        | ty::Bound(..)
-        | ty::Infer(ty::TyVar(_)) => Err(NoSolution),
+        | ty::Placeholder(..) => Err(NoSolution),
 
-        ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => bug!(),
+        ty::Bound(..)
+        | ty::Infer(ty::TyVar(_) | ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => {
+            bug!("unexpected type `{ty}`")
+        }
 
         ty::RawPtr(ty::TypeAndMut { ty: element_ty, .. }) | ty::Ref(_, element_ty, _) => {
             Ok(vec![element_ty])
@@ -52,9 +58,9 @@ pub(super) fn instantiate_constituent_tys_for_auto_trait<'tcx>(
             Ok(vec![generator_substs.tupled_upvars_ty(), generator_substs.witness()])
         }
 
-        ty::GeneratorWitness(types) => {
-            Ok(infcx.replace_bound_vars_with_placeholders(types).to_vec())
-        }
+        ty::GeneratorWitness(types) => Ok(ecx.instantiate_binder_with_placeholders(types).to_vec()),
+
+        ty::GeneratorWitnessMIR(..) => todo!(),
 
         // For `PhantomData<T>`, we pass `T`.
         ty::Adt(def, substs) if def.is_phantom_data() => Ok(vec![substs.type_at(0)]),
@@ -65,13 +71,13 @@ pub(super) fn instantiate_constituent_tys_for_auto_trait<'tcx>(
             // We can resolve the `impl Trait` to its concrete type,
             // which enforces a DAG between the functions requiring
             // the auto trait bounds in question.
-            Ok(vec![tcx.bound_type_of(def_id).subst(tcx, substs)])
+            Ok(vec![tcx.type_of(def_id).subst(tcx, substs)])
         }
     }
 }
 
 pub(super) fn instantiate_constituent_tys_for_sized_trait<'tcx>(
-    infcx: &InferCtxt<'tcx>,
+    ecx: &EvalCtxt<'_, 'tcx>,
     ty: Ty<'tcx>,
 ) -> Result<Vec<Ty<'tcx>>, NoSolution> {
     match *ty.kind() {
@@ -87,6 +93,7 @@ pub(super) fn instantiate_constituent_tys_for_sized_trait<'tcx>(
         | ty::Ref(..)
         | ty::Generator(..)
         | ty::GeneratorWitness(..)
+        | ty::GeneratorWitnessMIR(..)
         | ty::Array(..)
         | ty::Closure(..)
         | ty::Never
@@ -99,27 +106,28 @@ pub(super) fn instantiate_constituent_tys_for_sized_trait<'tcx>(
         | ty::Foreign(..)
         | ty::Alias(..)
         | ty::Param(_)
-        | ty::Infer(ty::TyVar(_)) => Err(NoSolution),
+        | ty::Placeholder(..) => Err(NoSolution),
 
-        ty::Placeholder(..)
-        | ty::Bound(..)
-        | ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => bug!(),
+        ty::Bound(..)
+        | ty::Infer(ty::TyVar(_) | ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => {
+            bug!("unexpected type `{ty}`")
+        }
 
         ty::Tuple(tys) => Ok(tys.to_vec()),
 
         ty::Adt(def, substs) => {
-            let sized_crit = def.sized_constraint(infcx.tcx);
+            let sized_crit = def.sized_constraint(ecx.tcx());
             Ok(sized_crit
                 .0
                 .iter()
-                .map(|ty| sized_crit.rebind(*ty).subst(infcx.tcx, substs))
+                .map(|ty| sized_crit.rebind(*ty).subst(ecx.tcx(), substs))
                 .collect())
         }
     }
 }
 
 pub(super) fn instantiate_constituent_tys_for_copy_clone_trait<'tcx>(
-    infcx: &InferCtxt<'tcx>,
+    ecx: &EvalCtxt<'_, 'tcx>,
     ty: Ty<'tcx>,
 ) -> Result<Vec<Ty<'tcx>>, NoSolution> {
     match *ty.kind() {
@@ -148,18 +156,19 @@ pub(super) fn instantiate_constituent_tys_for_copy_clone_trait<'tcx>(
         | ty::Adt(_, _)
         | ty::Alias(_, _)
         | ty::Param(_)
-        | ty::Infer(ty::TyVar(_)) => Err(NoSolution),
+        | ty::Placeholder(..) => Err(NoSolution),
 
-        ty::Placeholder(..)
-        | ty::Bound(..)
-        | ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => bug!(),
+        ty::Bound(..)
+        | ty::Infer(ty::TyVar(_) | ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => {
+            bug!("unexpected type `{ty}`")
+        }
 
         ty::Tuple(tys) => Ok(tys.to_vec()),
 
         ty::Closure(_, substs) => Ok(vec![substs.as_closure().tupled_upvars_ty()]),
 
         ty::Generator(_, substs, Movability::Movable) => {
-            if infcx.tcx.features().generator_clone {
+            if ecx.tcx().features().generator_clone {
                 let generator = substs.as_generator();
                 Ok(vec![generator.tupled_upvars_ty(), generator.witness()])
             } else {
@@ -167,12 +176,13 @@ pub(super) fn instantiate_constituent_tys_for_copy_clone_trait<'tcx>(
             }
         }
 
-        ty::GeneratorWitness(types) => {
-            Ok(infcx.replace_bound_vars_with_placeholders(types).to_vec())
-        }
+        ty::GeneratorWitness(types) => Ok(ecx.instantiate_binder_with_placeholders(types).to_vec()),
+
+        ty::GeneratorWitnessMIR(..) => todo!(),
     }
 }
 
+// Returns a binder of the tupled inputs types and output type from a builtin callable type.
 pub(crate) fn extract_tupled_inputs_and_output_from_callable<'tcx>(
     tcx: TyCtxt<'tcx>,
     self_ty: Ty<'tcx>,
@@ -180,13 +190,11 @@ pub(crate) fn extract_tupled_inputs_and_output_from_callable<'tcx>(
 ) -> Result<Option<ty::Binder<'tcx, (Ty<'tcx>, Ty<'tcx>)>>, NoSolution> {
     match *self_ty.kind() {
         ty::FnDef(def_id, substs) => Ok(Some(
-            tcx.bound_fn_sig(def_id)
+            tcx.fn_sig(def_id)
                 .subst(tcx, substs)
-                .map_bound(|sig| (tcx.mk_tup(sig.inputs().iter()), sig.output())),
+                .map_bound(|sig| (tcx.mk_tup(sig.inputs()), sig.output())),
         )),
-        ty::FnPtr(sig) => {
-            Ok(Some(sig.map_bound(|sig| (tcx.mk_tup(sig.inputs().iter()), sig.output()))))
-        }
+        ty::FnPtr(sig) => Ok(Some(sig.map_bound(|sig| (tcx.mk_tup(sig.inputs()), sig.output())))),
         ty::Closure(_, substs) => {
             let closure_substs = substs.as_closure();
             match closure_substs.kind_ty().to_opt_closure_kind() {
@@ -211,13 +219,127 @@ pub(crate) fn extract_tupled_inputs_and_output_from_callable<'tcx>(
         | ty::Dynamic(_, _, _)
         | ty::Generator(_, _, _)
         | ty::GeneratorWitness(_)
+        | ty::GeneratorWitnessMIR(..)
         | ty::Never
         | ty::Tuple(_)
         | ty::Alias(_, _)
         | ty::Param(_)
-        | ty::Placeholder(_)
-        | ty::Bound(_, _)
-        | ty::Infer(_)
+        | ty::Placeholder(..)
+        | ty::Infer(ty::IntVar(_) | ty::FloatVar(_))
         | ty::Error(_) => Err(NoSolution),
+
+        ty::Bound(..)
+        | ty::Infer(ty::TyVar(_) | ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => {
+            bug!("unexpected type `{self_ty}`")
+        }
+    }
+}
+
+/// Assemble a list of predicates that would be present on a theoretical
+/// user impl for an object type. These predicates must be checked any time
+/// we assemble a built-in object candidate for an object type, since they
+/// are not implied by the well-formedness of the type.
+///
+/// For example, given the following traits:
+///
+/// ```rust,ignore (theoretical code)
+/// trait Foo: Baz {
+///     type Bar: Copy;
+/// }
+///
+/// trait Baz {}
+/// ```
+///
+/// For the dyn type `dyn Foo<Item = Ty>`, we can imagine there being a
+/// pair of theoretical impls:
+///
+/// ```rust,ignore (theoretical code)
+/// impl Foo for dyn Foo<Item = Ty>
+/// where
+///     Self: Baz,
+///     <Self as Foo>::Bar: Copy,
+/// {
+///     type Bar = Ty;
+/// }
+///
+/// impl Baz for dyn Foo<Item = Ty> {}
+/// ```
+///
+/// However, in order to make such impls well-formed, we need to do an
+/// additional step of eagerly folding the associated types in the where
+/// clauses of the impl. In this example, that means replacing
+/// `<Self as Foo>::Bar` with `Ty` in the first impl.
+pub(crate) fn predicates_for_object_candidate<'tcx>(
+    ecx: &EvalCtxt<'_, 'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
+    trait_ref: ty::TraitRef<'tcx>,
+    object_bound: &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>>,
+) -> Vec<ty::Predicate<'tcx>> {
+    let tcx = ecx.tcx();
+    let mut requirements = vec![];
+    requirements.extend(
+        tcx.super_predicates_of(trait_ref.def_id).instantiate(tcx, trait_ref.substs).predicates,
+    );
+    for item in tcx.associated_items(trait_ref.def_id).in_definition_order() {
+        // FIXME(associated_const_equality): Also add associated consts to
+        // the requirements here.
+        if item.kind == ty::AssocKind::Type {
+            requirements.extend(tcx.item_bounds(item.def_id).subst(tcx, trait_ref.substs));
+        }
+    }
+
+    let mut replace_projection_with = FxHashMap::default();
+    for bound in object_bound {
+        if let ty::ExistentialPredicate::Projection(proj) = bound.skip_binder() {
+            let proj = proj.with_self_ty(tcx, trait_ref.self_ty());
+            let old_ty = replace_projection_with.insert(proj.def_id(), bound.rebind(proj));
+            assert_eq!(
+                old_ty,
+                None,
+                "{} has two substitutions: {} and {}",
+                proj.projection_ty,
+                proj.term,
+                old_ty.unwrap()
+            );
+        }
+    }
+
+    requirements.fold_with(&mut ReplaceProjectionWith {
+        ecx,
+        param_env,
+        mapping: replace_projection_with,
+    })
+}
+
+struct ReplaceProjectionWith<'a, 'tcx> {
+    ecx: &'a EvalCtxt<'a, 'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
+    mapping: FxHashMap<DefId, ty::PolyProjectionPredicate<'tcx>>,
+}
+
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ReplaceProjectionWith<'_, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
+        self.ecx.tcx()
+    }
+
+    fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
+        if let ty::Alias(ty::Projection, alias_ty) = *ty.kind()
+            && let Some(replacement) = self.mapping.get(&alias_ty.def_id)
+        {
+            // We may have a case where our object type's projection bound is higher-ranked,
+            // but the where clauses we instantiated are not. We can solve this by instantiating
+            // the binder at the usage site.
+            let proj = self.ecx.instantiate_binder_with_infer(*replacement);
+            // FIXME: Technically this folder could be fallible?
+            let nested = self
+                .ecx
+                .eq(self.param_env, alias_ty, proj.projection_ty)
+                .expect("expected to be able to unify goal projection with dyn's projection");
+            // FIXME: Technically we could register these too..
+            assert!(nested.is_empty(), "did not expect unification to have any nested goals");
+            proj.term.ty().unwrap()
+        } else {
+            ty.super_fold_with(self)
+        }
     }
 }
diff --git a/compiler/rustc_trait_selection/src/traits/auto_trait.rs b/compiler/rustc_trait_selection/src/traits/auto_trait.rs
index 948632ccc..1fb8659bb 100644
--- a/compiler/rustc_trait_selection/src/traits/auto_trait.rs
+++ b/compiler/rustc_trait_selection/src/traits/auto_trait.rs
@@ -9,7 +9,7 @@ use crate::infer::InferCtxt;
 use crate::traits::project::ProjectAndUnifyResult;
 use rustc_middle::mir::interpret::ErrorHandled;
 use rustc_middle::ty::fold::{TypeFolder, TypeSuperFoldable};
-use rustc_middle::ty::visit::TypeVisitable;
+use rustc_middle::ty::visit::TypeVisitableExt;
 use rustc_middle::ty::{ImplPolarity, Region, RegionVid};
 
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
@@ -350,14 +350,14 @@ impl<'tcx> AutoTraitFinder<'tcx> {
             )
             .map(|o| o.predicate);
             new_env = ty::ParamEnv::new(
-                tcx.mk_predicates(normalized_preds),
+                tcx.mk_predicates_from_iter(normalized_preds),
                 param_env.reveal(),
                 param_env.constness(),
             );
         }
 
         let final_user_env = ty::ParamEnv::new(
-            tcx.mk_predicates(user_computed_preds.into_iter()),
+            tcx.mk_predicates_from_iter(user_computed_preds.into_iter()),
             user_env.reveal(),
             user_env.constness(),
         );
@@ -823,14 +823,18 @@ impl<'tcx> AutoTraitFinder<'tcx> {
                         _ => return false,
                     }
                 }
+
                 // There's not really much we can do with these predicates -
                 // we start out with a `ParamEnv` with no inference variables,
                 // and these don't correspond to adding any new bounds to
                 // the `ParamEnv`.
                 ty::PredicateKind::WellFormed(..)
+                | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
+                | ty::PredicateKind::AliasEq(..)
                 | ty::PredicateKind::ObjectSafe(..)
                 | ty::PredicateKind::ClosureKind(..)
                 | ty::PredicateKind::Subtype(..)
+                // FIXME(generic_const_exprs): you can absolutely add this as a where clauses
                 | ty::PredicateKind::ConstEvaluatable(..)
                 | ty::PredicateKind::Coerce(..)
                 | ty::PredicateKind::TypeWellFormedFromEnv(..) => {}
@@ -855,8 +859,8 @@ pub struct RegionReplacer<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
 }
 
-impl<'a, 'tcx> TypeFolder<'tcx> for RegionReplacer<'a, 'tcx> {
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for RegionReplacer<'a, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
diff --git a/compiler/rustc_trait_selection/src/traits/chalk_fulfill.rs b/compiler/rustc_trait_selection/src/traits/chalk_fulfill.rs
index e88950523..b42a49eb4 100644
--- a/compiler/rustc_trait_selection/src/traits/chalk_fulfill.rs
+++ b/compiler/rustc_trait_selection/src/traits/chalk_fulfill.rs
@@ -7,24 +7,18 @@ use crate::traits::{
     ChalkEnvironmentAndGoal, FulfillmentError, FulfillmentErrorCode, PredicateObligation,
     SelectionError, TraitEngine,
 };
-use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
-use rustc_middle::ty::{self, TypeVisitable};
+use rustc_data_structures::fx::FxIndexSet;
+use rustc_middle::ty::TypeVisitableExt;
 
 pub struct FulfillmentContext<'tcx> {
     obligations: FxIndexSet<PredicateObligation<'tcx>>,
 
-    relationships: FxHashMap<ty::TyVid, ty::FoundRelationships>,
-
     usable_in_snapshot: bool,
 }
 
 impl FulfillmentContext<'_> {
     pub(super) fn new() -> Self {
-        FulfillmentContext {
-            obligations: FxIndexSet::default(),
-            relationships: FxHashMap::default(),
-            usable_in_snapshot: false,
-        }
+        FulfillmentContext { obligations: FxIndexSet::default(), usable_in_snapshot: false }
     }
 
     pub(crate) fn new_in_snapshot() -> Self {
@@ -43,20 +37,10 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentContext<'tcx> {
         }
         let obligation = infcx.resolve_vars_if_possible(obligation);
 
-        super::relationships::update(self, infcx, &obligation);
-
         self.obligations.insert(obligation);
     }
 
-    fn select_all_or_error(&mut self, infcx: &InferCtxt<'tcx>) -> Vec<FulfillmentError<'tcx>> {
-        {
-            let errors = self.select_where_possible(infcx);
-
-            if !errors.is_empty() {
-                return errors;
-            }
-        }
-
+    fn collect_remaining_errors(&mut self) -> Vec<FulfillmentError<'tcx>> {
         // any remaining obligations are errors
         self.obligations
             .iter()
@@ -151,11 +135,14 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentContext<'tcx> {
         errors
     }
 
-    fn pending_obligations(&self) -> Vec<PredicateObligation<'tcx>> {
-        self.obligations.iter().cloned().collect()
+    fn drain_unstalled_obligations(
+        &mut self,
+        _: &InferCtxt<'tcx>,
+    ) -> Vec<PredicateObligation<'tcx>> {
+        unimplemented!()
     }
 
-    fn relationships(&mut self) -> &mut FxHashMap<ty::TyVid, ty::FoundRelationships> {
-        &mut self.relationships
+    fn pending_obligations(&self) -> Vec<PredicateObligation<'tcx>> {
+        self.obligations.iter().cloned().collect()
     }
 }
diff --git a/compiler/rustc_trait_selection/src/traits/coherence.rs b/compiler/rustc_trait_selection/src/traits/coherence.rs
index 225c1050c..6b688c322 100644
--- a/compiler/rustc_trait_selection/src/traits/coherence.rs
+++ b/compiler/rustc_trait_selection/src/traits/coherence.rs
@@ -17,12 +17,11 @@ use crate::traits::{
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_errors::Diagnostic;
 use rustc_hir::def_id::{DefId, CRATE_DEF_ID, LOCAL_CRATE};
-use rustc_hir::CRATE_HIR_ID;
 use rustc_infer::infer::{DefiningAnchor, InferCtxt, TyCtxtInferExt};
 use rustc_infer::traits::util;
 use rustc_middle::traits::specialization_graph::OverlapMode;
 use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams};
-use rustc_middle::ty::visit::TypeVisitable;
+use rustc_middle::ty::visit::{TypeVisitable, TypeVisitableExt};
 use rustc_middle::ty::{self, ImplSubject, Ty, TyCtxt, TypeVisitor};
 use rustc_span::symbol::sym;
 use rustc_span::DUMMY_SP;
@@ -83,8 +82,8 @@ pub fn overlapping_impls(
         (Some(a), Some(b)) => iter::zip(a.skip_binder().substs, b.skip_binder().substs)
             .all(|(arg1, arg2)| drcx.generic_args_may_unify(arg1, arg2)),
         (None, None) => {
-            let self_ty1 = tcx.type_of(impl1_def_id);
-            let self_ty2 = tcx.type_of(impl2_def_id);
+            let self_ty1 = tcx.type_of(impl1_def_id).skip_binder();
+            let self_ty2 = tcx.type_of(impl2_def_id).skip_binder();
             drcx.types_may_unify(self_ty1, self_ty2)
         }
         _ => bug!("unexpected impls: {impl1_def_id:?} {impl2_def_id:?}"),
@@ -125,7 +124,7 @@ fn with_fresh_ty_vars<'cx, 'tcx>(
 
     let header = ty::ImplHeader {
         impl_def_id,
-        self_ty: tcx.bound_type_of(impl_def_id).subst(tcx, impl_substs),
+        self_ty: tcx.type_of(impl_def_id).subst(tcx, impl_substs),
         trait_ref: tcx.impl_trait_ref(impl_def_id).map(|i| i.subst(tcx, impl_substs)),
         predicates: tcx.predicates_of(impl_def_id).instantiate(tcx, impl_substs).predicates,
     };
@@ -218,6 +217,7 @@ fn equate_impl_headers<'cx, 'tcx>(
     selcx
         .infcx
         .at(&ObligationCause::dummy(), ty::ParamEnv::empty())
+        .define_opaque_types(true)
         .eq_impl_headers(impl1_header, impl2_header)
         .map(|infer_ok| infer_ok.obligations)
         .ok()
@@ -382,18 +382,14 @@ fn resolve_negative_obligation<'tcx>(
         return false;
     }
 
-    let (body_id, body_def_id) = if let Some(body_def_id) = body_def_id.as_local() {
-        (tcx.hir().local_def_id_to_hir_id(body_def_id), body_def_id)
-    } else {
-        (CRATE_HIR_ID, CRATE_DEF_ID)
-    };
+    let body_def_id = body_def_id.as_local().unwrap_or(CRATE_DEF_ID);
 
     let ocx = ObligationCtxt::new(&infcx);
     let wf_tys = ocx.assumed_wf_types(param_env, DUMMY_SP, body_def_id);
     let outlives_env = OutlivesEnvironment::with_bounds(
         param_env,
         Some(&infcx),
-        infcx.implied_bounds_tys(param_env, body_id, wf_tys),
+        infcx.implied_bounds_tys(param_env, body_def_id, wf_tys),
     );
 
     infcx.process_registered_region_obligations(outlives_env.region_bound_pairs(), param_env);
@@ -632,7 +628,7 @@ enum OrphanCheckEarlyExit<'tcx> {
     LocalTy(Ty<'tcx>),
 }
 
-impl<'tcx> TypeVisitor<'tcx> for OrphanChecker<'tcx> {
+impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for OrphanChecker<'tcx> {
     type BreakTy = OrphanCheckEarlyExit<'tcx>;
     fn visit_region(&mut self, _r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
         ControlFlow::Continue(())
@@ -701,7 +697,9 @@ impl<'tcx> TypeVisitor<'tcx> for OrphanChecker<'tcx> {
             // This should only be created when checking whether we have to check whether some
             // auto trait impl applies. There will never be multiple impls, so we can just
             // act as if it were a local type here.
-            ty::GeneratorWitness(_) => ControlFlow::Break(OrphanCheckEarlyExit::LocalTy(ty)),
+            ty::GeneratorWitness(_) | ty::GeneratorWitnessMIR(..) => {
+                ControlFlow::Break(OrphanCheckEarlyExit::LocalTy(ty))
+            }
             ty::Alias(ty::Opaque, ..) => {
                 // This merits some explanation.
                 // Normally, opaque types are not involved when performing
diff --git a/compiler/rustc_trait_selection/src/traits/const_evaluatable.rs b/compiler/rustc_trait_selection/src/traits/const_evaluatable.rs
index f779d9dd8..345e84990 100644
--- a/compiler/rustc_trait_selection/src/traits/const_evaluatable.rs
+++ b/compiler/rustc_trait_selection/src/traits/const_evaluatable.rs
@@ -14,7 +14,7 @@ use rustc_middle::mir::interpret::ErrorHandled;
 
 use rustc_middle::traits::ObligationCause;
 use rustc_middle::ty::abstract_const::NotConstEvaluatable;
-use rustc_middle::ty::{self, TyCtxt, TypeVisitable, TypeVisitor};
+use rustc_middle::ty::{self, TyCtxt, TypeVisitable, TypeVisitableExt, TypeVisitor};
 
 use rustc_span::Span;
 use std::ops::ControlFlow;
@@ -171,7 +171,7 @@ fn satisfied_from_param_env<'tcx>(
         single_match: Option<Result<ty::Const<'tcx>, ()>>,
     }
 
-    impl<'a, 'tcx> TypeVisitor<'tcx> for Visitor<'a, 'tcx> {
+    impl<'a, 'tcx> TypeVisitor<TyCtxt<'tcx>> for Visitor<'a, 'tcx> {
         type BreakTy = ();
         fn visit_const(&mut self, c: ty::Const<'tcx>) -> ControlFlow<Self::BreakTy> {
             debug!("is_const_evaluatable: candidate={:?}", c);
@@ -219,7 +219,7 @@ fn satisfied_from_param_env<'tcx>(
     }
 
     if let Some(Ok(c)) = single_match {
-        let ocx = ObligationCtxt::new(infcx);
+        let ocx = ObligationCtxt::new_in_snapshot(infcx);
         assert!(ocx.eq(&ObligationCause::dummy(), param_env, c.ty(), ct.ty()).is_ok());
         assert!(ocx.eq(&ObligationCause::dummy(), param_env, c, ct).is_ok());
         assert!(ocx.select_all_or_error().is_empty());
diff --git a/compiler/rustc_trait_selection/src/traits/engine.rs b/compiler/rustc_trait_selection/src/traits/engine.rs
index 369f80139..b20636174 100644
--- a/compiler/rustc_trait_selection/src/traits/engine.rs
+++ b/compiler/rustc_trait_selection/src/traits/engine.rs
@@ -104,7 +104,7 @@ impl<'a, 'tcx> ObligationCtxt<'a, 'tcx> {
         });
     }
 
-    pub fn normalize<T: TypeFoldable<'tcx>>(
+    pub fn normalize<T: TypeFoldable<TyCtxt<'tcx>>>(
         &self,
         cause: &ObligationCause<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
@@ -128,6 +128,7 @@ impl<'a, 'tcx> ObligationCtxt<'a, 'tcx> {
     {
         self.infcx
             .at(cause, param_env)
+            .define_opaque_types(true)
             .eq_exp(a_is_expected, a, b)
             .map(|infer_ok| self.register_infer_ok_obligations(infer_ok))
     }
@@ -141,6 +142,7 @@ impl<'a, 'tcx> ObligationCtxt<'a, 'tcx> {
     ) -> Result<(), TypeError<'tcx>> {
         self.infcx
             .at(cause, param_env)
+            .define_opaque_types(true)
             .eq(expected, actual)
             .map(|infer_ok| self.register_infer_ok_obligations(infer_ok))
     }
@@ -155,6 +157,7 @@ impl<'a, 'tcx> ObligationCtxt<'a, 'tcx> {
     ) -> Result<(), TypeError<'tcx>> {
         self.infcx
             .at(cause, param_env)
+            .define_opaque_types(true)
             .sup(expected, actual)
             .map(|infer_ok| self.register_infer_ok_obligations(infer_ok))
     }
@@ -169,6 +172,7 @@ impl<'a, 'tcx> ObligationCtxt<'a, 'tcx> {
     ) -> Result<(), TypeError<'tcx>> {
         self.infcx
             .at(cause, param_env)
+            .define_opaque_types(true)
             .sup(expected, actual)
             .map(|infer_ok| self.register_infer_ok_obligations(infer_ok))
     }
@@ -190,8 +194,7 @@ impl<'a, 'tcx> ObligationCtxt<'a, 'tcx> {
         let tcx = self.infcx.tcx;
         let assumed_wf_types = tcx.assumed_wf_types(def_id);
         let mut implied_bounds = FxIndexSet::default();
-        let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
-        let cause = ObligationCause::misc(span, hir_id);
+        let cause = ObligationCause::misc(span, def_id);
         for ty in assumed_wf_types {
             // FIXME(@lcnr): rustc currently does not check wf for types
             // pre-normalization, meaning that implied bounds are sometimes
@@ -217,7 +220,7 @@ impl<'a, 'tcx> ObligationCtxt<'a, 'tcx> {
         answer: T,
     ) -> Fallible<CanonicalQueryResponse<'tcx, T>>
     where
-        T: Debug + TypeFoldable<'tcx>,
+        T: Debug + TypeFoldable<TyCtxt<'tcx>>,
         Canonical<'tcx, QueryResponse<'tcx, T>>: ArenaAllocatable<'tcx>,
     {
         self.infcx.make_canonicalized_query_response(
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/ambiguity.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/ambiguity.rs
index 0419bb3f7..84045c4d0 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/ambiguity.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/ambiguity.rs
@@ -22,7 +22,7 @@ pub fn recompute_applicable_impls<'tcx>(
     let impl_may_apply = |impl_def_id| {
         let ocx = ObligationCtxt::new_in_snapshot(infcx);
         let placeholder_obligation =
-            infcx.replace_bound_vars_with_placeholders(obligation.predicate);
+            infcx.instantiate_binder_with_placeholders(obligation.predicate);
         let obligation_trait_ref =
             ocx.normalize(&ObligationCause::dummy(), param_env, placeholder_obligation.trait_ref);
 
@@ -47,11 +47,11 @@ pub fn recompute_applicable_impls<'tcx>(
     let param_env_candidate_may_apply = |poly_trait_predicate: ty::PolyTraitPredicate<'tcx>| {
         let ocx = ObligationCtxt::new_in_snapshot(infcx);
         let placeholder_obligation =
-            infcx.replace_bound_vars_with_placeholders(obligation.predicate);
+            infcx.instantiate_binder_with_placeholders(obligation.predicate);
         let obligation_trait_ref =
             ocx.normalize(&ObligationCause::dummy(), param_env, placeholder_obligation.trait_ref);
 
-        let param_env_predicate = infcx.replace_bound_vars_with_fresh_vars(
+        let param_env_predicate = infcx.instantiate_binder_with_fresh_vars(
             DUMMY_SP,
             LateBoundRegionConversionTime::HigherRankedType,
             poly_trait_predicate,
@@ -81,7 +81,7 @@ pub fn recompute_applicable_impls<'tcx>(
     );
 
     let predicates =
-        tcx.predicates_of(obligation.cause.body_id.owner.to_def_id()).instantiate_identity(tcx);
+        tcx.predicates_of(obligation.cause.body_id.to_def_id()).instantiate_identity(tcx);
     for obligation in elaborate_predicates_with_span(tcx, predicates.into_iter()) {
         let kind = obligation.predicate.kind();
         if let ty::PredicateKind::Clause(ty::Clause::Trait(trait_pred)) = kind.skip_binder()
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/method_chain.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/method_chain.rs
index ba9ee57d4..1174efdbf 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/method_chain.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/method_chain.rs
@@ -1,5 +1,7 @@
 use crate::infer::InferCtxt;
 
+use rustc_infer::infer::ObligationEmittingRelation;
+use rustc_infer::traits::PredicateObligations;
 use rustc_middle::ty::error::TypeError;
 use rustc_middle::ty::relate::{self, Relate, RelateResult, TypeRelation};
 use rustc_middle::ty::{self, Ty, TyCtxt};
@@ -88,3 +90,16 @@ impl<'a, 'tcx> TypeRelation<'tcx> for CollectAllMismatches<'a, 'tcx> {
         Ok(a.rebind(self.relate(a.skip_binder(), b.skip_binder())?))
     }
 }
+
+impl<'tcx> ObligationEmittingRelation<'tcx> for CollectAllMismatches<'_, 'tcx> {
+    fn register_obligations(&mut self, _obligations: PredicateObligations<'tcx>) {
+        // FIXME(deferred_projection_equality)
+    }
+
+    fn register_predicates(
+        &mut self,
+        _obligations: impl IntoIterator<Item: ty::ToPredicate<'tcx>>,
+    ) {
+        // FIXME(deferred_projection_equality)
+    }
+}
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs
index 52971486c..a844a1494 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs
@@ -38,7 +38,7 @@ use rustc_middle::ty::fold::{TypeFolder, TypeSuperFoldable};
 use rustc_middle::ty::print::{with_forced_trimmed_paths, FmtPrinter, Print};
 use rustc_middle::ty::{
     self, SubtypePredicate, ToPolyTraitRef, ToPredicate, TraitRef, Ty, TyCtxt, TypeFoldable,
-    TypeVisitable,
+    TypeVisitable, TypeVisitableExt,
 };
 use rustc_session::config::TraitSolver;
 use rustc_session::Limit;
@@ -101,6 +101,18 @@ pub trait InferCtxtExt<'tcx> {
 }
 
 pub trait TypeErrCtxtExt<'tcx> {
+    fn build_overflow_error<T>(
+        &self,
+        predicate: &T,
+        span: Span,
+        suggest_increasing_limit: bool,
+    ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed>
+    where
+        T: fmt::Display
+            + TypeFoldable<TyCtxt<'tcx>>
+            + Print<'tcx, FmtPrinter<'tcx, 'tcx>, Output = FmtPrinter<'tcx, 'tcx>>,
+        <T as Print<'tcx, FmtPrinter<'tcx, 'tcx>>>::Error: std::fmt::Debug;
+
     fn report_overflow_error<T>(
         &self,
         predicate: &T,
@@ -110,7 +122,7 @@ pub trait TypeErrCtxtExt<'tcx> {
     ) -> !
     where
         T: fmt::Display
-            + TypeFoldable<'tcx>
+            + TypeFoldable<TyCtxt<'tcx>>
             + Print<'tcx, FmtPrinter<'tcx, 'tcx>, Output = FmtPrinter<'tcx, 'tcx>>,
         <T as Print<'tcx, FmtPrinter<'tcx, 'tcx>>>::Error: std::fmt::Debug;
 
@@ -480,7 +492,27 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
     ) -> !
     where
         T: fmt::Display
-            + TypeFoldable<'tcx>
+            + TypeFoldable<TyCtxt<'tcx>>
+            + Print<'tcx, FmtPrinter<'tcx, 'tcx>, Output = FmtPrinter<'tcx, 'tcx>>,
+        <T as Print<'tcx, FmtPrinter<'tcx, 'tcx>>>::Error: std::fmt::Debug,
+    {
+        let mut err = self.build_overflow_error(predicate, span, suggest_increasing_limit);
+        mutate(&mut err);
+        err.emit();
+
+        self.tcx.sess.abort_if_errors();
+        bug!();
+    }
+
+    fn build_overflow_error<T>(
+        &self,
+        predicate: &T,
+        span: Span,
+        suggest_increasing_limit: bool,
+    ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed>
+    where
+        T: fmt::Display
+            + TypeFoldable<TyCtxt<'tcx>>
             + Print<'tcx, FmtPrinter<'tcx, 'tcx>, Output = FmtPrinter<'tcx, 'tcx>>,
         <T as Print<'tcx, FmtPrinter<'tcx, 'tcx>>>::Error: std::fmt::Debug,
     {
@@ -511,11 +543,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             self.suggest_new_overflow_limit(&mut err);
         }
 
-        mutate(&mut err);
-
-        err.emit();
-        self.tcx.sess.abort_if_errors();
-        bug!();
+        err
     }
 
     /// Reports that an overflow has occurred and halts compilation. We
@@ -839,14 +867,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             err.note(s.as_str());
                         }
                         if let Some(ref s) = parent_label {
-                            let body = tcx
-                                .hir()
-                                .opt_local_def_id(obligation.cause.body_id)
-                                .unwrap_or_else(|| {
-                                    tcx.hir().body_owner_def_id(hir::BodyId {
-                                        hir_id: obligation.cause.body_id,
-                                    })
-                                });
+                            let body = obligation.cause.body_id;
                             err.span_label(tcx.def_span(body), s);
                         }
 
@@ -934,6 +955,8 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             );
                         }
 
+                        let body_hir_id =
+                            self.tcx.hir().local_def_id_to_hir_id(obligation.cause.body_id);
                         // Try to report a help message
                         if is_fn_trait
                             && let Ok((implemented_kind, params)) = self.type_implements_fn_trait(
@@ -1014,7 +1037,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             if !self.report_similar_impl_candidates(
                                 impl_candidates,
                                 trait_ref,
-                                obligation.cause.body_id,
+                                body_hir_id,
                                 &mut err,
                                 true,
                             ) {
@@ -1050,7 +1073,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                                     self.report_similar_impl_candidates(
                                         impl_candidates,
                                         trait_ref,
-                                        obligation.cause.body_id,
+                                        body_hir_id,
                                         &mut err,
                                         true,
                                     );
@@ -1207,20 +1230,23 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     }
 
                     ty::PredicateKind::WellFormed(ty) => {
-                        if self.tcx.sess.opts.unstable_opts.trait_solver == TraitSolver::Classic {
-                            // WF predicates cannot themselves make
-                            // errors. They can only block due to
-                            // ambiguity; otherwise, they always
-                            // degenerate into other obligations
-                            // (which may fail).
-                            span_bug!(span, "WF predicate not satisfied for {:?}", ty);
-                        } else {
-                            // FIXME: we'll need a better message which takes into account
-                            // which bounds actually failed to hold.
-                            self.tcx.sess.struct_span_err(
-                                span,
-                                &format!("the type `{}` is not well-formed", ty),
-                            )
+                        match self.tcx.sess.opts.unstable_opts.trait_solver {
+                            TraitSolver::Classic => {
+                                // WF predicates cannot themselves make
+                                // errors. They can only block due to
+                                // ambiguity; otherwise, they always
+                                // degenerate into other obligations
+                                // (which may fail).
+                                span_bug!(span, "WF predicate not satisfied for {:?}", ty);
+                            }
+                            TraitSolver::Chalk | TraitSolver::Next => {
+                                // FIXME: we'll need a better message which takes into account
+                                // which bounds actually failed to hold.
+                                self.tcx.sess.struct_span_err(
+                                    span,
+                                    &format!("the type `{}` is not well-formed", ty),
+                                )
+                            }
                         }
                     }
 
@@ -1252,6 +1278,18 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         span,
                         "TypeWellFormedFromEnv predicate should only exist in the environment"
                     ),
+
+                    ty::PredicateKind::AliasEq(..) => span_bug!(
+                        span,
+                        "AliasEq predicate should never be the predicate cause of a SelectionError"
+                    ),
+
+                    ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(ct, ty)) => {
+                        self.tcx.sess.struct_span_err(
+                            span,
+                            &format!("the constant `{}` is not of type `{}`", ct, ty),
+                        )
+                    }
                 }
             }
 
@@ -1598,7 +1636,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 // Eventually I'll need to implement param-env-aware
                 // `Γ₁ ⊦ φ₁ => Γ₂ ⊦ φ₂` logic.
                 let param_env = ty::ParamEnv::empty();
-                if self.can_sub(param_env, error, implication).is_ok() {
+                if self.can_sub(param_env, error, implication) {
                     debug!("error_implies: {:?} -> {:?} -> {:?}", cond, error, implication);
                     return true;
                 }
@@ -1690,7 +1728,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             let (values, err) = if let ty::PredicateKind::Clause(ty::Clause::Projection(data)) =
                 bound_predicate.skip_binder()
             {
-                let data = self.replace_bound_vars_with_fresh_vars(
+                let data = self.instantiate_binder_with_fresh_vars(
                     obligation.cause.span,
                     infer::LateBoundRegionConversionTime::HigherRankedType,
                     bound_predicate.rebind(data),
@@ -1843,10 +1881,14 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 
         with_forced_trimmed_paths! {
             if Some(pred.projection_ty.def_id) == self.tcx.lang_items().fn_once_output() {
+                let fn_kind = self_ty.prefix_string(self.tcx);
+                let item = match self_ty.kind() {
+                    ty::FnDef(def, _) => self.tcx.item_name(*def).to_string(),
+                    _ => self_ty.to_string(),
+                };
                 Some(format!(
-                    "expected `{self_ty}` to be a {fn_kind} that returns `{expected_ty}`, but it \
+                    "expected `{item}` to be a {fn_kind} that returns `{expected_ty}`, but it \
                      returns `{normalized_ty}`",
-                    fn_kind = self_ty.prefix_string(self.tcx)
                 ))
             } else if Some(trait_def_id) == self.tcx.lang_items().future_trait() {
                 Some(format!(
@@ -1896,6 +1938,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 ty::Generator(..) => Some(16),
                 ty::Foreign(..) => Some(17),
                 ty::GeneratorWitness(..) => Some(18),
+                ty::GeneratorWitnessMIR(..) => Some(19),
                 ty::Placeholder(..) | ty::Bound(..) | ty::Infer(..) | ty::Error(_) => None,
             }
         }
@@ -2062,7 +2105,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 // Ignore automatically derived impls and `!Trait` impls.
                 .filter(|&def_id| {
                     self.tcx.impl_polarity(def_id) != ty::ImplPolarity::Negative
-                        || self.tcx.is_builtin_derive(def_id)
+                        || self.tcx.is_automatically_derived(def_id)
                 })
                 .filter_map(|def_id| self.tcx.impl_trait_ref(def_id))
                 .map(ty::EarlyBinder::subst_identity)
@@ -2305,10 +2348,12 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                                 predicate.to_opt_poly_trait_pred().unwrap(),
                             );
                             if impl_candidates.len() < 10 {
+                                let hir =
+                                    self.tcx.hir().local_def_id_to_hir_id(obligation.cause.body_id);
                                 self.report_similar_impl_candidates(
                                     impl_candidates,
                                     trait_ref,
-                                    body_id.map(|id| id.hir_id).unwrap_or(obligation.cause.body_id),
+                                    body_id.map(|id| id.hir_id).unwrap_or(hir),
                                     &mut err,
                                     false,
                                 );
@@ -2395,7 +2440,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             };
                             let mut suggestions = vec![(
                                 path.span.shrink_to_lo(),
-                                format!("<{} as ", self.tcx.type_of(impl_def_id))
+                                format!("<{} as ", self.tcx.type_of(impl_def_id).subst_identity())
                             )];
                             if let Some(generic_arg) = trait_path_segment.args {
                                 let between_span = trait_path_segment.ident.span.between(generic_arg.span_ext);
@@ -2637,8 +2682,8 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             var_map: FxHashMap<Ty<'tcx>, Ty<'tcx>>,
         }
 
-        impl<'a, 'tcx> TypeFolder<'tcx> for ParamToVarFolder<'a, 'tcx> {
-            fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+        impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for ParamToVarFolder<'a, 'tcx> {
+            fn interner(&self) -> TyCtxt<'tcx> {
                 self.infcx.tcx
             }
 
@@ -2828,7 +2873,7 @@ pub struct FindExprBySpan<'hir> {
 }
 
 impl<'hir> FindExprBySpan<'hir> {
-    fn new(span: Span) -> Self {
+    pub fn new(span: Span) -> Self {
         Self { span, result: None, ty_result: None }
     }
 }
@@ -2926,7 +2971,7 @@ impl ArgKind {
 
 struct HasNumericInferVisitor;
 
-impl<'tcx> ty::TypeVisitor<'tcx> for HasNumericInferVisitor {
+impl<'tcx> ty::TypeVisitor<TyCtxt<'tcx>> for HasNumericInferVisitor {
     type BreakTy = ();
 
     fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs
index 18d308f71..b3bf9ad59 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs
@@ -60,7 +60,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
     ) -> Option<(DefId, SubstsRef<'tcx>)> {
         let tcx = self.tcx;
         let param_env = obligation.param_env;
-        let trait_ref = tcx.erase_late_bound_regions(trait_ref);
+        let trait_ref = self.instantiate_binder_with_placeholders(trait_ref);
         let trait_self_ty = trait_ref.self_ty();
 
         let mut self_match_impls = vec![];
@@ -72,7 +72,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 
             let impl_self_ty = impl_trait_ref.self_ty();
 
-            if let Ok(..) = self.can_eq(param_env, trait_self_ty, impl_self_ty) {
+            if self.can_eq(param_env, trait_self_ty, impl_self_ty) {
                 self_match_impls.push((def_id, impl_substs));
 
                 if iter::zip(
@@ -149,10 +149,9 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             .unwrap_or_else(|| (trait_ref.def_id(), trait_ref.skip_binder().substs));
         let trait_ref = trait_ref.skip_binder();
 
-        let mut flags = vec![(
-            sym::ItemContext,
-            self.describe_enclosure(obligation.cause.body_id).map(|s| s.to_owned()),
-        )];
+        let body_hir = self.tcx.hir().local_def_id_to_hir_id(obligation.cause.body_id);
+        let mut flags =
+            vec![(sym::ItemContext, self.describe_enclosure(body_hir).map(|s| s.to_owned()))];
 
         match obligation.cause.code() {
             ObligationCauseCode::BuiltinDerivedObligation(..)
@@ -201,7 +200,10 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             if let Some(def) = self_ty.ty_adt_def() {
                 // We also want to be able to select self's original
                 // signature with no type arguments resolved
-                flags.push((sym::_Self, Some(self.tcx.type_of(def.did()).to_string())));
+                flags.push((
+                    sym::_Self,
+                    Some(self.tcx.type_of(def.did()).subst_identity().to_string()),
+                ));
             }
 
             for param in generics.params.iter() {
@@ -219,7 +221,10 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     if let Some(def) = param_ty.ty_adt_def() {
                         // We also want to be able to select the parameter's
                         // original signature with no type arguments resolved
-                        flags.push((name, Some(self.tcx.type_of(def.did()).to_string())));
+                        flags.push((
+                            name,
+                            Some(self.tcx.type_of(def.did()).subst_identity().to_string()),
+                        ));
                     }
                 }
             }
@@ -252,7 +257,10 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 if let Some(def) = aty.ty_adt_def() {
                     // We also want to be able to select the slice's type's original
                     // signature with no type arguments resolved
-                    flags.push((sym::_Self, Some(format!("[{}]", self.tcx.type_of(def.did())))));
+                    flags.push((
+                        sym::_Self,
+                        Some(format!("[{}]", self.tcx.type_of(def.did()).subst_identity())),
+                    ));
                 }
                 if aty.is_integral() {
                     flags.push((sym::_Self, Some("[{integral}]".to_string())));
@@ -262,7 +270,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             // Arrays give us `[]`, `[{ty}; _]` and `[{ty}; N]`
             if let ty::Array(aty, len) = self_ty.kind() {
                 flags.push((sym::_Self, Some("[]".to_string())));
-                let len = len.kind().try_to_value().and_then(|v| v.try_to_machine_usize(self.tcx));
+                let len = len.kind().try_to_value().and_then(|v| v.try_to_target_usize(self.tcx));
                 flags.push((sym::_Self, Some(format!("[{}; _]", aty))));
                 if let Some(n) = len {
                     flags.push((sym::_Self, Some(format!("[{}; {}]", aty, n))));
@@ -270,7 +278,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 if let Some(def) = aty.ty_adt_def() {
                     // We also want to be able to select the array's type's original
                     // signature with no type arguments resolved
-                    let def_ty = self.tcx.type_of(def.did());
+                    let def_ty = self.tcx.type_of(def.did()).subst_identity();
                     flags.push((sym::_Self, Some(format!("[{def_ty}; _]"))));
                     if let Some(n) = len {
                         flags.push((sym::_Self, Some(format!("[{def_ty}; {n}]"))));
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
index 39e50b2ac..66d74fd05 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
@@ -9,7 +9,6 @@ use crate::infer::InferCtxt;
 use crate::traits::{NormalizeExt, ObligationCtxt};
 
 use hir::def::CtorOf;
-use hir::{Expr, HirId};
 use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_errors::{
@@ -20,8 +19,10 @@ use rustc_hir as hir;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
 use rustc_hir::intravisit::Visitor;
+use rustc_hir::is_range_literal;
 use rustc_hir::lang_items::LangItem;
 use rustc_hir::{AsyncGeneratorKind, GeneratorKind, Node};
+use rustc_hir::{Expr, HirId};
 use rustc_infer::infer::error_reporting::TypeErrCtxt;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
 use rustc_infer::infer::{InferOk, LateBoundRegionConversionTime};
@@ -32,8 +33,9 @@ use rustc_middle::ty::{
     self, suggest_arbitrary_trait_bound, suggest_constraining_type_param, AdtKind, DefIdTree,
     GeneratorDiagnosticData, GeneratorInteriorTypeCause, Infer, InferTy, InternalSubsts,
     IsSuggestable, ToPredicate, Ty, TyCtxt, TypeAndMut, TypeFoldable, TypeFolder,
-    TypeSuperFoldable, TypeVisitable, TypeckResults,
+    TypeSuperFoldable, TypeVisitableExt, TypeckResults,
 };
+use rustc_span::def_id::LocalDefId;
 use rustc_span::symbol::{sym, Ident, Symbol};
 use rustc_span::{BytePos, DesugaringKind, ExpnKind, MacroKind, Span, DUMMY_SP};
 use rustc_target::spec::abi;
@@ -80,11 +82,8 @@ impl<'tcx, 'a> GeneratorData<'tcx, 'a> {
                     upvars.iter().find_map(|(upvar_id, upvar)| {
                         let upvar_ty = typeck_results.node_type(*upvar_id);
                         let upvar_ty = infer_context.resolve_vars_if_possible(upvar_ty);
-                        if ty_matches(ty::Binder::dummy(upvar_ty)) {
-                            Some(GeneratorInteriorOrUpvar::Upvar(upvar.span))
-                        } else {
-                            None
-                        }
+                        ty_matches(ty::Binder::dummy(upvar_ty))
+                            .then(|| GeneratorInteriorOrUpvar::Upvar(upvar.span))
                     })
                 })
             }
@@ -97,6 +96,7 @@ impl<'tcx, 'a> GeneratorData<'tcx, 'a> {
     // obligation
     fn get_from_await_ty<F>(
         &self,
+        tcx: TyCtxt<'tcx>,
         visitor: AwaitsVisitor,
         hir: map::Map<'tcx>,
         ty_matches: F,
@@ -132,10 +132,8 @@ impl<'tcx, 'a> GeneratorData<'tcx, 'a> {
                                         .cloned()
                                         .unwrap_or_else(|| {
                                             bug!(
-                                                "node_type: no type for node `{}`",
-                                                ty::tls::with(|tcx| tcx
-                                                    .hir()
-                                                    .node_to_string(await_expr.hir_id))
+                                                "node_type: no type for node {}",
+                                                tcx.hir().node_to_string(await_expr.hir_id)
                                             )
                                         })
                                 },
@@ -179,7 +177,7 @@ pub trait TypeErrCtxtExt<'tcx> {
         err: &mut Diagnostic,
         trait_pred: ty::PolyTraitPredicate<'tcx>,
         associated_item: Option<(&'static str, Ty<'tcx>)>,
-        body_id: hir::HirId,
+        body_id: LocalDefId,
     );
 
     fn suggest_dereferences(
@@ -398,7 +396,7 @@ fn predicate_constraint(generics: &hir::Generics<'_>, pred: ty::Predicate<'_>) -
 /// param for cleaner code.
 fn suggest_restriction<'tcx>(
     tcx: TyCtxt<'tcx>,
-    hir_id: HirId,
+    item_id: LocalDefId,
     hir_generics: &hir::Generics<'tcx>,
     msg: &str,
     err: &mut Diagnostic,
@@ -417,7 +415,6 @@ fn suggest_restriction<'tcx>(
     {
         return;
     }
-    let Some(item_id) = hir_id.as_owner() else { return; };
     let generics = tcx.generics_of(item_id);
     // Given `fn foo(t: impl Trait)` where `Trait` requires assoc type `A`...
     if let Some((param, bound_str, fn_sig)) =
@@ -522,7 +519,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         mut err: &mut Diagnostic,
         trait_pred: ty::PolyTraitPredicate<'tcx>,
         associated_ty: Option<(&'static str, Ty<'tcx>)>,
-        body_id: hir::HirId,
+        mut body_id: LocalDefId,
     ) {
         let trait_pred = self.resolve_numeric_literals_with_default(trait_pred);
 
@@ -535,8 +532,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 
         // FIXME: Add check for trait bound that is already present, particularly `?Sized` so we
         //        don't suggest `T: Sized + ?Sized`.
-        let mut hir_id = body_id;
-        while let Some(node) = self.tcx.hir().find(hir_id) {
+        while let Some(node) = self.tcx.hir().find_by_def_id(body_id) {
             match node {
                 hir::Node::Item(hir::Item {
                     ident,
@@ -547,7 +543,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     // Restricting `Self` for a single method.
                     suggest_restriction(
                         self.tcx,
-                        hir_id,
+                        body_id,
                         &generics,
                         "`Self`",
                         err,
@@ -567,7 +563,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     assert!(param_ty);
                     // Restricting `Self` for a single method.
                     suggest_restriction(
-                        self.tcx, hir_id, &generics, "`Self`", err, None, projection, trait_pred,
+                        self.tcx, body_id, &generics, "`Self`", err, None, projection, trait_pred,
                         None,
                     );
                     return;
@@ -589,7 +585,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     // Missing restriction on associated type of type parameter (unmet projection).
                     suggest_restriction(
                         self.tcx,
-                        hir_id,
+                        body_id,
                         &generics,
                         "the associated type",
                         err,
@@ -609,7 +605,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     // Missing restriction on associated type of type parameter (unmet projection).
                     suggest_restriction(
                         self.tcx,
-                        hir_id,
+                        body_id,
                         &generics,
                         "the associated type",
                         err,
@@ -680,6 +676,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         &param_name,
                         &constraint,
                         Some(trait_pred.def_id()),
+                        None,
                     ) {
                         return;
                     }
@@ -713,8 +710,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 
                 _ => {}
             }
-
-            hir_id = self.tcx.hir().get_parent_item(hir_id).into();
+            body_id = self.tcx.local_parent(body_id);
         }
     }
 
@@ -749,10 +745,11 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             let real_ty = real_trait_pred.self_ty();
             // We `erase_late_bound_regions` here because `make_subregion` does not handle
             // `ReLateBound`, and we don't particularly care about the regions.
-            if self
-                .can_eq(obligation.param_env, self.tcx.erase_late_bound_regions(real_ty), arg_ty)
-                .is_err()
-            {
+            if !self.can_eq(
+                obligation.param_env,
+                self.tcx.erase_late_bound_regions(real_ty),
+                arg_ty,
+            ) {
                 continue;
             }
 
@@ -770,15 +767,13 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             obligation.param_env,
                             real_trait_pred_and_ty,
                         );
-                        if obligations
+                        let may_hold = obligations
                             .iter()
                             .chain([&obligation])
                             .all(|obligation| self.predicate_may_hold(obligation))
-                        {
-                            Some(steps)
-                        } else {
-                            None
-                        }
+                            .then_some(steps);
+
+                        may_hold
                     })
                 {
                     if steps > 0 {
@@ -899,14 +894,15 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             return false;
         }
 
-        let self_ty = self.replace_bound_vars_with_fresh_vars(
+        let self_ty = self.instantiate_binder_with_fresh_vars(
             DUMMY_SP,
             LateBoundRegionConversionTime::FnCall,
             trait_pred.self_ty(),
         );
 
+        let body_hir_id = self.tcx.hir().local_def_id_to_hir_id(obligation.cause.body_id);
         let Some((def_id_or_name, output, inputs)) = self.extract_callable_info(
-            obligation.cause.body_id,
+            body_hir_id,
             obligation.param_env,
             self_ty,
         ) else { return false; };
@@ -931,7 +927,10 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 DefKind::Ctor(CtorOf::Variant, _) => {
                     "use parentheses to construct this tuple variant".to_string()
                 }
-                kind => format!("use parentheses to call this {}", kind.descr(def_id)),
+                kind => format!(
+                    "use parentheses to call this {}",
+                    self.tcx.def_kind_descr(kind, def_id)
+                ),
             },
             DefIdOrName::Name(name) => format!("use parentheses to call this {name}"),
         };
@@ -1004,8 +1003,9 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             span.remove_mark();
         }
         let mut expr_finder = FindExprBySpan::new(span);
-        let Some(hir::Node::Expr(body)) = self.tcx.hir().find(obligation.cause.body_id) else { return; };
-        expr_finder.visit_expr(&body);
+        let Some(body_id) = self.tcx.hir().maybe_body_owned_by(obligation.cause.body_id) else { return; };
+        let body = self.tcx.hir().body(body_id);
+        expr_finder.visit_expr(body.value);
         let Some(expr) = expr_finder.result else { return; };
         let Some(typeck) = &self.typeck_results else { return; };
         let Some(ty) = typeck.expr_ty_adjusted_opt(expr) else { return; };
@@ -1059,9 +1059,8 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         trait_pred: ty::PolyTraitPredicate<'tcx>,
     ) -> bool {
         let self_ty = self.resolve_vars_if_possible(trait_pred.self_ty());
-        let ty = self.tcx.erase_late_bound_regions(self_ty);
-        let owner = self.tcx.hir().get_parent_item(obligation.cause.body_id);
-        let Some(generics) = self.tcx.hir().get_generics(owner.def_id) else { return false };
+        let ty = self.instantiate_binder_with_placeholders(self_ty);
+        let Some(generics) = self.tcx.hir().get_generics(obligation.cause.body_id) else { return false };
         let ty::Ref(_, inner_ty, hir::Mutability::Not) = ty.kind() else { return false };
         let ty::Param(param) = inner_ty.kind() else { return false };
         let ObligationCauseCode::FunctionArgumentObligation { arg_hir_id, .. } = obligation.cause.code() else { return false };
@@ -1088,6 +1087,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     param.name.as_str(),
                     "Clone",
                     Some(clone_trait),
+                    None,
                 );
             }
             err.span_suggestion_verbose(
@@ -1104,6 +1104,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
     /// Extracts information about a callable type for diagnostics. This is a
     /// heuristic -- it doesn't necessarily mean that a type is always callable,
     /// because the callable type must also be well-formed to be called.
+    // FIXME(vincenzopalazzo): move the HirId to a LocalDefId
     fn extract_callable_info(
         &self,
         hir_id: HirId,
@@ -1189,7 +1190,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             }
         }) else { return None; };
 
-        let output = self.replace_bound_vars_with_fresh_vars(
+        let output = self.instantiate_binder_with_fresh_vars(
             DUMMY_SP,
             LateBoundRegionConversionTime::FnCall,
             output,
@@ -1198,7 +1199,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             .skip_binder()
             .iter()
             .map(|ty| {
-                self.replace_bound_vars_with_fresh_vars(
+                self.instantiate_binder_with_fresh_vars(
                     DUMMY_SP,
                     LateBoundRegionConversionTime::FnCall,
                     inputs.rebind(*ty),
@@ -1348,14 +1349,41 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             Applicability::MaybeIncorrect,
                         );
                     } else {
+                        // Issue #104961, we need to add parentheses properly for compond expressions
+                        // for example, `x.starts_with("hi".to_string() + "you")`
+                        // should be `x.starts_with(&("hi".to_string() + "you"))`
+                        let Some(body_id) = self.tcx.hir().maybe_body_owned_by(obligation.cause.body_id) else { return false; };
+                        let body = self.tcx.hir().body(body_id);
+                        let mut expr_finder = FindExprBySpan::new(span);
+                        expr_finder.visit_expr(body.value);
+                        let Some(expr) = expr_finder.result else { return false; };
+                        let needs_parens = match expr.kind {
+                            // parenthesize if needed (Issue #46756)
+                            hir::ExprKind::Cast(_, _) | hir::ExprKind::Binary(_, _, _) => true,
+                            // parenthesize borrows of range literals (Issue #54505)
+                            _ if is_range_literal(expr) => true,
+                            _ => false,
+                        };
+
                         let is_mut = mut_ref_self_ty_satisfies_pred || ref_inner_ty_mut;
-                        err.span_suggestion_verbose(
-                            span.shrink_to_lo(),
-                            &format!(
-                                "consider{} borrowing here",
-                                if is_mut { " mutably" } else { "" }
-                            ),
-                            format!("&{}", if is_mut { "mut " } else { "" }),
+                        let span = if needs_parens { span } else { span.shrink_to_lo() };
+                        let sugg_prefix = format!("&{}", if is_mut { "mut " } else { "" });
+                        let sugg_msg = &format!(
+                            "consider{} borrowing here",
+                            if is_mut { " mutably" } else { "" }
+                        );
+
+                        let suggestions = if !needs_parens {
+                            vec![(span.shrink_to_lo(), format!("{}", sugg_prefix))]
+                        } else {
+                            vec![
+                                (span.shrink_to_lo(), format!("{}(", sugg_prefix)),
+                                (span.shrink_to_hi(), ")".to_string()),
+                            ]
+                        };
+                        err.multipart_suggestion_verbose(
+                            sugg_msg,
+                            suggestions,
                             Applicability::MaybeIncorrect,
                         );
                     }
@@ -1429,10 +1457,11 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             span.remove_mark();
         }
         let mut expr_finder = super::FindExprBySpan::new(span);
-        let Some(hir::Node::Expr(body)) = self.tcx.hir().find(obligation.cause.body_id) else {
+        let Some(body_id) = self.tcx.hir().maybe_body_owned_by(obligation.cause.body_id) else {
             return false;
         };
-        expr_finder.visit_expr(&body);
+        let body = self.tcx.hir().body(body_id);
+        expr_finder.visit_expr(body.value);
         let mut maybe_suggest = |suggested_ty, count, suggestions| {
             // Remapping bound vars here
             let trait_pred_and_suggested_ty =
@@ -1670,8 +1699,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         trait_pred: ty::PolyTraitPredicate<'tcx>,
     ) -> bool {
         let hir = self.tcx.hir();
-        let parent_node = hir.parent_id(obligation.cause.body_id);
-        let node = hir.find(parent_node);
+        let node = hir.find_by_def_id(obligation.cause.body_id);
         if let Some(hir::Node::Item(hir::Item { kind: hir::ItemKind::Fn(sig, _, body_id), .. })) = node
             && let hir::ExprKind::Block(blk, _) = &hir.body(*body_id).value.kind
             && sig.decl.output.span().overlaps(span)
@@ -1707,8 +1735,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 
     fn return_type_span(&self, obligation: &PredicateObligation<'tcx>) -> Option<Span> {
         let hir = self.tcx.hir();
-        let parent_node = hir.parent_id(obligation.cause.body_id);
-        let Some(hir::Node::Item(hir::Item { kind: hir::ItemKind::Fn(sig, ..), .. })) = hir.find(parent_node) else {
+        let Some(hir::Node::Item(hir::Item { kind: hir::ItemKind::Fn(sig, ..), .. })) = hir.find_by_def_id(obligation.cause.body_id) else {
             return None;
         };
 
@@ -1732,8 +1759,8 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         }
 
         let hir = self.tcx.hir();
-        let fn_hir_id = hir.parent_id(obligation.cause.body_id);
-        let node = hir.find(fn_hir_id);
+        let fn_hir_id = hir.local_def_id_to_hir_id(obligation.cause.body_id);
+        let node = hir.find_by_def_id(obligation.cause.body_id);
         let Some(hir::Node::Item(hir::Item {
             kind: hir::ItemKind::Fn(sig, _, body_id),
             ..
@@ -1749,7 +1776,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 // If the `dyn Trait` is not object safe, do not suggest `Box<dyn Trait>`.
                 predicates
                     .principal_def_id()
-                    .map_or(true, |def_id| self.tcx.object_safety_violations(def_id).is_empty())
+                    .map_or(true, |def_id| self.tcx.check_is_object_safe(def_id))
             }
             // We only want to suggest `impl Trait` to `dyn Trait`s.
             // For example, `fn foo() -> str` needs to be filtered out.
@@ -1806,7 +1833,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 
         match liberated_sig.output().kind() {
             ty::Dynamic(predicates, _, ty::Dyn) => {
-                let cause = ObligationCause::misc(ret_ty.span, fn_hir_id);
+                let cause = ObligationCause::misc(ret_ty.span, obligation.cause.body_id);
                 let param_env = ty::ParamEnv::empty();
 
                 if !only_never_return {
@@ -1944,8 +1971,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         }
 
         let hir = self.tcx.hir();
-        let parent_node = hir.parent_id(obligation.cause.body_id);
-        let node = hir.find(parent_node);
+        let node = hir.find_by_def_id(obligation.cause.body_id);
         if let Some(hir::Node::Item(hir::Item { kind: hir::ItemKind::Fn(_, _, body_id), .. })) =
             node
         {
@@ -1989,7 +2015,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             let sig = match inputs.kind() {
                 ty::Tuple(inputs) if infcx.tcx.is_fn_trait(trait_ref.def_id()) => {
                     infcx.tcx.mk_fn_sig(
-                        inputs.iter(),
+                        *inputs,
                         infcx.next_ty_var(TypeVariableOrigin {
                             span: DUMMY_SP,
                             kind: TypeVariableOriginKind::MiscVariable,
@@ -2000,7 +2026,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     )
                 }
                 _ => infcx.tcx.mk_fn_sig(
-                    std::iter::once(inputs),
+                    [inputs],
                     infcx.next_ty_var(TypeVariableOrigin {
                         span: DUMMY_SP,
                         kind: TypeVariableOriginKind::MiscVariable,
@@ -2116,7 +2142,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 err.note(&format!(
                     "{}s cannot be accessed directly on a `trait`, they can only be \
                         accessed through a specific `impl`",
-                    assoc_item.kind.as_def_kind().descr(item_def_id)
+                    self.tcx.def_kind_descr(assoc_item.kind.as_def_kind(), item_def_id)
                 ));
                 err.span_suggestion(
                     span,
@@ -2225,7 +2251,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     );
 
                     match *ty.kind() {
-                        ty::Generator(did, ..) => {
+                        ty::Generator(did, ..) | ty::GeneratorWitnessMIR(did, _) => {
                             generator = generator.or(Some(did));
                             outer_generator = Some(did);
                         }
@@ -2255,7 +2281,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     );
 
                     match *ty.kind() {
-                        ty::Generator(did, ..) => {
+                        ty::Generator(did, ..) | ty::GeneratorWitnessMIR(did, ..) => {
                             generator = generator.or(Some(did));
                             outer_generator = Some(did);
                         }
@@ -2344,9 +2370,14 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             _ => return false,
         };
 
+        let generator_within_in_progress_typeck = match &self.typeck_results {
+            Some(t) => t.hir_owner.to_def_id() == generator_did_root,
+            _ => false,
+        };
+
         let mut interior_or_upvar_span = None;
 
-        let from_awaited_ty = generator_data.get_from_await_ty(visitor, hir, ty_matches);
+        let from_awaited_ty = generator_data.get_from_await_ty(self.tcx, visitor, hir, ty_matches);
         debug!(?from_awaited_ty);
 
         // The generator interior types share the same binders
@@ -2363,6 +2394,35 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 *span,
                 Some((*scope_span, *yield_span, *expr, from_awaited_ty)),
             ));
+
+            if interior_or_upvar_span.is_none() && generator_data.is_foreign() {
+                interior_or_upvar_span = Some(GeneratorInteriorOrUpvar::Interior(*span, None));
+            }
+        } else if self.tcx.sess.opts.unstable_opts.drop_tracking_mir
+            // Avoid disclosing internal information to downstream crates.
+            && generator_did.is_local()
+            // Try to avoid cycles.
+            && !generator_within_in_progress_typeck
+        {
+            let generator_info = &self.tcx.mir_generator_witnesses(generator_did);
+            debug!(?generator_info);
+
+            'find_source: for (variant, source_info) in
+                generator_info.variant_fields.iter().zip(&generator_info.variant_source_info)
+            {
+                debug!(?variant);
+                for &local in variant {
+                    let decl = &generator_info.field_tys[local];
+                    debug!(?decl);
+                    if ty_matches(ty::Binder::dummy(decl.ty)) && !decl.ignore_for_traits {
+                        interior_or_upvar_span = Some(GeneratorInteriorOrUpvar::Interior(
+                            decl.source_info.span,
+                            Some((None, source_info.span, None, from_awaited_ty)),
+                        ));
+                        break 'find_source;
+                    }
+                }
+            }
         }
 
         if interior_or_upvar_span.is_none() {
@@ -2727,7 +2787,13 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             _ => true,
                         };
                     if ident.span.is_visible(sm) && !ident.span.overlaps(span) && !same_line {
-                        multispan.push_span_label(ident.span, "required by a bound in this");
+                        multispan.push_span_label(
+                            ident.span,
+                            format!(
+                                "required by a bound in this {}",
+                                tcx.def_kind(item_def_id).descr(item_def_id)
+                            ),
+                        );
                     }
                 }
                 let descr = format!("required by a bound in `{item_name}`");
@@ -3011,6 +3077,20 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                                 }
                                 err.note(msg.trim_end_matches(", "))
                             }
+                            ty::GeneratorWitnessMIR(def_id, substs) => {
+                                use std::fmt::Write;
+
+                                // FIXME: this is kind of an unusual format for rustc, can we make it more clear?
+                                // Maybe we should just remove this note altogether?
+                                // FIXME: only print types which don't meet the trait requirement
+                                let mut msg =
+                                    "required because it captures the following types: ".to_owned();
+                                for bty in tcx.generator_hidden_types(*def_id) {
+                                    let ty = bty.subst(tcx, substs);
+                                    write!(msg, "`{}`, ", ty).unwrap();
+                                }
+                                err.note(msg.trim_end_matches(", "))
+                            }
                             ty::Generator(def_id, _, _) => {
                                 let sp = self.tcx.def_span(def_id);
 
@@ -3027,6 +3107,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                                 self.tcx.def_span(def_id),
                                 "required because it's used within this closure",
                             ),
+                            ty::Str => err.note("`str` is considered to contain a `[u8]` slice for auto trait purposes"),
                             _ => err.note(&msg),
                         };
                     }
@@ -3072,7 +3153,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     parent_trait_pred.print_modifiers_and_trait_path()
                 );
                 let mut is_auto_trait = false;
-                match self.tcx.hir().get_if_local(data.impl_def_id) {
+                match self.tcx.hir().get_if_local(data.impl_or_alias_def_id) {
                     Some(Node::Item(hir::Item {
                         kind: hir::ItemKind::Trait(is_auto, ..),
                         ident,
@@ -3283,12 +3364,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         trait_pred: ty::PolyTraitPredicate<'tcx>,
         span: Span,
     ) {
-        let body_hir_id = obligation.cause.body_id;
-        let item_id = self.tcx.hir().parent_id(body_hir_id);
-
-        if let Some(body_id) =
-            self.tcx.hir().maybe_body_owned_by(self.tcx.hir().local_def_id(item_id))
-        {
+        if let Some(body_id) = self.tcx.hir().maybe_body_owned_by(obligation.cause.body_id) {
             let body = self.tcx.hir().body(body_id);
             if let Some(hir::GeneratorKind::Async(_)) = body.generator_kind {
                 let future_trait = self.tcx.require_lang_item(LangItem::Future, None);
@@ -3459,7 +3535,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         {
             if let hir::Expr { kind: hir::ExprKind::Block(..), .. } = expr {
                 let expr = expr.peel_blocks();
-                let ty = typeck_results.expr_ty_adjusted_opt(expr).unwrap_or(tcx.ty_error());
+                let ty = typeck_results.expr_ty_adjusted_opt(expr).unwrap_or(tcx.ty_error_misc());
                 let span = expr.span;
                 if Some(span) != err.span.primary_span() {
                     err.span_label(
@@ -3502,7 +3578,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 {
                     type_diffs = vec![
                         Sorts(ty::error::ExpectedFound {
-                            expected: self.tcx.mk_ty(ty::Alias(ty::Projection, where_pred.skip_binder().projection_ty)),
+                            expected: self.tcx.mk_alias(ty::Projection, where_pred.skip_binder().projection_ty),
                             found,
                         }),
                     ];
@@ -3562,7 +3638,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         let mut assocs = vec![];
         let mut expr = expr;
         let mut prev_ty = self.resolve_vars_if_possible(
-            typeck_results.expr_ty_adjusted_opt(expr).unwrap_or(tcx.ty_error()),
+            typeck_results.expr_ty_adjusted_opt(expr).unwrap_or(tcx.ty_error_misc()),
         );
         while let hir::ExprKind::MethodCall(_path_segment, rcvr_expr, _args, span) = expr.kind {
             // Point at every method call in the chain with the resulting type.
@@ -3573,7 +3649,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 self.probe_assoc_types_at_expr(&type_diffs, span, prev_ty, expr.hir_id, param_env);
             assocs.push(assocs_in_this_method);
             prev_ty = self.resolve_vars_if_possible(
-                typeck_results.expr_ty_adjusted_opt(expr).unwrap_or(tcx.ty_error()),
+                typeck_results.expr_ty_adjusted_opt(expr).unwrap_or(tcx.ty_error_misc()),
             );
 
             if let hir::ExprKind::Path(hir::QPath::Resolved(None, path)) = expr.kind
@@ -3591,7 +3667,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 if let hir::Node::Param(param) = parent {
                     // ...and it is a an fn argument.
                     let prev_ty = self.resolve_vars_if_possible(
-                        typeck_results.node_type_opt(param.hir_id).unwrap_or(tcx.ty_error()),
+                        typeck_results.node_type_opt(param.hir_id).unwrap_or(tcx.ty_error_misc()),
                     );
                     let assocs_in_this_method = self.probe_assoc_types_at_expr(&type_diffs, param.ty_span, prev_ty, param.hir_id, param_env);
                     if assocs_in_this_method.iter().any(|a| a.is_some()) {
@@ -3620,7 +3696,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     let Some((span, (assoc, ty))) = entry else { continue; };
                     if primary_spans.is_empty() || type_diffs.iter().any(|diff| {
                         let Sorts(expected_found) = diff else { return false; };
-                        self.can_eq(param_env, expected_found.found, ty).is_ok()
+                        self.can_eq(param_env, expected_found.found, ty)
                     }) {
                         // FIXME: this doesn't quite work for `Iterator::collect`
                         // because we have `Vec<i32>` and `()`, but we'd want `i32`
@@ -3647,10 +3723,10 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         let ty_str = with_forced_trimmed_paths!(self.ty_to_string(ty));
 
                         let assoc = with_forced_trimmed_paths!(self.tcx.def_path_str(assoc));
-                        if self.can_eq(param_env, ty, *prev_ty).is_err() {
+                        if !self.can_eq(param_env, ty, *prev_ty) {
                             if type_diffs.iter().any(|diff| {
                                 let Sorts(expected_found) = diff else { return false; };
-                                self.can_eq(param_env, expected_found.found, ty).is_ok()
+                                self.can_eq(param_env, expected_found.found, ty)
                             }) {
                                 primary_spans.push(span);
                             }
@@ -3727,9 +3803,14 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     term: ty_var.into(),
                 },
             )));
+            let body_def_id = self.tcx.hir().enclosing_body_owner(body_id);
             // Add `<ExprTy as Iterator>::Item = _` obligation.
             ocx.register_obligation(Obligation::misc(
-                self.tcx, span, body_id, param_env, projection,
+                self.tcx,
+                span,
+                body_def_id,
+                param_env,
+                projection,
             ));
             if ocx.select_where_possible().is_empty() {
                 // `ty_var` now holds the type that `Item` is for `ExprTy`.
@@ -3758,13 +3839,13 @@ fn hint_missing_borrow<'tcx>(
     err: &mut Diagnostic,
 ) {
     let found_args = match found.kind() {
-        ty::FnPtr(f) => f.inputs().skip_binder().iter(),
+        ty::FnPtr(f) => infcx.instantiate_binder_with_placeholders(*f).inputs().iter(),
         kind => {
             span_bug!(span, "found was converted to a FnPtr above but is now {:?}", kind)
         }
     };
     let expected_args = match expected.kind() {
-        ty::FnPtr(f) => f.inputs().skip_binder().iter(),
+        ty::FnPtr(f) => infcx.instantiate_binder_with_placeholders(*f).inputs().iter(),
         kind => {
             span_bug!(span, "expected was converted to a FnPtr above but is now {:?}", kind)
         }
@@ -3775,12 +3856,12 @@ fn hint_missing_borrow<'tcx>(
 
     let args = fn_decl.inputs.iter().map(|ty| ty);
 
-    fn get_deref_type_and_refs(mut ty: Ty<'_>) -> (Ty<'_>, usize) {
-        let mut refs = 0;
+    fn get_deref_type_and_refs(mut ty: Ty<'_>) -> (Ty<'_>, Vec<hir::Mutability>) {
+        let mut refs = vec![];
 
-        while let ty::Ref(_, new_ty, _) = ty.kind() {
+        while let ty::Ref(_, new_ty, mutbl) = ty.kind() {
             ty = *new_ty;
-            refs += 1;
+            refs.push(*mutbl);
         }
 
         (ty, refs)
@@ -3793,12 +3874,22 @@ fn hint_missing_borrow<'tcx>(
         let (found_ty, found_refs) = get_deref_type_and_refs(*found_arg);
         let (expected_ty, expected_refs) = get_deref_type_and_refs(*expected_arg);
 
-        if infcx.can_eq(param_env, found_ty, expected_ty).is_ok() {
-            if found_refs < expected_refs {
-                to_borrow.push((arg.span.shrink_to_lo(), "&".repeat(expected_refs - found_refs)));
-            } else if found_refs > expected_refs {
+        if infcx.can_eq(param_env, found_ty, expected_ty) {
+            // FIXME: This could handle more exotic cases like mutability mismatches too!
+            if found_refs.len() < expected_refs.len()
+                && found_refs[..] == expected_refs[expected_refs.len() - found_refs.len()..]
+            {
+                to_borrow.push((
+                    arg.span.shrink_to_lo(),
+                    expected_refs[..expected_refs.len() - found_refs.len()]
+                        .iter()
+                        .map(|mutbl| format!("&{}", mutbl.prefix_str()))
+                        .collect::<Vec<_>>()
+                        .join(""),
+                ));
+            } else if found_refs.len() > expected_refs.len() {
                 let mut span = arg.span.shrink_to_lo();
-                let mut left = found_refs - expected_refs;
+                let mut left = found_refs.len() - expected_refs.len();
                 let mut ty = arg;
                 while let hir::TyKind::Ref(_, mut_ty) = &ty.kind && left > 0 {
                     span = span.with_hi(mut_ty.ty.span.lo());
@@ -3996,7 +4087,7 @@ struct ReplaceImplTraitFolder<'tcx> {
     replace_ty: Ty<'tcx>,
 }
 
-impl<'tcx> TypeFolder<'tcx> for ReplaceImplTraitFolder<'tcx> {
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ReplaceImplTraitFolder<'tcx> {
     fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
         if let ty::Param(ty::ParamTy { index, .. }) = t.kind() {
             if self.param.index == *index {
@@ -4006,7 +4097,7 @@ impl<'tcx> TypeFolder<'tcx> for ReplaceImplTraitFolder<'tcx> {
         t.super_fold_with(self)
     }
 
-    fn tcx(&self) -> TyCtxt<'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 }
diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs
index 76a755ed9..944436ab8 100644
--- a/compiler/rustc_trait_selection/src/traits/fulfill.rs
+++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs
@@ -1,5 +1,4 @@
 use crate::infer::{InferCtxt, TyOrConstInferVar};
-use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::obligation_forest::ProcessResult;
 use rustc_data_structures::obligation_forest::{Error, ForestObligation, Outcome};
 use rustc_data_structures::obligation_forest::{ObligationForest, ObligationProcessor};
@@ -9,7 +8,7 @@ use rustc_middle::mir::interpret::ErrorHandled;
 use rustc_middle::ty::abstract_const::NotConstEvaluatable;
 use rustc_middle::ty::error::{ExpectedFound, TypeError};
 use rustc_middle::ty::subst::SubstsRef;
-use rustc_middle::ty::{self, Binder, Const, TypeVisitable};
+use rustc_middle::ty::{self, Binder, Const, TypeVisitableExt};
 use std::marker::PhantomData;
 
 use super::const_evaluatable;
@@ -54,8 +53,6 @@ pub struct FulfillmentContext<'tcx> {
     // fulfillment context.
     predicates: ObligationForest<PendingPredicateObligation<'tcx>>,
 
-    relationships: FxHashMap<ty::TyVid, ty::FoundRelationships>,
-
     // Is it OK to register obligations into this infcx inside
     // an infcx snapshot?
     //
@@ -85,19 +82,11 @@ static_assert_size!(PendingPredicateObligation<'_>, 72);
 impl<'a, 'tcx> FulfillmentContext<'tcx> {
     /// Creates a new fulfillment context.
     pub(super) fn new() -> FulfillmentContext<'tcx> {
-        FulfillmentContext {
-            predicates: ObligationForest::new(),
-            relationships: FxHashMap::default(),
-            usable_in_snapshot: false,
-        }
+        FulfillmentContext { predicates: ObligationForest::new(), usable_in_snapshot: false }
     }
 
     pub(super) fn new_in_snapshot() -> FulfillmentContext<'tcx> {
-        FulfillmentContext {
-            predicates: ObligationForest::new(),
-            relationships: FxHashMap::default(),
-            usable_in_snapshot: true,
-        }
+        FulfillmentContext { predicates: ObligationForest::new(), usable_in_snapshot: true }
     }
 
     /// Attempts to select obligations using `selcx`.
@@ -139,20 +128,11 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentContext<'tcx> {
 
         assert!(!infcx.is_in_snapshot() || self.usable_in_snapshot);
 
-        super::relationships::update(self, infcx, &obligation);
-
         self.predicates
             .register_obligation(PendingPredicateObligation { obligation, stalled_on: vec![] });
     }
 
-    fn select_all_or_error(&mut self, infcx: &InferCtxt<'tcx>) -> Vec<FulfillmentError<'tcx>> {
-        {
-            let errors = self.select_where_possible(infcx);
-            if !errors.is_empty() {
-                return errors;
-            }
-        }
-
+    fn collect_remaining_errors(&mut self) -> Vec<FulfillmentError<'tcx>> {
         self.predicates.to_errors(CodeAmbiguity).into_iter().map(to_fulfillment_error).collect()
     }
 
@@ -161,12 +141,57 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentContext<'tcx> {
         self.select(selcx)
     }
 
-    fn pending_obligations(&self) -> Vec<PredicateObligation<'tcx>> {
-        self.predicates.map_pending_obligations(|o| o.obligation.clone())
+    fn drain_unstalled_obligations(
+        &mut self,
+        infcx: &InferCtxt<'tcx>,
+    ) -> Vec<PredicateObligation<'tcx>> {
+        let mut processor = DrainProcessor { removed_predicates: Vec::new(), infcx };
+        let outcome: Outcome<_, _> = self.predicates.process_obligations(&mut processor);
+        assert!(outcome.errors.is_empty());
+        return processor.removed_predicates;
+
+        struct DrainProcessor<'a, 'tcx> {
+            infcx: &'a InferCtxt<'tcx>,
+            removed_predicates: Vec<PredicateObligation<'tcx>>,
+        }
+
+        impl<'tcx> ObligationProcessor for DrainProcessor<'_, 'tcx> {
+            type Obligation = PendingPredicateObligation<'tcx>;
+            type Error = !;
+            type OUT = Outcome<Self::Obligation, Self::Error>;
+
+            fn needs_process_obligation(&self, pending_obligation: &Self::Obligation) -> bool {
+                pending_obligation
+                    .stalled_on
+                    .iter()
+                    .any(|&var| self.infcx.ty_or_const_infer_var_changed(var))
+            }
+
+            fn process_obligation(
+                &mut self,
+                pending_obligation: &mut PendingPredicateObligation<'tcx>,
+            ) -> ProcessResult<PendingPredicateObligation<'tcx>, !> {
+                assert!(self.needs_process_obligation(pending_obligation));
+                self.removed_predicates.push(pending_obligation.obligation.clone());
+                ProcessResult::Changed(vec![])
+            }
+
+            fn process_backedge<'c, I>(
+                &mut self,
+                cycle: I,
+                _marker: PhantomData<&'c PendingPredicateObligation<'tcx>>,
+            ) -> Result<(), !>
+            where
+                I: Clone + Iterator<Item = &'c PendingPredicateObligation<'tcx>>,
+            {
+                self.removed_predicates.extend(cycle.map(|c| c.obligation.clone()));
+                Ok(())
+            }
+        }
     }
 
-    fn relationships(&mut self) -> &mut FxHashMap<ty::TyVid, ty::FoundRelationships> {
-        &mut self.relationships
+    fn pending_obligations(&self) -> Vec<PredicateObligation<'tcx>> {
+        self.predicates.map_pending_obligations(|o| o.obligation.clone())
     }
 }
 
@@ -187,36 +212,44 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
 
     /// Identifies whether a predicate obligation needs processing.
     ///
-    /// This is always inlined, despite its size, because it has a single
-    /// callsite and it is called *very* frequently.
+    /// This is always inlined because it has a single callsite and it is
+    /// called *very* frequently. Be careful modifying this code! Several
+    /// compile-time benchmarks are very sensitive to even small changes.
     #[inline(always)]
     fn needs_process_obligation(&self, pending_obligation: &Self::Obligation) -> bool {
         // If we were stalled on some unresolved variables, first check whether
         // any of them have been resolved; if not, don't bother doing more work
         // yet.
-        match pending_obligation.stalled_on.len() {
-            // Match arms are in order of frequency, which matters because this
-            // code is so hot. 1 and 0 dominate; 2+ is fairly rare.
-            1 => {
-                let infer_var = pending_obligation.stalled_on[0];
-                self.selcx.infcx.ty_or_const_infer_var_changed(infer_var)
-            }
-            0 => {
-                // In this case we haven't changed, but wish to make a change.
-                true
-            }
-            _ => {
-                // This `for` loop was once a call to `all()`, but this lower-level
-                // form was a perf win. See #64545 for details.
-                (|| {
-                    for &infer_var in &pending_obligation.stalled_on {
-                        if self.selcx.infcx.ty_or_const_infer_var_changed(infer_var) {
-                            return true;
-                        }
+        let stalled_on = &pending_obligation.stalled_on;
+        match stalled_on.len() {
+            // This case is the hottest most of the time, being hit up to 99%
+            // of the time. `keccak` and `cranelift-codegen-0.82.1` are
+            // benchmarks that particularly stress this path.
+            1 => self.selcx.infcx.ty_or_const_infer_var_changed(stalled_on[0]),
+
+            // In this case we haven't changed, but wish to make a change. Note
+            // that this is a special case, and is not equivalent to the `_`
+            // case below, which would return `false` for an empty `stalled_on`
+            // vector.
+            //
+            // This case is usually hit only 1% of the time or less, though it
+            // reaches 20% in `wasmparser-0.101.0`.
+            0 => true,
+
+            // This case is usually hit only 1% of the time or less, though it
+            // reaches 95% in `mime-0.3.16`, 64% in `wast-54.0.0`, and 12% in
+            // `inflate-0.4.5`.
+            //
+            // The obvious way of writing this, with a call to `any()` and no
+            // closure, is currently slower than this version.
+            _ => (|| {
+                for &infer_var in stalled_on {
+                    if self.selcx.infcx.ty_or_const_infer_var_changed(infer_var) {
+                        return true;
                     }
-                    false
-                })()
-            }
+                }
+                false
+            })(),
         }
     }
 
@@ -288,6 +321,7 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
                 }
                 ty::PredicateKind::Clause(ty::Clause::RegionOutlives(_))
                 | ty::PredicateKind::Clause(ty::Clause::TypeOutlives(_))
+                | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
                 | ty::PredicateKind::WellFormed(_)
                 | ty::PredicateKind::ObjectSafe(_)
                 | ty::PredicateKind::ClosureKind(..)
@@ -296,13 +330,16 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
                 | ty::PredicateKind::ConstEvaluatable(..)
                 | ty::PredicateKind::ConstEquate(..) => {
                     let pred =
-                        ty::Binder::dummy(infcx.replace_bound_vars_with_placeholders(binder));
+                        ty::Binder::dummy(infcx.instantiate_binder_with_placeholders(binder));
                     ProcessResult::Changed(mk_pending(vec![obligation.with(infcx.tcx, pred)]))
                 }
                 ty::PredicateKind::Ambiguous => ProcessResult::Unchanged,
                 ty::PredicateKind::TypeWellFormedFromEnv(..) => {
                     bug!("TypeWellFormedFromEnv is only used for Chalk")
                 }
+                ty::PredicateKind::AliasEq(..) => {
+                    bug!("AliasEq is only used for new solver")
+                }
             },
             Some(pred) => match pred {
                 ty::PredicateKind::Clause(ty::Clause::Trait(data)) => {
@@ -344,7 +381,7 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
                 }
 
                 ty::PredicateKind::ObjectSafe(trait_def_id) => {
-                    if !self.selcx.tcx().is_object_safe(trait_def_id) {
+                    if !self.selcx.tcx().check_is_object_safe(trait_def_id) {
                         ProcessResult::Error(CodeSelectionError(Unimplemented))
                     } else {
                         ProcessResult::Changed(vec![])
@@ -569,6 +606,22 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
                 ty::PredicateKind::TypeWellFormedFromEnv(..) => {
                     bug!("TypeWellFormedFromEnv is only used for Chalk")
                 }
+                ty::PredicateKind::AliasEq(..) => {
+                    bug!("AliasEq is only used for new solver")
+                }
+                ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(ct, ty)) => {
+                    match self
+                        .selcx
+                        .infcx
+                        .at(&obligation.cause, obligation.param_env)
+                        .eq(ct.ty(), ty)
+                    {
+                        Ok(inf_ok) => ProcessResult::Changed(mk_pending(inf_ok.into_obligations())),
+                        Err(_) => ProcessResult::Error(FulfillmentErrorCode::CodeSelectionError(
+                            SelectionError::Unimplemented,
+                        )),
+                    }
+                }
             },
         }
     }
diff --git a/compiler/rustc_trait_selection/src/traits/misc.rs b/compiler/rustc_trait_selection/src/traits/misc.rs
index a41a601f2..b94346b09 100644
--- a/compiler/rustc_trait_selection/src/traits/misc.rs
+++ b/compiler/rustc_trait_selection/src/traits/misc.rs
@@ -1,12 +1,15 @@
 //! Miscellaneous type-system utilities that are too small to deserve their own modules.
 
-use crate::traits::{self, ObligationCause};
+use crate::traits::{self, ObligationCause, ObligationCtxt};
 
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_hir as hir;
+use rustc_infer::infer::canonical::Canonical;
 use rustc_infer::infer::{RegionResolutionError, TyCtxtInferExt};
+use rustc_infer::traits::query::NoSolution;
 use rustc_infer::{infer::outlives::env::OutlivesEnvironment, traits::FulfillmentError};
-use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, ParamEnv, Ty, TyCtxt, TypeVisitableExt};
+use rustc_span::DUMMY_SP;
 
 use super::outlives_bounds::InferCtxtExt;
 
@@ -131,3 +134,19 @@ pub fn type_allowed_to_implement_copy<'tcx>(
 
     Ok(())
 }
+
+pub fn check_tys_might_be_eq<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    canonical: Canonical<'tcx, (ParamEnv<'tcx>, Ty<'tcx>, Ty<'tcx>)>,
+) -> Result<(), NoSolution> {
+    let (infcx, (param_env, ty_a, ty_b), _) =
+        tcx.infer_ctxt().build_with_canonical(DUMMY_SP, &canonical);
+    let ocx = ObligationCtxt::new(&infcx);
+
+    let result = ocx.eq(&ObligationCause::dummy(), param_env, ty_a, ty_b);
+    // use `select_where_possible` instead of `select_all_or_error` so that
+    // we don't get errors from obligations being ambiguous.
+    let errors = ocx.select_where_possible();
+
+    if errors.len() > 0 || result.is_err() { Err(NoSolution) } else { Ok(()) }
+}
diff --git a/compiler/rustc_trait_selection/src/traits/mod.rs b/compiler/rustc_trait_selection/src/traits/mod.rs
index f036a311d..4e30108be 100644
--- a/compiler/rustc_trait_selection/src/traits/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/mod.rs
@@ -14,7 +14,6 @@ mod object_safety;
 pub mod outlives_bounds;
 mod project;
 pub mod query;
-pub(crate) mod relationships;
 mod select;
 mod specialize;
 mod structural_match;
@@ -27,12 +26,11 @@ use crate::infer::{InferCtxt, TyCtxtInferExt};
 use crate::traits::error_reporting::TypeErrCtxtExt as _;
 use crate::traits::query::evaluate_obligation::InferCtxtExt as _;
 use rustc_errors::ErrorGuaranteed;
-use rustc_hir as hir;
-use rustc_hir::def_id::DefId;
 use rustc_middle::ty::fold::TypeFoldable;
-use rustc_middle::ty::visit::TypeVisitable;
+use rustc_middle::ty::visit::{TypeVisitable, TypeVisitableExt};
 use rustc_middle::ty::{self, DefIdTree, ToPredicate, Ty, TyCtxt, TypeSuperVisitable};
 use rustc_middle::ty::{InternalSubsts, SubstsRef};
+use rustc_span::def_id::{DefId, CRATE_DEF_ID};
 use rustc_span::Span;
 
 use std::fmt::Debug;
@@ -143,7 +141,7 @@ pub fn type_known_to_meet_bound_modulo_regions<'tcx>(
 fn pred_known_to_hold_modulo_regions<'tcx>(
     infcx: &InferCtxt<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
-    pred: impl ToPredicate<'tcx> + TypeVisitable<'tcx>,
+    pred: impl ToPredicate<'tcx> + TypeVisitable<TyCtxt<'tcx>>,
     span: Span,
 ) -> bool {
     let has_non_region_infer = pred.has_non_region_infer();
@@ -152,29 +150,29 @@ fn pred_known_to_hold_modulo_regions<'tcx>(
         // We can use a dummy node-id here because we won't pay any mind
         // to region obligations that arise (there shouldn't really be any
         // anyhow).
-        cause: ObligationCause::misc(span, hir::CRATE_HIR_ID),
+        cause: ObligationCause::misc(span, CRATE_DEF_ID),
         recursion_depth: 0,
         predicate: pred.to_predicate(infcx.tcx),
     };
 
-    let result = infcx.predicate_must_hold_modulo_regions(&obligation);
+    let result = infcx.evaluate_obligation_no_overflow(&obligation);
     debug!(?result);
 
-    if result && has_non_region_infer {
+    if result.must_apply_modulo_regions() && !has_non_region_infer {
+        true
+    } else if result.may_apply() {
         // Because of inference "guessing", selection can sometimes claim
         // to succeed while the success requires a guess. To ensure
         // this function's result remains infallible, we must confirm
         // that guess. While imperfect, I believe this is sound.
 
         // FIXME(@lcnr): this function doesn't seem right.
+        //
         // The handling of regions in this area of the code is terrible,
         // see issue #29149. We should be able to improve on this with
         // NLL.
         let errors = fully_solve_obligation(infcx, obligation);
 
-        // Note: we only assume something is `Copy` if we can
-        // *definitively* show that it implements `Copy`. Otherwise,
-        // assume it is move; linear is always ok.
         match &errors[..] {
             [] => true,
             errors => {
@@ -183,7 +181,7 @@ fn pred_known_to_hold_modulo_regions<'tcx>(
             }
         }
     } else {
-        result
+        false
     }
 }
 
@@ -285,7 +283,7 @@ pub fn normalize_param_env_or_error<'tcx>(
     debug!("normalize_param_env_or_error: elaborated-predicates={:?}", predicates);
 
     let elaborated_env = ty::ParamEnv::new(
-        tcx.intern_predicates(&predicates),
+        tcx.mk_predicates(&predicates),
         unnormalized_env.reveal(),
         unnormalized_env.constness(),
     );
@@ -337,10 +335,9 @@ pub fn normalize_param_env_or_error<'tcx>(
     // Not sure whether it is better to include the unnormalized TypeOutlives predicates
     // here. I believe they should not matter, because we are ignoring TypeOutlives param-env
     // predicates here anyway. Keeping them here anyway because it seems safer.
-    let outlives_env: Vec<_> =
-        non_outlives_predicates.iter().chain(&outlives_predicates).cloned().collect();
+    let outlives_env = non_outlives_predicates.iter().chain(&outlives_predicates).cloned();
     let outlives_env = ty::ParamEnv::new(
-        tcx.intern_predicates(&outlives_env),
+        tcx.mk_predicates_from_iter(outlives_env),
         unnormalized_env.reveal(),
         unnormalized_env.constness(),
     );
@@ -360,7 +357,7 @@ pub fn normalize_param_env_or_error<'tcx>(
     predicates.extend(outlives_predicates);
     debug!("normalize_param_env_or_error: final predicates={:?}", predicates);
     ty::ParamEnv::new(
-        tcx.intern_predicates(&predicates),
+        tcx.mk_predicates(&predicates),
         unnormalized_env.reveal(),
         unnormalized_env.constness(),
     )
@@ -375,7 +372,7 @@ pub fn fully_normalize<'tcx, T>(
     value: T,
 ) -> Result<T, Vec<FulfillmentError<'tcx>>>
 where
-    T: TypeFoldable<'tcx>,
+    T: TypeFoldable<TyCtxt<'tcx>>,
 {
     let ocx = ObligationCtxt::new(infcx);
     debug!(?value);
@@ -485,7 +482,7 @@ fn is_impossible_method(tcx: TyCtxt<'_>, (impl_def_id, trait_item_def_id): (DefI
         generics: &'tcx ty::Generics,
         trait_item_def_id: DefId,
     }
-    impl<'tcx> ty::TypeVisitor<'tcx> for ReferencesOnlyParentGenerics<'tcx> {
+    impl<'tcx> ty::TypeVisitor<TyCtxt<'tcx>> for ReferencesOnlyParentGenerics<'tcx> {
         type BreakTy = ();
         fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
             // If this is a parameter from the trait item's own generics, then bail
@@ -527,16 +524,14 @@ fn is_impossible_method(tcx: TyCtxt<'_>, (impl_def_id, trait_item_def_id): (DefI
 
     let mut visitor = ReferencesOnlyParentGenerics { tcx, generics, trait_item_def_id };
     let predicates_for_trait = predicates.predicates.iter().filter_map(|(pred, span)| {
-        if pred.visit_with(&mut visitor).is_continue() {
-            Some(Obligation::new(
+        pred.visit_with(&mut visitor).is_continue().then(|| {
+            Obligation::new(
                 tcx,
                 ObligationCause::dummy_with_span(*span),
                 param_env,
                 ty::EarlyBinder(*pred).subst(tcx, impl_trait_ref.substs),
-            ))
-        } else {
-            None
-        }
+            )
+        })
     });
 
     let infcx = tcx.infer_ctxt().ignoring_regions().build();
@@ -558,6 +553,7 @@ pub fn provide(providers: &mut ty::query::Providers) {
         specialization_graph_of: specialize::specialization_graph_provider,
         specializes: specialize::specializes,
         subst_and_check_impossible_predicates,
+        check_tys_might_be_eq: misc::check_tys_might_be_eq,
         is_impossible_method,
         ..*providers
     };
diff --git a/compiler/rustc_trait_selection/src/traits/object_safety.rs b/compiler/rustc_trait_selection/src/traits/object_safety.rs
index c9121212c..4eacb5211 100644
--- a/compiler/rustc_trait_selection/src/traits/object_safety.rs
+++ b/compiler/rustc_trait_selection/src/traits/object_safety.rs
@@ -18,10 +18,10 @@ use rustc_errors::{DelayDm, FatalError, MultiSpan};
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 use rustc_middle::ty::subst::{GenericArg, InternalSubsts};
+use rustc_middle::ty::ToPredicate;
 use rustc_middle::ty::{
     self, EarlyBinder, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitor,
 };
-use rustc_middle::ty::{Predicate, ToPredicate};
 use rustc_session::lint::builtin::WHERE_CLAUSES_OBJECT_SAFETY;
 use rustc_span::symbol::Symbol;
 use rustc_span::Span;
@@ -62,11 +62,42 @@ fn object_safety_violations(tcx: TyCtxt<'_>, trait_def_id: DefId) -> &'_ [Object
     )
 }
 
+fn check_is_object_safe(tcx: TyCtxt<'_>, trait_def_id: DefId) -> bool {
+    let violations = tcx.object_safety_violations(trait_def_id);
+
+    if violations.is_empty() {
+        return true;
+    }
+
+    // If the trait contains any other violations, then let the error reporting path
+    // report it instead of emitting a warning here.
+    if violations.iter().all(|violation| {
+        matches!(
+            violation,
+            ObjectSafetyViolation::Method(_, MethodViolationCode::WhereClauseReferencesSelf, _)
+        )
+    }) {
+        for violation in violations {
+            if let ObjectSafetyViolation::Method(
+                _,
+                MethodViolationCode::WhereClauseReferencesSelf,
+                span,
+            ) = violation
+            {
+                lint_object_unsafe_trait(tcx, *span, trait_def_id, &violation);
+            }
+        }
+        return true;
+    }
+
+    false
+}
+
 /// We say a method is *vtable safe* if it can be invoked on a trait
 /// object. Note that object-safe traits can have some
 /// non-vtable-safe methods, so long as they require `Self: Sized` or
 /// otherwise ensure that they cannot be used when `Self = Trait`.
-pub fn is_vtable_safe_method(tcx: TyCtxt<'_>, trait_def_id: DefId, method: &ty::AssocItem) -> bool {
+pub fn is_vtable_safe_method(tcx: TyCtxt<'_>, trait_def_id: DefId, method: ty::AssocItem) -> bool {
     debug_assert!(tcx.generics_of(trait_def_id).has_self);
     debug!("is_vtable_safe_method({:?}, {:?})", trait_def_id, method);
     // Any method that has a `Self: Sized` bound cannot be called.
@@ -89,23 +120,10 @@ fn object_safety_violations_for_trait(
         .associated_items(trait_def_id)
         .in_definition_order()
         .filter(|item| item.kind == ty::AssocKind::Fn)
-        .filter_map(|item| {
-            object_safety_violation_for_method(tcx, trait_def_id, &item)
+        .filter_map(|&item| {
+            object_safety_violation_for_method(tcx, trait_def_id, item)
                 .map(|(code, span)| ObjectSafetyViolation::Method(item.name, code, span))
         })
-        .filter(|violation| {
-            if let ObjectSafetyViolation::Method(
-                _,
-                MethodViolationCode::WhereClauseReferencesSelf,
-                span,
-            ) = violation
-            {
-                lint_object_unsafe_trait(tcx, *span, trait_def_id, &violation);
-                false
-            } else {
-                true
-            }
-        })
         .collect();
 
     // Check the trait itself.
@@ -289,7 +307,7 @@ fn predicate_references_self<'tcx>(
     match predicate.kind().skip_binder() {
         ty::PredicateKind::Clause(ty::Clause::Trait(ref data)) => {
             // In the case of a trait predicate, we can skip the "self" type.
-            if data.trait_ref.substs[1..].iter().any(has_self_ty) { Some(sp) } else { None }
+            data.trait_ref.substs[1..].iter().any(has_self_ty).then_some(sp)
         }
         ty::PredicateKind::Clause(ty::Clause::Projection(ref data)) => {
             // And similarly for projections. This should be redundant with
@@ -307,8 +325,14 @@ fn predicate_references_self<'tcx>(
             //
             // This is ALT2 in issue #56288, see that for discussion of the
             // possible alternatives.
-            if data.projection_ty.substs[1..].iter().any(has_self_ty) { Some(sp) } else { None }
+            data.projection_ty.substs[1..].iter().any(has_self_ty).then_some(sp)
+        }
+        ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(_ct, ty)) => {
+            has_self_ty(&ty.into()).then_some(sp)
         }
+
+        ty::PredicateKind::AliasEq(..) => bug!("`AliasEq` not allowed as assumption"),
+
         ty::PredicateKind::WellFormed(..)
         | ty::PredicateKind::ObjectSafe(..)
         | ty::PredicateKind::Clause(ty::Clause::TypeOutlives(..))
@@ -316,6 +340,7 @@ fn predicate_references_self<'tcx>(
         | ty::PredicateKind::ClosureKind(..)
         | ty::PredicateKind::Subtype(..)
         | ty::PredicateKind::Coerce(..)
+        // FIXME(generic_const_exprs): this can mention `Self`
         | ty::PredicateKind::ConstEvaluatable(..)
         | ty::PredicateKind::ConstEquate(..)
         | ty::PredicateKind::Ambiguous
@@ -341,6 +366,7 @@ fn generics_require_sized_self(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
                 trait_pred.def_id() == sized_def_id && trait_pred.self_ty().is_param(0)
             }
             ty::PredicateKind::Clause(ty::Clause::Projection(..))
+            | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
             | ty::PredicateKind::Subtype(..)
             | ty::PredicateKind::Coerce(..)
             | ty::PredicateKind::Clause(ty::Clause::RegionOutlives(..))
@@ -350,6 +376,7 @@ fn generics_require_sized_self(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
             | ty::PredicateKind::Clause(ty::Clause::TypeOutlives(..))
             | ty::PredicateKind::ConstEvaluatable(..)
             | ty::PredicateKind::ConstEquate(..)
+            | ty::PredicateKind::AliasEq(..)
             | ty::PredicateKind::Ambiguous
             | ty::PredicateKind::TypeWellFormedFromEnv(..) => false,
         }
@@ -360,7 +387,7 @@ fn generics_require_sized_self(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
 fn object_safety_violation_for_method(
     tcx: TyCtxt<'_>,
     trait_def_id: DefId,
-    method: &ty::AssocItem,
+    method: ty::AssocItem,
 ) -> Option<(MethodViolationCode, Span)> {
     debug!("object_safety_violation_for_method({:?}, {:?})", trait_def_id, method);
     // Any method that has a `Self : Sized` requisite is otherwise
@@ -393,9 +420,9 @@ fn object_safety_violation_for_method(
 fn virtual_call_violation_for_method<'tcx>(
     tcx: TyCtxt<'tcx>,
     trait_def_id: DefId,
-    method: &ty::AssocItem,
+    method: ty::AssocItem,
 ) -> Option<MethodViolationCode> {
-    let sig = tcx.fn_sig(method.def_id);
+    let sig = tcx.fn_sig(method.def_id).subst_identity();
 
     // The method's first parameter must be named `self`
     if !method.fn_has_self_parameter {
@@ -505,8 +532,7 @@ fn virtual_call_violation_for_method<'tcx>(
                 }
             }
 
-            let trait_object_ty =
-                object_ty_for_trait(tcx, trait_def_id, tcx.mk_region(ty::ReStatic));
+            let trait_object_ty = object_ty_for_trait(tcx, trait_def_id, tcx.lifetimes.re_static);
 
             // e.g., `Rc<dyn Trait>`
             let trait_object_receiver =
@@ -529,16 +555,56 @@ fn virtual_call_violation_for_method<'tcx>(
 
     // NOTE: This check happens last, because it results in a lint, and not a
     // hard error.
-    if tcx
-        .predicates_of(method.def_id)
-        .predicates
-        .iter()
-        // A trait object can't claim to live more than the concrete type,
-        // so outlives predicates will always hold.
-        .cloned()
-        .filter(|(p, _)| p.to_opt_type_outlives().is_none())
-        .any(|pred| contains_illegal_self_type_reference(tcx, trait_def_id, pred))
-    {
+    if tcx.predicates_of(method.def_id).predicates.iter().any(|&(pred, span)| {
+        // dyn Trait is okay:
+        //
+        //     trait Trait {
+        //         fn f(&self) where Self: 'static;
+        //     }
+        //
+        // because a trait object can't claim to live longer than the concrete
+        // type. If the lifetime bound holds on dyn Trait then it's guaranteed
+        // to hold as well on the concrete type.
+        if pred.to_opt_type_outlives().is_some() {
+            return false;
+        }
+
+        // dyn Trait is okay:
+        //
+        //     auto trait AutoTrait {}
+        //
+        //     trait Trait {
+        //         fn f(&self) where Self: AutoTrait;
+        //     }
+        //
+        // because `impl AutoTrait for dyn Trait` is disallowed by coherence.
+        // Traits with a default impl are implemented for a trait object if and
+        // only if the autotrait is one of the trait object's trait bounds, like
+        // in `dyn Trait + AutoTrait`. This guarantees that trait objects only
+        // implement auto traits if the underlying type does as well.
+        if let ty::PredicateKind::Clause(ty::Clause::Trait(ty::TraitPredicate {
+            trait_ref: pred_trait_ref,
+            constness: ty::BoundConstness::NotConst,
+            polarity: ty::ImplPolarity::Positive,
+        })) = pred.kind().skip_binder()
+            && pred_trait_ref.self_ty() == tcx.types.self_param
+            && tcx.trait_is_auto(pred_trait_ref.def_id)
+        {
+            // Consider bounds like `Self: Bound<Self>`. Auto traits are not
+            // allowed to have generic parameters so `auto trait Bound<T> {}`
+            // would already have reported an error at the definition of the
+            // auto trait.
+            if pred_trait_ref.substs.len() != 1 {
+                tcx.sess.diagnostic().delay_span_bug(
+                    span,
+                    "auto traits cannot have generic parameters",
+                );
+            }
+            return false;
+        }
+
+        contains_illegal_self_type_reference(tcx, trait_def_id, pred)
+    }) {
         return Some(MethodViolationCode::WhereClauseReferencesSelf);
     }
 
@@ -588,11 +654,9 @@ fn object_ty_for_trait<'tcx>(
             debug!(?obligation);
             let pred = obligation.predicate.to_opt_poly_projection_pred()?;
             Some(pred.map_bound(|p| {
-                ty::ExistentialPredicate::Projection(ty::ExistentialProjection {
-                    def_id: p.projection_ty.def_id,
-                    substs: p.projection_ty.substs,
-                    term: p.term,
-                })
+                ty::ExistentialPredicate::Projection(ty::ExistentialProjection::erase_self_ty(
+                    tcx, p,
+                ))
             }))
         })
         .collect();
@@ -602,8 +666,9 @@ fn object_ty_for_trait<'tcx>(
     elaborated_predicates.sort_by(|a, b| a.skip_binder().stable_cmp(tcx, &b.skip_binder()));
     elaborated_predicates.dedup();
 
-    let existential_predicates = tcx
-        .mk_poly_existential_predicates(iter::once(trait_predicate).chain(elaborated_predicates));
+    let existential_predicates = tcx.mk_poly_existential_predicates_from_iter(
+        iter::once(trait_predicate).chain(elaborated_predicates),
+    );
     debug!(?existential_predicates);
 
     tcx.mk_dynamic(existential_predicates, lifetime, ty::Dyn)
@@ -658,7 +723,7 @@ fn object_ty_for_trait<'tcx>(
 #[allow(dead_code)]
 fn receiver_is_dispatchable<'tcx>(
     tcx: TyCtxt<'tcx>,
-    method: &ty::AssocItem,
+    method: ty::AssocItem,
     receiver_ty: Ty<'tcx>,
 ) -> bool {
     debug!("receiver_is_dispatchable: method = {:?}, receiver_ty = {:?}", method, receiver_ty);
@@ -702,11 +767,11 @@ fn receiver_is_dispatchable<'tcx>(
             ty::Binder::dummy(tcx.mk_trait_ref(trait_def_id, substs)).to_predicate(tcx)
         };
 
-        let caller_bounds: Vec<Predicate<'tcx>> =
-            param_env.caller_bounds().iter().chain([unsize_predicate, trait_predicate]).collect();
+        let caller_bounds =
+            param_env.caller_bounds().iter().chain([unsize_predicate, trait_predicate]);
 
         ty::ParamEnv::new(
-            tcx.intern_predicates(&caller_bounds),
+            tcx.mk_predicates_from_iter(caller_bounds),
             param_env.reveal(),
             param_env.constness(),
         )
@@ -726,7 +791,7 @@ fn receiver_is_dispatchable<'tcx>(
     infcx.predicate_must_hold_modulo_regions(&obligation)
 }
 
-fn contains_illegal_self_type_reference<'tcx, T: TypeVisitable<'tcx>>(
+fn contains_illegal_self_type_reference<'tcx, T: TypeVisitable<TyCtxt<'tcx>>>(
     tcx: TyCtxt<'tcx>,
     trait_def_id: DefId,
     value: T,
@@ -776,7 +841,7 @@ fn contains_illegal_self_type_reference<'tcx, T: TypeVisitable<'tcx>>(
         supertraits: Option<Vec<DefId>>,
     }
 
-    impl<'tcx> TypeVisitor<'tcx> for IllegalSelfTypeVisitor<'tcx> {
+    impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for IllegalSelfTypeVisitor<'tcx> {
         type BreakTy = ();
 
         fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
@@ -866,5 +931,6 @@ pub fn contains_illegal_impl_trait_in_trait<'tcx>(
 }
 
 pub fn provide(providers: &mut ty::query::Providers) {
-    *providers = ty::query::Providers { object_safety_violations, ..*providers };
+    *providers =
+        ty::query::Providers { object_safety_violations, check_is_object_safe, ..*providers };
 }
diff --git a/compiler/rustc_trait_selection/src/traits/outlives_bounds.rs b/compiler/rustc_trait_selection/src/traits/outlives_bounds.rs
index f2c5f730b..6cb64ad57 100644
--- a/compiler/rustc_trait_selection/src/traits/outlives_bounds.rs
+++ b/compiler/rustc_trait_selection/src/traits/outlives_bounds.rs
@@ -3,9 +3,8 @@ use crate::traits::query::type_op::{self, TypeOp, TypeOpOutput};
 use crate::traits::query::NoSolution;
 use crate::traits::ObligationCause;
 use rustc_data_structures::fx::FxIndexSet;
-use rustc_hir as hir;
-use rustc_hir::HirId;
 use rustc_middle::ty::{self, ParamEnv, Ty};
+use rustc_span::def_id::LocalDefId;
 
 pub use rustc_middle::traits::query::OutlivesBound;
 
@@ -14,14 +13,14 @@ pub trait InferCtxtExt<'a, 'tcx> {
     fn implied_outlives_bounds(
         &self,
         param_env: ty::ParamEnv<'tcx>,
-        body_id: hir::HirId,
+        body_id: LocalDefId,
         ty: Ty<'tcx>,
     ) -> Vec<OutlivesBound<'tcx>>;
 
     fn implied_bounds_tys(
         &'a self,
         param_env: ty::ParamEnv<'tcx>,
-        body_id: hir::HirId,
+        body_id: LocalDefId,
         tys: FxIndexSet<Ty<'tcx>>,
     ) -> Bounds<'a, 'tcx>;
 }
@@ -50,10 +49,10 @@ impl<'a, 'tcx: 'a> InferCtxtExt<'a, 'tcx> for InferCtxt<'tcx> {
     fn implied_outlives_bounds(
         &self,
         param_env: ty::ParamEnv<'tcx>,
-        body_id: hir::HirId,
+        body_id: LocalDefId,
         ty: Ty<'tcx>,
     ) -> Vec<OutlivesBound<'tcx>> {
-        let span = self.tcx.hir().span(body_id);
+        let span = self.tcx.def_span(body_id);
         let result = param_env
             .and(type_op::implied_outlives_bounds::ImpliedOutlivesBounds { ty })
             .fully_perform(self);
@@ -102,7 +101,7 @@ impl<'a, 'tcx: 'a> InferCtxtExt<'a, 'tcx> for InferCtxt<'tcx> {
     fn implied_bounds_tys(
         &'a self,
         param_env: ParamEnv<'tcx>,
-        body_id: HirId,
+        body_id: LocalDefId,
         tys: FxIndexSet<Ty<'tcx>>,
     ) -> Bounds<'a, 'tcx> {
         tys.into_iter()
diff --git a/compiler/rustc_trait_selection/src/traits/project.rs b/compiler/rustc_trait_selection/src/traits/project.rs
index fbc7ecced..870ecc2a9 100644
--- a/compiler/rustc_trait_selection/src/traits/project.rs
+++ b/compiler/rustc_trait_selection/src/traits/project.rs
@@ -31,7 +31,7 @@ use rustc_infer::infer::resolve::OpportunisticRegionResolver;
 use rustc_infer::traits::ImplSourceBuiltinData;
 use rustc_middle::traits::select::OverflowError;
 use rustc_middle::ty::fold::{TypeFoldable, TypeFolder, TypeSuperFoldable};
-use rustc_middle::ty::visit::{MaxUniverse, TypeVisitable};
+use rustc_middle::ty::visit::{MaxUniverse, TypeVisitable, TypeVisitableExt};
 use rustc_middle::ty::DefIdTree;
 use rustc_middle::ty::{self, Term, ToPredicate, Ty, TyCtxt};
 use rustc_span::symbol::sym;
@@ -53,11 +53,11 @@ pub trait NormalizeExt<'tcx> {
     ///
     /// This normalization should be used when the type contains inference variables or the
     /// projection may be fallible.
-    fn normalize<T: TypeFoldable<'tcx>>(&self, t: T) -> InferOk<'tcx, T>;
+    fn normalize<T: TypeFoldable<TyCtxt<'tcx>>>(&self, t: T) -> InferOk<'tcx, T>;
 }
 
 impl<'tcx> NormalizeExt<'tcx> for At<'_, 'tcx> {
-    fn normalize<T: TypeFoldable<'tcx>>(&self, value: T) -> InferOk<'tcx, T> {
+    fn normalize<T: TypeFoldable<TyCtxt<'tcx>>>(&self, value: T) -> InferOk<'tcx, T> {
         let mut selcx = SelectionContext::new(self.infcx);
         let Normalized { value, obligations } =
             normalize_with_depth(&mut selcx, self.param_env, self.cause.clone(), 0, value);
@@ -90,15 +90,7 @@ enum ProjectionCandidate<'tcx> {
     /// From an "impl" (or a "pseudo-impl" returned by select)
     Select(Selection<'tcx>),
 
-    ImplTraitInTrait(ImplTraitInTraitCandidate<'tcx>),
-}
-
-#[derive(PartialEq, Eq, Debug)]
-enum ImplTraitInTraitCandidate<'tcx> {
-    // The `impl Trait` from a trait function's default body
-    Trait,
-    // A concrete type provided from a trait's `impl Trait` from an impl
-    Impl(ImplSourceUserDefinedData<'tcx, PredicateObligation<'tcx>>),
+    ImplTraitInTrait(ImplSourceUserDefinedData<'tcx, PredicateObligation<'tcx>>),
 }
 
 enum ProjectionCandidateSet<'tcx> {
@@ -215,7 +207,7 @@ pub(super) fn poly_project_and_unify_type<'cx, 'tcx>(
     let r = infcx.commit_if_ok(|_snapshot| {
         let old_universe = infcx.universe();
         let placeholder_predicate =
-            infcx.replace_bound_vars_with_placeholders(obligation.predicate);
+            infcx.instantiate_binder_with_placeholders(obligation.predicate);
         let new_universe = infcx.universe();
 
         let placeholder_obligation = obligation.with(infcx.tcx, placeholder_predicate);
@@ -294,7 +286,12 @@ fn project_and_unify_type<'cx, 'tcx>(
         );
     obligations.extend(new);
 
-    match infcx.at(&obligation.cause, obligation.param_env).eq(normalized, actual) {
+    match infcx
+        .at(&obligation.cause, obligation.param_env)
+        // This is needed to support nested opaque types like `impl Fn() -> impl Trait`
+        .define_opaque_types(true)
+        .eq(normalized, actual)
+    {
         Ok(InferOk { obligations: inferred_obligations, value: () }) => {
             obligations.extend(inferred_obligations);
             ProjectAndUnifyResult::Holds(obligations)
@@ -315,7 +312,7 @@ pub(crate) fn normalize_with_depth<'a, 'b, 'tcx, T>(
     value: T,
 ) -> Normalized<'tcx, T>
 where
-    T: TypeFoldable<'tcx>,
+    T: TypeFoldable<TyCtxt<'tcx>>,
 {
     let mut obligations = Vec::new();
     let value = normalize_with_depth_to(selcx, param_env, cause, depth, value, &mut obligations);
@@ -332,7 +329,7 @@ pub(crate) fn normalize_with_depth_to<'a, 'b, 'tcx, T>(
     obligations: &mut Vec<PredicateObligation<'tcx>>,
 ) -> T
 where
-    T: TypeFoldable<'tcx>,
+    T: TypeFoldable<TyCtxt<'tcx>>,
 {
     debug!(obligations.len = obligations.len());
     let mut normalizer = AssocTypeNormalizer::new(selcx, param_env, cause, depth, obligations);
@@ -352,7 +349,7 @@ pub(crate) fn try_normalize_with_depth_to<'a, 'b, 'tcx, T>(
     obligations: &mut Vec<PredicateObligation<'tcx>>,
 ) -> T
 where
-    T: TypeFoldable<'tcx>,
+    T: TypeFoldable<TyCtxt<'tcx>>,
 {
     debug!(obligations.len = obligations.len());
     let mut normalizer = AssocTypeNormalizer::new_without_eager_inference_replacement(
@@ -368,7 +365,10 @@ where
     result
 }
 
-pub(crate) fn needs_normalization<'tcx, T: TypeVisitable<'tcx>>(value: &T, reveal: Reveal) -> bool {
+pub(crate) fn needs_normalization<'tcx, T: TypeVisitable<TyCtxt<'tcx>>>(
+    value: &T,
+    reveal: Reveal,
+) -> bool {
     match reveal {
         Reveal::UserFacing => value
             .has_type_flags(ty::TypeFlags::HAS_TY_PROJECTION | ty::TypeFlags::HAS_CT_PROJECTION),
@@ -430,7 +430,7 @@ impl<'a, 'b, 'tcx> AssocTypeNormalizer<'a, 'b, 'tcx> {
         }
     }
 
-    fn fold<T: TypeFoldable<'tcx>>(&mut self, value: T) -> T {
+    fn fold<T: TypeFoldable<TyCtxt<'tcx>>>(&mut self, value: T) -> T {
         let value = self.selcx.infcx.resolve_vars_if_possible(value);
         debug!(?value);
 
@@ -448,12 +448,12 @@ impl<'a, 'b, 'tcx> AssocTypeNormalizer<'a, 'b, 'tcx> {
     }
 }
 
-impl<'a, 'b, 'tcx> TypeFolder<'tcx> for AssocTypeNormalizer<'a, 'b, 'tcx> {
-    fn tcx<'c>(&'c self) -> TyCtxt<'tcx> {
+impl<'a, 'b, 'tcx> TypeFolder<TyCtxt<'tcx>> for AssocTypeNormalizer<'a, 'b, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.selcx.tcx()
     }
 
-    fn fold_binder<T: TypeFoldable<'tcx>>(
+    fn fold_binder<T: TypeFoldable<TyCtxt<'tcx>>>(
         &mut self,
         t: ty::Binder<'tcx, T>,
     ) -> ty::Binder<'tcx, T> {
@@ -503,7 +503,7 @@ impl<'a, 'b, 'tcx> TypeFolder<'tcx> for AssocTypeNormalizer<'a, 'b, 'tcx> {
                     Reveal::UserFacing => ty.super_fold_with(self),
 
                     Reveal::All => {
-                        let recursion_limit = self.tcx().recursion_limit();
+                        let recursion_limit = self.interner().recursion_limit();
                         if !recursion_limit.value_within_limit(self.depth) {
                             self.selcx.infcx.err_ctxt().report_overflow_error(
                                 &ty,
@@ -514,8 +514,8 @@ impl<'a, 'b, 'tcx> TypeFolder<'tcx> for AssocTypeNormalizer<'a, 'b, 'tcx> {
                         }
 
                         let substs = substs.fold_with(self);
-                        let generic_ty = self.tcx().bound_type_of(def_id);
-                        let concrete_ty = generic_ty.subst(self.tcx(), substs);
+                        let generic_ty = self.interner().type_of(def_id);
+                        let concrete_ty = generic_ty.subst(self.interner(), substs);
                         self.depth += 1;
                         let folded_ty = self.fold_ty(concrete_ty);
                         self.depth -= 1;
@@ -672,7 +672,12 @@ pub struct BoundVarReplacer<'me, 'tcx> {
 ///
 /// FIXME(@lcnr): We may even consider experimenting with eagerly replacing bound vars during
 /// normalization as well, at which point this function will be unnecessary and can be removed.
-pub fn with_replaced_escaping_bound_vars<'a, 'tcx, T: TypeFoldable<'tcx>, R: TypeFoldable<'tcx>>(
+pub fn with_replaced_escaping_bound_vars<
+    'a,
+    'tcx,
+    T: TypeFoldable<TyCtxt<'tcx>>,
+    R: TypeFoldable<TyCtxt<'tcx>>,
+>(
     infcx: &'a InferCtxt<'tcx>,
     universe_indices: &'a mut Vec<Option<ty::UniverseIndex>>,
     value: T,
@@ -698,7 +703,7 @@ pub fn with_replaced_escaping_bound_vars<'a, 'tcx, T: TypeFoldable<'tcx>, R: Typ
 impl<'me, 'tcx> BoundVarReplacer<'me, 'tcx> {
     /// Returns `Some` if we *were* able to replace bound vars. If there are any bound vars that
     /// use a binding level above `universe_indices.len()`, we fail.
-    pub fn replace_bound_vars<T: TypeFoldable<'tcx>>(
+    pub fn replace_bound_vars<T: TypeFoldable<TyCtxt<'tcx>>>(
         infcx: &'me InferCtxt<'tcx>,
         universe_indices: &'me mut Vec<Option<ty::UniverseIndex>>,
         value: T,
@@ -740,12 +745,12 @@ impl<'me, 'tcx> BoundVarReplacer<'me, 'tcx> {
     }
 }
 
-impl<'tcx> TypeFolder<'tcx> for BoundVarReplacer<'_, 'tcx> {
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for BoundVarReplacer<'_, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.infcx.tcx
     }
 
-    fn fold_binder<T: TypeFoldable<'tcx>>(
+    fn fold_binder<T: TypeFoldable<TyCtxt<'tcx>>>(
         &mut self,
         t: ty::Binder<'tcx, T>,
     ) -> ty::Binder<'tcx, T> {
@@ -767,7 +772,7 @@ impl<'tcx> TypeFolder<'tcx> for BoundVarReplacer<'_, 'tcx> {
                 let universe = self.universe_for(debruijn);
                 let p = ty::PlaceholderRegion { universe, name: br.kind };
                 self.mapped_regions.insert(p, br);
-                self.infcx.tcx.mk_region(ty::RePlaceholder(p))
+                self.infcx.tcx.mk_re_placeholder(p)
             }
             _ => r,
         }
@@ -783,9 +788,9 @@ impl<'tcx> TypeFolder<'tcx> for BoundVarReplacer<'_, 'tcx> {
             }
             ty::Bound(debruijn, bound_ty) if debruijn >= self.current_index => {
                 let universe = self.universe_for(debruijn);
-                let p = ty::PlaceholderType { universe, name: bound_ty.var };
+                let p = ty::PlaceholderType { universe, name: bound_ty.kind };
                 self.mapped_types.insert(p, bound_ty);
-                self.infcx.tcx.mk_ty(ty::Placeholder(p))
+                self.infcx.tcx.mk_placeholder(p)
             }
             _ if t.has_vars_bound_at_or_above(self.current_index) => t.super_fold_with(self),
             _ => t,
@@ -826,7 +831,7 @@ pub struct PlaceholderReplacer<'me, 'tcx> {
 }
 
 impl<'me, 'tcx> PlaceholderReplacer<'me, 'tcx> {
-    pub fn replace_placeholders<T: TypeFoldable<'tcx>>(
+    pub fn replace_placeholders<T: TypeFoldable<TyCtxt<'tcx>>>(
         infcx: &'me InferCtxt<'tcx>,
         mapped_regions: BTreeMap<ty::PlaceholderRegion, ty::BoundRegion>,
         mapped_types: BTreeMap<ty::PlaceholderType, ty::BoundTy>,
@@ -846,12 +851,12 @@ impl<'me, 'tcx> PlaceholderReplacer<'me, 'tcx> {
     }
 }
 
-impl<'tcx> TypeFolder<'tcx> for PlaceholderReplacer<'_, 'tcx> {
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for PlaceholderReplacer<'_, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.infcx.tcx
     }
 
-    fn fold_binder<T: TypeFoldable<'tcx>>(
+    fn fold_binder<T: TypeFoldable<TyCtxt<'tcx>>>(
         &mut self,
         t: ty::Binder<'tcx, T>,
     ) -> ty::Binder<'tcx, T> {
@@ -888,7 +893,7 @@ impl<'tcx> TypeFolder<'tcx> for PlaceholderReplacer<'_, 'tcx> {
                         let db = ty::DebruijnIndex::from_usize(
                             self.universe_indices.len() - index + self.current_index.as_usize() - 1,
                         );
-                        self.tcx().mk_region(ty::ReLateBound(db, *replace_var))
+                        self.interner().mk_re_late_bound(db, *replace_var)
                     }
                     None => r1,
                 }
@@ -915,7 +920,7 @@ impl<'tcx> TypeFolder<'tcx> for PlaceholderReplacer<'_, 'tcx> {
                         let db = ty::DebruijnIndex::from_usize(
                             self.universe_indices.len() - index + self.current_index.as_usize() - 1,
                         );
-                        self.tcx().mk_ty(ty::Bound(db, *replace_var))
+                        self.interner().mk_bound(db, *replace_var)
                     }
                     None => ty,
                 }
@@ -939,7 +944,7 @@ impl<'tcx> TypeFolder<'tcx> for PlaceholderReplacer<'_, 'tcx> {
                     let db = ty::DebruijnIndex::from_usize(
                         self.universe_indices.len() - index + self.current_index.as_usize() - 1,
                     );
-                    self.tcx().mk_const(ty::ConstKind::Bound(db, *replace_var), ct.ty())
+                    self.interner().mk_const(ty::ConstKind::Bound(db, *replace_var), ct.ty())
                 }
                 None => ct,
             }
@@ -1170,7 +1175,7 @@ fn opt_normalize_projection_type<'a, 'b, 'tcx>(
 /// paths you want to take. To make things worse, it was possible for
 /// cycles to arise, where you basically had a setup like `<MyType<$0>
 /// as Trait>::Foo == $0`. Here, normalizing `<MyType<$0> as
-/// Trait>::Foo> to `[type error]` would lead to an obligation of
+/// Trait>::Foo>` to `[type error]` would lead to an obligation of
 /// `<MyType<[type error]> as Trait>::Foo`. We are supposed to report
 /// an error for this obligation, but we legitimately should not,
 /// because it contains `[type error]`. Yuck! (See issue #29857 for
@@ -1208,8 +1213,8 @@ struct Progress<'tcx> {
 }
 
 impl<'tcx> Progress<'tcx> {
-    fn error(tcx: TyCtxt<'tcx>) -> Self {
-        Progress { term: tcx.ty_error().into(), obligations: vec![] }
+    fn error(tcx: TyCtxt<'tcx>, guar: ErrorGuaranteed) -> Self {
+        Progress { term: tcx.ty_error(guar).into(), obligations: vec![] }
     }
 
     fn with_addl_obligations(mut self, mut obligations: Vec<PredicateObligation<'tcx>>) -> Self {
@@ -1235,8 +1240,8 @@ fn project<'cx, 'tcx>(
         )));
     }
 
-    if obligation.predicate.references_error() {
-        return Ok(Projected::Progress(Progress::error(selcx.tcx())));
+    if let Err(guar) = obligation.predicate.error_reported() {
+        return Ok(Projected::Progress(Progress::error(selcx.tcx(), guar)));
     }
 
     let mut candidates = ProjectionCandidateSet::None;
@@ -1292,17 +1297,6 @@ fn assemble_candidate_for_impl_trait_in_trait<'cx, 'tcx>(
     let tcx = selcx.tcx();
     if tcx.def_kind(obligation.predicate.def_id) == DefKind::ImplTraitPlaceholder {
         let trait_fn_def_id = tcx.impl_trait_in_trait_parent(obligation.predicate.def_id);
-        // If we are trying to project an RPITIT with trait's default `Self` parameter,
-        // then we must be within a default trait body.
-        if obligation.predicate.self_ty()
-            == ty::InternalSubsts::identity_for_item(tcx, obligation.predicate.def_id).type_at(0)
-            && tcx.associated_item(trait_fn_def_id).defaultness(tcx).has_value()
-        {
-            candidate_set.push_candidate(ProjectionCandidate::ImplTraitInTrait(
-                ImplTraitInTraitCandidate::Trait,
-            ));
-            return;
-        }
 
         let trait_def_id = tcx.parent(trait_fn_def_id);
         let trait_substs =
@@ -1313,23 +1307,21 @@ fn assemble_candidate_for_impl_trait_in_trait<'cx, 'tcx>(
         let _ = selcx.infcx.commit_if_ok(|_| {
             match selcx.select(&obligation.with(tcx, trait_predicate)) {
                 Ok(Some(super::ImplSource::UserDefined(data))) => {
-                    candidate_set.push_candidate(ProjectionCandidate::ImplTraitInTrait(
-                        ImplTraitInTraitCandidate::Impl(data),
-                    ));
+                    candidate_set.push_candidate(ProjectionCandidate::ImplTraitInTrait(data));
                     Ok(())
                 }
                 Ok(None) => {
                     candidate_set.mark_ambiguous();
-                    return Err(());
+                    Err(())
                 }
                 Ok(Some(_)) => {
                     // Don't know enough about the impl to provide a useful signature
-                    return Err(());
+                    Err(())
                 }
                 Err(e) => {
                     debug!(error = ?e, "selection error");
                     candidate_set.mark_error(e);
-                    return Err(());
+                    Err(())
                 }
             }
         });
@@ -1605,6 +1597,7 @@ fn assemble_candidates_from_impls<'cx, 'tcx>(
                         | ty::Closure(..)
                         | ty::Generator(..)
                         | ty::GeneratorWitness(..)
+                        | ty::GeneratorWitnessMIR(..)
                         | ty::Never
                         | ty::Tuple(..)
                         // Integers and floats always have `u8` as their discriminant.
@@ -1654,6 +1647,7 @@ fn assemble_candidates_from_impls<'cx, 'tcx>(
                         | ty::Closure(..)
                         | ty::Generator(..)
                         | ty::GeneratorWitness(..)
+                        | ty::GeneratorWitnessMIR(..)
                         | ty::Never
                         // Extern types have unit metadata, according to RFC 2850
                         | ty::Foreign(_)
@@ -1775,18 +1769,9 @@ fn confirm_candidate<'cx, 'tcx>(
         ProjectionCandidate::Select(impl_source) => {
             confirm_select_candidate(selcx, obligation, impl_source)
         }
-        ProjectionCandidate::ImplTraitInTrait(ImplTraitInTraitCandidate::Impl(data)) => {
+        ProjectionCandidate::ImplTraitInTrait(data) => {
             confirm_impl_trait_in_trait_candidate(selcx, obligation, data)
         }
-        // If we're projecting an RPITIT for a default trait body, that's just
-        // the same def-id, but as an opaque type (with regular RPIT semantics).
-        ProjectionCandidate::ImplTraitInTrait(ImplTraitInTraitCandidate::Trait) => Progress {
-            term: selcx
-                .tcx()
-                .mk_opaque(obligation.predicate.def_id, obligation.predicate.substs)
-                .into(),
-            obligations: vec![],
-        },
     };
 
     // When checking for cycle during evaluation, we compare predicates with
@@ -1921,7 +1906,7 @@ fn confirm_builtin_candidate<'cx, 'tcx>(
 ) -> Progress<'tcx> {
     let tcx = selcx.tcx();
     let self_ty = obligation.predicate.self_ty();
-    let substs = tcx.mk_substs([self_ty.into()].iter());
+    let substs = tcx.mk_substs(&[self_ty.into()]);
     let lang_items = tcx.lang_items();
     let item_def_id = obligation.predicate.def_id;
     let trait_def_id = tcx.trait_of_item(item_def_id).unwrap();
@@ -2044,7 +2029,7 @@ fn confirm_param_env_candidate<'cx, 'tcx>(
     let cause = &obligation.cause;
     let param_env = obligation.param_env;
 
-    let cache_entry = infcx.replace_bound_vars_with_fresh_vars(
+    let cache_entry = infcx.instantiate_binder_with_fresh_vars(
         cause.span,
         LateBoundRegionConversionTime::HigherRankedType,
         poly_cache_entry,
@@ -2112,8 +2097,9 @@ fn confirm_impl_candidate<'cx, 'tcx>(
     let trait_def_id = tcx.trait_id_of_impl(impl_def_id).unwrap();
 
     let param_env = obligation.param_env;
-    let Ok(assoc_ty) = specialization_graph::assoc_def(tcx, impl_def_id, assoc_item_id) else {
-        return Progress { term: tcx.ty_error().into(), obligations: nested };
+    let assoc_ty = match specialization_graph::assoc_def(tcx, impl_def_id, assoc_item_id) {
+        Ok(assoc_ty) => assoc_ty,
+        Err(guar) => return Progress::error(tcx, guar),
     };
 
     if !assoc_ty.item.defaultness(tcx).has_value() {
@@ -2125,7 +2111,7 @@ fn confirm_impl_candidate<'cx, 'tcx>(
             "confirm_impl_candidate: no associated type {:?} for {:?}",
             assoc_ty.item.name, obligation.predicate
         );
-        return Progress { term: tcx.ty_error().into(), obligations: nested };
+        return Progress { term: tcx.ty_error_misc().into(), obligations: nested };
     }
     // If we're trying to normalize `<Vec<u32> as X>::A<S>` using
     //`impl<T> X for Vec<T> { type A<Y> = Box<Y>; }`, then:
@@ -2136,7 +2122,7 @@ fn confirm_impl_candidate<'cx, 'tcx>(
     let substs = obligation.predicate.substs.rebase_onto(tcx, trait_def_id, substs);
     let substs =
         translate_substs(selcx.infcx, param_env, impl_def_id, substs, assoc_ty.defining_node);
-    let ty = tcx.bound_type_of(assoc_ty.item.def_id);
+    let ty = tcx.type_of(assoc_ty.item.def_id);
     let is_const = matches!(tcx.def_kind(assoc_ty.item.def_id), DefKind::AssocConst);
     let term: ty::EarlyBinder<ty::Term<'tcx>> = if is_const {
         let identity_substs =
@@ -2147,7 +2133,7 @@ fn confirm_impl_candidate<'cx, 'tcx>(
     } else {
         ty.map_bound(|ty| ty.into())
     };
-    if !check_substs_compatible(tcx, &assoc_ty.item, substs) {
+    if !check_substs_compatible(tcx, assoc_ty.item, substs) {
         let err = tcx.ty_error_with_message(
             obligation.cause.span,
             "impl item and trait item have different parameters",
@@ -2162,7 +2148,7 @@ fn confirm_impl_candidate<'cx, 'tcx>(
 // Verify that the trait item and its implementation have compatible substs lists
 fn check_substs_compatible<'tcx>(
     tcx: TyCtxt<'tcx>,
-    assoc_item: &ty::AssocItem,
+    assoc_item: ty::AssocItem,
     substs: ty::SubstsRef<'tcx>,
 ) -> bool {
     fn check_substs_compatible_inner<'tcx>(
@@ -2209,11 +2195,13 @@ fn confirm_impl_trait_in_trait_candidate<'tcx>(
     let mut obligations = data.nested;
 
     let trait_fn_def_id = tcx.impl_trait_in_trait_parent(obligation.predicate.def_id);
-    let Ok(leaf_def) = specialization_graph::assoc_def(tcx, data.impl_def_id, trait_fn_def_id) else {
-        return Progress { term: tcx.ty_error().into(), obligations };
+    let leaf_def = match specialization_graph::assoc_def(tcx, data.impl_def_id, trait_fn_def_id) {
+        Ok(assoc_ty) => assoc_ty,
+        Err(guar) => return Progress::error(tcx, guar),
     };
-    if !leaf_def.item.defaultness(tcx).has_value() {
-        return Progress { term: tcx.ty_error().into(), obligations };
+    // We don't support specialization for RPITITs anyways... yet.
+    if !leaf_def.is_final() {
+        return Progress { term: tcx.ty_error_misc().into(), obligations };
     }
 
     // Use the default `impl Trait` for the trait, e.g., for a default trait body
@@ -2236,7 +2224,7 @@ fn confirm_impl_trait_in_trait_candidate<'tcx>(
         leaf_def.defining_node,
     );
 
-    if !check_substs_compatible(tcx, &leaf_def.item, impl_fn_substs) {
+    if !check_substs_compatible(tcx, leaf_def.item, impl_fn_substs) {
         let err = tcx.ty_error_with_message(
             obligation.cause.span,
             "impl method and trait method have different parameters",
@@ -2284,7 +2272,7 @@ fn confirm_impl_trait_in_trait_candidate<'tcx>(
         obligation.recursion_depth + 1,
         tcx.bound_return_position_impl_trait_in_trait_tys(impl_fn_def_id)
             .map_bound(|tys| {
-                tys.map_or_else(|_| tcx.ty_error(), |tys| tys[&obligation.predicate.def_id])
+                tys.map_or_else(|guar| tcx.ty_error(guar), |tys| tys[&obligation.predicate.def_id])
             })
             .subst(tcx, impl_fn_substs),
         &mut obligations,
diff --git a/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs b/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs
index 0f21813bc..455b53bfb 100644
--- a/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs
@@ -31,6 +31,7 @@ pub fn trivial_dropck_outlives<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool {
         | ty::FnPtr(_)
         | ty::Char
         | ty::GeneratorWitness(..)
+        | ty::GeneratorWitnessMIR(..)
         | ty::RawPtr(_)
         | ty::Ref(..)
         | ty::Str
diff --git a/compiler/rustc_trait_selection/src/traits/query/evaluate_obligation.rs b/compiler/rustc_trait_selection/src/traits/query/evaluate_obligation.rs
index 09b58894d..f183248f2 100644
--- a/compiler/rustc_trait_selection/src/traits/query/evaluate_obligation.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/evaluate_obligation.rs
@@ -1,7 +1,9 @@
 use rustc_middle::ty;
+use rustc_session::config::TraitSolver;
 
 use crate::infer::canonical::OriginalQueryValues;
 use crate::infer::InferCtxt;
+use crate::solve::{Certainty, Goal, InferCtxtEvalExt, MaybeCause};
 use crate::traits::{EvaluationResult, OverflowError, PredicateObligation, SelectionContext};
 
 pub trait InferCtxtExt<'tcx> {
@@ -77,12 +79,38 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
             _ => obligation.param_env.without_const(),
         };
 
-        let c_pred = self
-            .canonicalize_query_keep_static(param_env.and(obligation.predicate), &mut _orig_values);
-        // Run canonical query. If overflow occurs, rerun from scratch but this time
-        // in standard trait query mode so that overflow is handled appropriately
-        // within `SelectionContext`.
-        self.tcx.at(obligation.cause.span()).evaluate_obligation(c_pred)
+        if self.tcx.sess.opts.unstable_opts.trait_solver != TraitSolver::Next {
+            let c_pred = self.canonicalize_query_keep_static(
+                param_env.and(obligation.predicate),
+                &mut _orig_values,
+            );
+            self.tcx.at(obligation.cause.span()).evaluate_obligation(c_pred)
+        } else {
+            self.probe(|snapshot| {
+                if let Ok((_, certainty)) =
+                    self.evaluate_root_goal(Goal::new(self.tcx, param_env, obligation.predicate))
+                {
+                    match certainty {
+                        Certainty::Yes => {
+                            if self.opaque_types_added_in_snapshot(snapshot) {
+                                Ok(EvaluationResult::EvaluatedToOkModuloOpaqueTypes)
+                            } else if self.region_constraints_added_in_snapshot(snapshot).is_some()
+                            {
+                                Ok(EvaluationResult::EvaluatedToOkModuloRegions)
+                            } else {
+                                Ok(EvaluationResult::EvaluatedToOk)
+                            }
+                        }
+                        Certainty::Maybe(MaybeCause::Ambiguity) => {
+                            Ok(EvaluationResult::EvaluatedToAmbig)
+                        }
+                        Certainty::Maybe(MaybeCause::Overflow) => Err(OverflowError::Canonical),
+                    }
+                } else {
+                    Ok(EvaluationResult::EvaluatedToErr)
+                }
+            })
+        }
     }
 
     // Helper function that canonicalizes and runs the query. If an
@@ -92,6 +120,9 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
         &self,
         obligation: &PredicateObligation<'tcx>,
     ) -> EvaluationResult {
+        // Run canonical query. If overflow occurs, rerun from scratch but this time
+        // in standard trait query mode so that overflow is handled appropriately
+        // within `SelectionContext`.
         match self.evaluate_obligation(obligation) {
             Ok(result) => result,
             Err(OverflowError::Canonical) => {
diff --git a/compiler/rustc_trait_selection/src/traits/query/normalize.rs b/compiler/rustc_trait_selection/src/traits/query/normalize.rs
index 27247271d..b0cec3ce7 100644
--- a/compiler/rustc_trait_selection/src/traits/query/normalize.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/normalize.rs
@@ -12,7 +12,7 @@ use rustc_data_structures::sso::SsoHashMap;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_infer::traits::Normalized;
 use rustc_middle::ty::fold::{FallibleTypeFolder, TypeFoldable, TypeSuperFoldable};
-use rustc_middle::ty::visit::{TypeSuperVisitable, TypeVisitable};
+use rustc_middle::ty::visit::{TypeSuperVisitable, TypeVisitable, TypeVisitableExt};
 use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitor};
 use rustc_span::DUMMY_SP;
 
@@ -32,7 +32,7 @@ pub trait QueryNormalizeExt<'tcx> {
     /// use [`TyCtxt::normalize_erasing_regions`], which wraps this procedure.
     fn query_normalize<T>(&self, value: T) -> Result<Normalized<'tcx, T>, NoSolution>
     where
-        T: TypeFoldable<'tcx>;
+        T: TypeFoldable<TyCtxt<'tcx>>;
 }
 
 impl<'cx, 'tcx> QueryNormalizeExt<'tcx> for At<'cx, 'tcx> {
@@ -51,7 +51,7 @@ impl<'cx, 'tcx> QueryNormalizeExt<'tcx> for At<'cx, 'tcx> {
     /// and other details are still "under development".
     fn query_normalize<T>(&self, value: T) -> Result<Normalized<'tcx, T>, NoSolution>
     where
-        T: TypeFoldable<'tcx>,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         debug!(
             "normalize::<{}>(value={:?}, param_env={:?}, cause={:?})",
@@ -115,8 +115,8 @@ struct MaxEscapingBoundVarVisitor {
     escaping: usize,
 }
 
-impl<'tcx> TypeVisitor<'tcx> for MaxEscapingBoundVarVisitor {
-    fn visit_binder<T: TypeVisitable<'tcx>>(
+impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for MaxEscapingBoundVarVisitor {
+    fn visit_binder<T: TypeVisitable<TyCtxt<'tcx>>>(
         &mut self,
         t: &ty::Binder<'tcx, T>,
     ) -> ControlFlow<Self::BreakTy> {
@@ -170,14 +170,14 @@ struct QueryNormalizer<'cx, 'tcx> {
     universes: Vec<Option<ty::UniverseIndex>>,
 }
 
-impl<'cx, 'tcx> FallibleTypeFolder<'tcx> for QueryNormalizer<'cx, 'tcx> {
+impl<'cx, 'tcx> FallibleTypeFolder<TyCtxt<'tcx>> for QueryNormalizer<'cx, 'tcx> {
     type Error = NoSolution;
 
-    fn tcx<'c>(&'c self) -> TyCtxt<'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.infcx.tcx
     }
 
-    fn try_fold_binder<T: TypeFoldable<'tcx>>(
+    fn try_fold_binder<T: TypeFoldable<TyCtxt<'tcx>>>(
         &mut self,
         t: ty::Binder<'tcx, T>,
     ) -> Result<ty::Binder<'tcx, T>, Self::Error> {
@@ -214,18 +214,22 @@ impl<'cx, 'tcx> FallibleTypeFolder<'tcx> for QueryNormalizer<'cx, 'tcx> {
 
                     Reveal::All => {
                         let substs = substs.try_fold_with(self)?;
-                        let recursion_limit = self.tcx().recursion_limit();
+                        let recursion_limit = self.interner().recursion_limit();
                         if !recursion_limit.value_within_limit(self.anon_depth) {
-                            self.infcx.err_ctxt().report_overflow_error(
-                                &ty,
-                                self.cause.span,
-                                true,
-                                |_| {},
-                            );
+                            // A closure or generator may have itself as in its upvars.
+                            // This should be checked handled by the recursion check for opaque
+                            // types, but we may end up here before that check can happen.
+                            // In that case, we delay a bug to mark the trip, and continue without
+                            // revealing the opaque.
+                            self.infcx
+                                .err_ctxt()
+                                .build_overflow_error(&ty, self.cause.span, true)
+                                .delay_as_bug();
+                            return ty.try_super_fold_with(self);
                         }
 
-                        let generic_ty = self.tcx().bound_type_of(def_id);
-                        let concrete_ty = generic_ty.subst(self.tcx(), substs);
+                        let generic_ty = self.interner().type_of(def_id);
+                        let concrete_ty = generic_ty.subst(self.interner(), substs);
                         self.anon_depth += 1;
                         if concrete_ty == ty {
                             bug!(
diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/mod.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/mod.rs
index 97002b461..9e8bc8bce 100644
--- a/compiler/rustc_trait_selection/src/traits/query/type_op/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/type_op/mod.rs
@@ -54,8 +54,8 @@ pub struct TypeOpOutput<'tcx, Op: TypeOp<'tcx>> {
 /// which produces the resulting query region constraints.
 ///
 /// [c]: https://rust-lang.github.io/chalk/book/canonical_queries/canonicalization.html
-pub trait QueryTypeOp<'tcx>: fmt::Debug + Copy + TypeFoldable<'tcx> + 'tcx {
-    type QueryResponse: TypeFoldable<'tcx>;
+pub trait QueryTypeOp<'tcx>: fmt::Debug + Copy + TypeFoldable<TyCtxt<'tcx>> + 'tcx {
+    type QueryResponse: TypeFoldable<TyCtxt<'tcx>>;
 
     /// Give query the option for a simple fast path that never
     /// actually hits the tcx cache lookup etc. Return `Some(r)` with
diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/normalize.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/normalize.rs
index 8f0b4de31..5b216c076 100644
--- a/compiler/rustc_trait_selection/src/traits/query/type_op/normalize.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/type_op/normalize.rs
@@ -1,7 +1,7 @@
 use crate::infer::canonical::{Canonical, CanonicalQueryResponse};
 use crate::traits::query::Fallible;
 use rustc_middle::ty::fold::TypeFoldable;
-use rustc_middle::ty::{self, Lift, ParamEnvAnd, Ty, TyCtxt};
+use rustc_middle::ty::{self, Lift, ParamEnvAnd, Ty, TyCtxt, TypeVisitableExt};
 use std::fmt;
 
 pub use rustc_middle::traits::query::type_op::Normalize;
@@ -24,7 +24,7 @@ where
     }
 }
 
-pub trait Normalizable<'tcx>: fmt::Debug + TypeFoldable<'tcx> + Lift<'tcx> + Copy {
+pub trait Normalizable<'tcx>: fmt::Debug + TypeFoldable<TyCtxt<'tcx>> + Lift<'tcx> + Copy {
     fn type_op_method(
         tcx: TyCtxt<'tcx>,
         canonicalized: Canonical<'tcx, ParamEnvAnd<'tcx, Normalize<Self>>>,
diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/outlives.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/outlives.rs
index 0d42cd825..21ef4e24f 100644
--- a/compiler/rustc_trait_selection/src/traits/query/type_op/outlives.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/type_op/outlives.rs
@@ -21,11 +21,7 @@ impl<'tcx> super::QueryTypeOp<'tcx> for DropckOutlives<'tcx> {
         tcx: TyCtxt<'tcx>,
         key: &ParamEnvAnd<'tcx, Self>,
     ) -> Option<Self::QueryResponse> {
-        if trivial_dropck_outlives(tcx, key.value.dropped_ty) {
-            Some(DropckOutlivesResult::default())
-        } else {
-            None
-        }
+        trivial_dropck_outlives(tcx, key.value.dropped_ty).then(DropckOutlivesResult::default)
     }
 
     fn perform_query(
diff --git a/compiler/rustc_trait_selection/src/traits/relationships.rs b/compiler/rustc_trait_selection/src/traits/relationships.rs
deleted file mode 100644
index 34b5fc489..000000000
--- a/compiler/rustc_trait_selection/src/traits/relationships.rs
+++ /dev/null
@@ -1,48 +0,0 @@
-use crate::infer::InferCtxt;
-use crate::traits::query::evaluate_obligation::InferCtxtExt;
-use crate::traits::PredicateObligation;
-use rustc_infer::traits::TraitEngine;
-use rustc_middle::ty;
-
-pub(crate) fn update<'tcx, T>(
-    engine: &mut T,
-    infcx: &InferCtxt<'tcx>,
-    obligation: &PredicateObligation<'tcx>,
-) where
-    T: TraitEngine<'tcx>,
-{
-    // (*) binder skipped
-    if let ty::PredicateKind::Clause(ty::Clause::Trait(tpred)) = obligation.predicate.kind().skip_binder()
-        && let Some(ty) = infcx.shallow_resolve(tpred.self_ty()).ty_vid().map(|t| infcx.root_var(t))
-        && infcx.tcx.lang_items().sized_trait().map_or(false, |st| st != tpred.trait_ref.def_id)
-    {
-        let new_self_ty = infcx.tcx.types.unit;
-
-        // Then construct a new obligation with Self = () added
-        // to the ParamEnv, and see if it holds.
-        let o = obligation.with(infcx.tcx,
-            obligation
-                .predicate
-                .kind()
-                .rebind(
-                    // (*) binder moved here
-                    ty::PredicateKind::Clause(ty::Clause::Trait(tpred.with_self_ty(infcx.tcx, new_self_ty)))
-                ),
-        );
-        // Don't report overflow errors. Otherwise equivalent to may_hold.
-        if let Ok(result) = infcx.probe(|_| infcx.evaluate_obligation(&o)) && result.may_apply() {
-            engine.relationships().entry(ty).or_default().self_in_trait = true;
-        }
-    }
-
-    if let ty::PredicateKind::Clause(ty::Clause::Projection(predicate)) =
-        obligation.predicate.kind().skip_binder()
-    {
-        // If the projection predicate (Foo::Bar == X) has X as a non-TyVid,
-        // we need to make it into one.
-        if let Some(vid) = predicate.term.ty().and_then(|ty| ty.ty_vid()) {
-            debug!("relationship: {:?}.output = true", vid);
-            engine.relationships().entry(vid).or_default().output = true;
-        }
-    }
-}
diff --git a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
index 2733d9643..e91057356 100644
--- a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
@@ -9,7 +9,7 @@ use hir::LangItem;
 use rustc_hir as hir;
 use rustc_infer::traits::ObligationCause;
 use rustc_infer::traits::{Obligation, SelectionError, TraitObligation};
-use rustc_middle::ty::{self, Ty, TypeVisitable};
+use rustc_middle::ty::{self, Ty, TypeVisitableExt};
 use rustc_target::spec::abi::Abi;
 
 use crate::traits;
@@ -94,7 +94,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 self.assemble_candidates_for_transmutability(obligation, &mut candidates);
             } else if lang_items.tuple_trait() == Some(def_id) {
                 self.assemble_candidate_for_tuple(obligation, &mut candidates);
-            } else if lang_items.pointer_sized() == Some(def_id) {
+            } else if lang_items.pointer_like() == Some(def_id) {
                 self.assemble_candidate_for_ptr_sized(obligation, &mut candidates);
             } else {
                 if lang_items.clone_trait() == Some(def_id) {
@@ -174,8 +174,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             .param_env
             .caller_bounds()
             .iter()
-            .filter_map(|p| p.to_opt_poly_trait_pred())
-            .filter(|p| !p.references_error());
+            .filter(|p| !p.references_error())
+            .filter_map(|p| p.to_opt_poly_trait_pred());
 
         // Micro-optimization: filter out predicates relating to different traits.
         let matching_bounds =
@@ -339,7 +339,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
 
         // Essentially any user-written impl will match with an error type,
         // so creating `ImplCandidates` isn't useful. However, we might
-        // end up finding a candidate elsewhere (e.g. a `BuiltinCandidate` for `Sized)
+        // end up finding a candidate elsewhere (e.g. a `BuiltinCandidate` for `Sized`)
         // This helps us avoid overflow: see issue #72839
         // Since compilation is already guaranteed to fail, this is just
         // to try to show the 'nicest' possible errors to the user.
@@ -396,7 +396,10 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     // still be provided by a manual implementation for
                     // this trait and type.
                 }
-                ty::Param(..) | ty::Alias(ty::Projection, ..) => {
+                ty::Param(..)
+                | ty::Alias(ty::Projection, ..)
+                | ty::Placeholder(..)
+                | ty::Bound(..) => {
                     // In these cases, we don't know what the actual
                     // type is. Therefore, we cannot break it down
                     // into its constituent types. So we don't
@@ -448,6 +451,10 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         );
 
         self.infcx.probe(|_snapshot| {
+            if obligation.has_non_region_late_bound() {
+                return;
+            }
+
             // The code below doesn't care about regions, and the
             // self-ty here doesn't escape this probe, so just erase
             // any LBR.
@@ -466,7 +473,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     if let Some(principal) = data.principal() {
                         if !self.infcx.tcx.features().object_safe_for_dispatch {
                             principal.with_self_ty(self.tcx(), self_ty)
-                        } else if self.tcx().is_object_safe(principal.def_id()) {
+                        } else if self.tcx().check_is_object_safe(principal.def_id()) {
                             principal.with_self_ty(self.tcx(), self_ty)
                         } else {
                             return;
@@ -488,7 +495,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
 
             let poly_trait_predicate = self.infcx.resolve_vars_if_possible(obligation.predicate);
             let placeholder_trait_predicate =
-                self.infcx.replace_bound_vars_with_placeholders(poly_trait_predicate);
+                self.infcx.instantiate_binder_with_placeholders(poly_trait_predicate);
 
             // Count only those upcast versions that match the trait-ref
             // we are looking for. Specifically, do not only check for the
@@ -765,7 +772,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             | ty::Closure(..)
             | ty::Generator(..)
             | ty::Tuple(_)
-            | ty::GeneratorWitness(_) => {
+            | ty::GeneratorWitness(_)
+            | ty::GeneratorWitnessMIR(..) => {
                 // These are built-in, and cannot have a custom `impl const Destruct`.
                 candidates.vec.push(ConstDestructCandidate(None));
             }
@@ -826,6 +834,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             | ty::Closure(_, _)
             | ty::Generator(_, _, _)
             | ty::GeneratorWitness(_)
+            | ty::GeneratorWitnessMIR(..)
             | ty::Never
             | ty::Alias(..)
             | ty::Param(_)
diff --git a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
index 82a59831b..21c158fd0 100644
--- a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
@@ -8,12 +8,11 @@
 //! https://rustc-dev-guide.rust-lang.org/traits/resolution.html#confirmation
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir::lang_items::LangItem;
-use rustc_index::bit_set::GrowableBitSet;
 use rustc_infer::infer::InferOk;
 use rustc_infer::infer::LateBoundRegionConversionTime::HigherRankedType;
 use rustc_middle::ty::{
-    self, Binder, GenericArg, GenericArgKind, GenericParamDefKind, InternalSubsts, SubstsRef,
-    ToPolyTraitRef, ToPredicate, TraitRef, Ty, TyCtxt,
+    self, Binder, GenericParamDefKind, InternalSubsts, SubstsRef, ToPolyTraitRef, ToPredicate,
+    TraitRef, Ty, TyCtxt, TypeVisitableExt,
 };
 use rustc_session::config::TraitSolver;
 use rustc_span::def_id::DefId;
@@ -152,7 +151,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
 
         let trait_predicate = self.infcx.shallow_resolve(obligation.predicate);
         let placeholder_trait_predicate =
-            self.infcx.replace_bound_vars_with_placeholders(trait_predicate).trait_ref;
+            self.infcx.instantiate_binder_with_placeholders(trait_predicate).trait_ref;
         let placeholder_self_ty = placeholder_trait_predicate.self_ty();
         let placeholder_trait_predicate = ty::Binder::dummy(placeholder_trait_predicate);
         let (def_id, substs) = match *placeholder_self_ty.kind() {
@@ -337,7 +336,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             let cause = obligation.derived_cause(BuiltinDerivedObligation);
 
             let poly_trait_ref = obligation.predicate.to_poly_trait_ref();
-            let trait_ref = self.infcx.replace_bound_vars_with_placeholders(poly_trait_ref);
+            let trait_ref = self.infcx.instantiate_binder_with_placeholders(poly_trait_ref);
             let trait_obligations: Vec<PredicateObligation<'_>> = self.impl_or_trait_obligations(
                 &cause,
                 obligation.recursion_depth + 1,
@@ -428,7 +427,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         let tcx = self.tcx();
         debug!(?obligation, ?index, "confirm_object_candidate");
 
-        let trait_predicate = self.infcx.replace_bound_vars_with_placeholders(obligation.predicate);
+        let trait_predicate = self.infcx.instantiate_binder_with_placeholders(obligation.predicate);
         let self_ty = self.infcx.shallow_resolve(trait_predicate.self_ty());
         let obligation_trait_ref = ty::Binder::dummy(trait_predicate.trait_ref);
         let ty::Dynamic(data, ..) = *self_ty.kind() else {
@@ -438,7 +437,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         let object_trait_ref = data.principal().unwrap_or_else(|| {
             span_bug!(obligation.cause.span, "object candidate with no principal")
         });
-        let object_trait_ref = self.infcx.replace_bound_vars_with_fresh_vars(
+        let object_trait_ref = self.infcx.instantiate_binder_with_fresh_vars(
             obligation.cause.span,
             HigherRankedType,
             object_trait_ref,
@@ -525,29 +524,29 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                             .kind
                         {
                             GenericParamDefKind::Type { .. } => {
-                                let kind = ty::BoundTyKind::Param(param.name);
+                                let kind = ty::BoundTyKind::Param(param.def_id, param.name);
                                 let bound_var = ty::BoundVariableKind::Ty(kind);
                                 bound_vars.push(bound_var);
-                                tcx.mk_ty(ty::Bound(
+                                tcx.mk_bound(
                                     ty::INNERMOST,
                                     ty::BoundTy {
                                         var: ty::BoundVar::from_usize(bound_vars.len() - 1),
                                         kind,
                                     },
-                                ))
+                                )
                                 .into()
                             }
                             GenericParamDefKind::Lifetime => {
                                 let kind = ty::BoundRegionKind::BrNamed(param.def_id, param.name);
                                 let bound_var = ty::BoundVariableKind::Region(kind);
                                 bound_vars.push(bound_var);
-                                tcx.mk_region(ty::ReLateBound(
+                                tcx.mk_re_late_bound(
                                     ty::INNERMOST,
                                     ty::BoundRegion {
                                         var: ty::BoundVar::from_usize(bound_vars.len() - 1),
                                         kind,
                                     },
-                                ))
+                                )
                                 .into()
                             }
                             GenericParamDefKind::Const { .. } => {
@@ -558,15 +557,15 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                                         ty::INNERMOST,
                                         ty::BoundVar::from_usize(bound_vars.len() - 1),
                                     ),
-                                    tcx.type_of(param.def_id),
+                                    tcx.type_of(param.def_id)
+                                        .no_bound_vars()
+                                        .expect("const parameter types cannot be generic"),
                                 )
                                 .into()
                             }
                         });
-                        let bound_vars = tcx.mk_bound_variable_kinds(bound_vars.into_iter());
-                        let assoc_ty_substs = tcx.intern_substs(&substs);
-
-                        let bound_vars = tcx.mk_bound_variable_kinds(bound_vars.into_iter());
+                        let bound_vars = tcx.mk_bound_variable_kinds(&bound_vars);
+                        let assoc_ty_substs = tcx.mk_substs(&substs);
                         let bound =
                             bound.map_bound(|b| b.kind().skip_binder()).subst(tcx, assoc_ty_substs);
                         tcx.mk_predicate(ty::Binder::bind_with_vars(bound, bound_vars))
@@ -630,7 +629,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         }
 
         // Confirm the `type Output: Sized;` bound that is present on `FnOnce`
-        let output_ty = self.infcx.replace_bound_vars_with_placeholders(sig.output());
+        let output_ty = self.infcx.instantiate_binder_with_placeholders(sig.output());
         let output_ty = normalize_with_depth_to(
             self,
             obligation.param_env,
@@ -653,7 +652,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         debug!(?obligation, "confirm_trait_alias_candidate");
 
         let alias_def_id = obligation.predicate.def_id();
-        let predicate = self.infcx.replace_bound_vars_with_placeholders(obligation.predicate);
+        let predicate = self.infcx.instantiate_binder_with_placeholders(obligation.predicate);
         let trait_ref = predicate.trait_ref;
         let trait_def_id = trait_ref.def_id;
         let substs = trait_ref.substs;
@@ -822,6 +821,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
 
         self.infcx
             .at(&obligation.cause, obligation.param_env)
+            // needed for tests/ui/type-alias-impl-trait/assoc-projection-ice.rs
+            .define_opaque_types(true)
             .sup(obligation_trait_ref, expected_trait_ref)
             .map(|InferOk { mut obligations, .. }| {
                 obligations.extend(nested);
@@ -879,7 +880,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                             .map(ty::ExistentialPredicate::AutoTrait)
                             .map(ty::Binder::dummy),
                     );
-                let existential_predicates = tcx.mk_poly_existential_predicates(iter);
+                let existential_predicates = tcx.mk_poly_existential_predicates_from_iter(iter);
                 let source_trait = tcx.mk_dynamic(existential_predicates, r_b, repr_a);
 
                 // Require that the traits involved in this upcast are **equal**;
@@ -978,7 +979,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                             .map(ty::ExistentialPredicate::AutoTrait)
                             .map(ty::Binder::dummy),
                     );
-                let existential_predicates = tcx.mk_poly_existential_predicates(iter);
+                let existential_predicates = tcx.mk_poly_existential_predicates_from_iter(iter);
                 let source_trait = tcx.mk_dynamic(existential_predicates, r_b, dyn_a);
 
                 // Require that the traits involved in this upcast are **equal**;
@@ -1009,7 +1010,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             // `T` -> `Trait`
             (_, &ty::Dynamic(ref data, r, ty::Dyn)) => {
                 let mut object_dids = data.auto_traits().chain(data.principal_def_id());
-                if let Some(did) = object_dids.find(|did| !tcx.is_object_safe(*did)) {
+                if let Some(did) = object_dids.find(|did| !tcx.check_is_object_safe(*did)) {
                     return Err(TraitNotObjectSafe(did));
                 }
 
@@ -1064,51 +1065,18 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
 
             // `Struct<T>` -> `Struct<U>`
             (&ty::Adt(def, substs_a), &ty::Adt(_, substs_b)) => {
-                let maybe_unsizing_param_idx = |arg: GenericArg<'tcx>| match arg.unpack() {
-                    GenericArgKind::Type(ty) => match ty.kind() {
-                        ty::Param(p) => Some(p.index),
-                        _ => None,
-                    },
-
-                    // Lifetimes aren't allowed to change during unsizing.
-                    GenericArgKind::Lifetime(_) => None,
-
-                    GenericArgKind::Const(ct) => match ct.kind() {
-                        ty::ConstKind::Param(p) => Some(p.index),
-                        _ => None,
-                    },
-                };
-
-                // FIXME(eddyb) cache this (including computing `unsizing_params`)
-                // by putting it in a query; it would only need the `DefId` as it
-                // looks at declared field types, not anything substituted.
-
-                // The last field of the structure has to exist and contain type/const parameters.
-                let (tail_field, prefix_fields) =
-                    def.non_enum_variant().fields.split_last().ok_or(Unimplemented)?;
-                let tail_field_ty = tcx.bound_type_of(tail_field.did);
-
-                let mut unsizing_params = GrowableBitSet::new_empty();
-                for arg in tail_field_ty.0.walk() {
-                    if let Some(i) = maybe_unsizing_param_idx(arg) {
-                        unsizing_params.insert(i);
-                    }
-                }
-
-                // Ensure none of the other fields mention the parameters used
-                // in unsizing.
-                for field in prefix_fields {
-                    for arg in tcx.type_of(field.did).walk() {
-                        if let Some(i) = maybe_unsizing_param_idx(arg) {
-                            unsizing_params.remove(i);
-                        }
-                    }
-                }
-
+                let unsizing_params = tcx.unsizing_params_for_adt(def.did());
                 if unsizing_params.is_empty() {
                     return Err(Unimplemented);
                 }
 
+                let tail_field = def
+                    .non_enum_variant()
+                    .fields
+                    .last()
+                    .expect("expected unsized ADT to have a tail field");
+                let tail_field_ty = tcx.type_of(tail_field.did);
+
                 // Extract `TailField<T>` and `TailField<U>` from `Struct<T>` and `Struct<U>`,
                 // normalizing in the process, since `type_of` returns something directly from
                 // astconv (which means it's un-normalized).
@@ -1131,7 +1099,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
 
                 // Check that the source struct with the target's
                 // unsizing parameters is equal to the target.
-                let substs = tcx.mk_substs(substs_a.iter().enumerate().map(|(i, k)| {
+                let substs = tcx.mk_substs_from_iter(substs_a.iter().enumerate().map(|(i, k)| {
                     if unsizing_params.contains(i as u32) { substs_b[i] } else { k }
                 }));
                 let new_struct = tcx.mk_adt(def, substs);
@@ -1163,7 +1131,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
 
                 // Check that the source tuple with the target's
                 // last element is equal to the target.
-                let new_tuple = tcx.mk_tup(a_mid.iter().copied().chain(iter::once(b_last)));
+                let new_tuple =
+                    tcx.mk_tup_from_iter(a_mid.iter().copied().chain(iter::once(b_last)));
                 let InferOk { obligations, .. } = self
                     .infcx
                     .at(&obligation.cause, obligation.param_env)
@@ -1223,7 +1192,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             let cause = obligation.derived_cause(|derived| {
                 ImplDerivedObligation(Box::new(ImplDerivedObligationCause {
                     derived,
-                    impl_def_id,
+                    impl_or_alias_def_id: impl_def_id,
+                    impl_def_predicate_index: None,
                     span: obligation.cause.span,
                 }))
             });
@@ -1285,6 +1255,14 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 ty::GeneratorWitness(tys) => {
                     stack.extend(tcx.erase_late_bound_regions(tys).to_vec());
                 }
+                ty::GeneratorWitnessMIR(def_id, substs) => {
+                    let tcx = self.tcx();
+                    stack.extend(tcx.generator_hidden_types(def_id).map(|bty| {
+                        let ty = bty.subst(tcx, substs);
+                        debug_assert!(!ty.has_late_bound_regions());
+                        ty
+                    }))
+                }
 
                 // If we have a projection type, make sure to normalize it so we replace it
                 // with a fresh infer variable
diff --git a/compiler/rustc_trait_selection/src/traits/select/mod.rs b/compiler/rustc_trait_selection/src/traits/select/mod.rs
index f90da95d5..7f454fbb3 100644
--- a/compiler/rustc_trait_selection/src/traits/select/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/mod.rs
@@ -27,6 +27,7 @@ use super::{
 
 use crate::infer::{InferCtxt, InferOk, TypeFreshener};
 use crate::traits::error_reporting::TypeErrCtxtExt;
+use crate::traits::project::try_normalize_with_depth_to;
 use crate::traits::project::ProjectAndUnifyResult;
 use crate::traits::project::ProjectionCacheKeyExt;
 use crate::traits::ProjectionCacheKey;
@@ -38,6 +39,8 @@ use rustc_errors::Diagnostic;
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 use rustc_infer::infer::LateBoundRegionConversionTime;
+use rustc_infer::traits::TraitEngine;
+use rustc_infer::traits::TraitEngineExt;
 use rustc_middle::dep_graph::{DepKind, DepNodeIndex};
 use rustc_middle::mir::interpret::ErrorHandled;
 use rustc_middle::ty::abstract_const::NotConstEvaluatable;
@@ -46,7 +49,8 @@ use rustc_middle::ty::fold::BottomUpFolder;
 use rustc_middle::ty::relate::TypeRelation;
 use rustc_middle::ty::SubstsRef;
 use rustc_middle::ty::{self, EarlyBinder, PolyProjectionPredicate, ToPolyTraitRef, ToPredicate};
-use rustc_middle::ty::{Ty, TyCtxt, TypeFoldable, TypeVisitable};
+use rustc_middle::ty::{Ty, TyCtxt, TypeFoldable, TypeVisitableExt};
+use rustc_session::config::TraitSolver;
 use rustc_span::symbol::sym;
 
 use std::cell::{Cell, RefCell};
@@ -147,7 +151,7 @@ struct TraitObligationStack<'prev, 'tcx> {
     /// you don't want to cache that `B: AutoTrait` or `A: AutoTrait`
     /// is `EvaluatedToOk`; this is because they were only considered
     /// ok on the premise that if `A: AutoTrait` held, but we indeed
-    /// encountered a problem (later on) with `A: AutoTrait. So we
+    /// encountered a problem (later on) with `A: AutoTrait`. So we
     /// currently set a flag on the stack node for `B: AutoTrait` (as
     /// well as the second instance of `A: AutoTrait`) to suppress
     /// caching.
@@ -374,11 +378,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                             let self_ty = trait_ref.self_ty();
                             let (trait_desc, self_desc) = with_no_trimmed_paths!({
                                 let trait_desc = trait_ref.print_only_trait_path().to_string();
-                                let self_desc = if self_ty.has_concrete_skeleton() {
-                                    Some(self_ty.to_string())
-                                } else {
-                                    None
-                                };
+                                let self_desc =
+                                    self_ty.has_concrete_skeleton().then(|| self_ty.to_string());
                                 (trait_desc, self_desc)
                             });
                             let cause = if let Conflict::Upstream = conflict {
@@ -544,10 +545,14 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         obligation: &PredicateObligation<'tcx>,
     ) -> Result<EvaluationResult, OverflowError> {
         self.evaluation_probe(|this| {
-            this.evaluate_predicate_recursively(
-                TraitObligationStackList::empty(&ProvisionalEvaluationCache::default()),
-                obligation.clone(),
-            )
+            if this.tcx().sess.opts.unstable_opts.trait_solver != TraitSolver::Next {
+                this.evaluate_predicate_recursively(
+                    TraitObligationStackList::empty(&ProvisionalEvaluationCache::default()),
+                    obligation.clone(),
+                )
+            } else {
+                this.evaluate_predicates_recursively_in_new_solver([obligation.clone()])
+            }
         })
     }
 
@@ -586,18 +591,40 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
     where
         I: IntoIterator<Item = PredicateObligation<'tcx>> + std::fmt::Debug,
     {
-        let mut result = EvaluatedToOk;
-        for obligation in predicates {
-            let eval = self.evaluate_predicate_recursively(stack, obligation.clone())?;
-            if let EvaluatedToErr = eval {
-                // fast-path - EvaluatedToErr is the top of the lattice,
-                // so we don't need to look on the other predicates.
-                return Ok(EvaluatedToErr);
-            } else {
-                result = cmp::max(result, eval);
+        if self.tcx().sess.opts.unstable_opts.trait_solver != TraitSolver::Next {
+            let mut result = EvaluatedToOk;
+            for obligation in predicates {
+                let eval = self.evaluate_predicate_recursively(stack, obligation.clone())?;
+                if let EvaluatedToErr = eval {
+                    // fast-path - EvaluatedToErr is the top of the lattice,
+                    // so we don't need to look on the other predicates.
+                    return Ok(EvaluatedToErr);
+                } else {
+                    result = cmp::max(result, eval);
+                }
             }
+            Ok(result)
+        } else {
+            self.evaluate_predicates_recursively_in_new_solver(predicates)
         }
-        Ok(result)
+    }
+
+    /// Evaluates the predicates using the new solver when `-Ztrait-solver=next` is enabled
+    fn evaluate_predicates_recursively_in_new_solver(
+        &mut self,
+        predicates: impl IntoIterator<Item = PredicateObligation<'tcx>>,
+    ) -> Result<EvaluationResult, OverflowError> {
+        let mut fulfill_cx = crate::solve::FulfillmentCtxt::new();
+        fulfill_cx.register_predicate_obligations(self.infcx, predicates);
+        // True errors
+        if !fulfill_cx.select_where_possible(self.infcx).is_empty() {
+            return Ok(EvaluatedToErr);
+        }
+        if !fulfill_cx.select_all_or_error(self.infcx).is_empty() {
+            return Ok(EvaluatedToAmbig);
+        }
+        // Regions and opaques are handled in the `evaluation_probe` by looking at the snapshot
+        Ok(EvaluatedToOk)
     }
 
     #[instrument(
@@ -700,7 +727,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                             // Otherwise, we can say that `T: NonAutoTrait` is
                             // true.
                             // Let's imagine we have a predicate stack like
-                            //         `Foo: Bar -> WF(T) -> T: NonAutoTrait -> T: Auto
+                            //         `Foo: Bar -> WF(T) -> T: NonAutoTrait -> T: Auto`
                             // depth   ^1                    ^2                 ^3
                             // and the current predicate is `WF(T)`. `wf_args`
                             // would contain `(T, 1)`. We want to check all
@@ -768,7 +795,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 }
 
                 ty::PredicateKind::ObjectSafe(trait_def_id) => {
-                    if self.tcx().is_object_safe(trait_def_id) {
+                    if self.tcx().check_is_object_safe(trait_def_id) {
                         Ok(EvaluatedToOk)
                     } else {
                         Ok(EvaluatedToErr)
@@ -962,7 +989,19 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 ty::PredicateKind::TypeWellFormedFromEnv(..) => {
                     bug!("TypeWellFormedFromEnv is only used for chalk")
                 }
+                ty::PredicateKind::AliasEq(..) => {
+                    bug!("AliasEq is only used for new solver")
+                }
                 ty::PredicateKind::Ambiguous => Ok(EvaluatedToAmbig),
+                ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(ct, ty)) => {
+                    match self.infcx.at(&obligation.cause, obligation.param_env).eq(ct.ty(), ty) {
+                        Ok(inf_ok) => self.evaluate_predicates_recursively(
+                            previous_stack,
+                            inf_ok.into_obligations(),
+                        ),
+                        Err(_) => Ok(EvaluatedToErr),
+                    }
+                }
             }
         })
     }
@@ -1017,7 +1056,51 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             return Ok(cycle_result);
         }
 
-        let (result, dep_node) = self.in_task(|this| this.evaluate_stack(&stack));
+        let (result, dep_node) = self.in_task(|this| {
+            let mut result = this.evaluate_stack(&stack)?;
+
+            // fix issue #103563, we don't normalize
+            // nested obligations which produced by `TraitDef` candidate
+            // (i.e. using bounds on assoc items as assumptions).
+            // because we don't have enough information to
+            // normalize these obligations before evaluating.
+            // so we will try to normalize the obligation and evaluate again.
+            // we will replace it with new solver in the future.
+            if EvaluationResult::EvaluatedToErr == result
+                && fresh_trait_pred.has_projections()
+                && fresh_trait_pred.is_global()
+            {
+                let mut nested_obligations = Vec::new();
+                let predicate = try_normalize_with_depth_to(
+                    this,
+                    param_env,
+                    obligation.cause.clone(),
+                    obligation.recursion_depth + 1,
+                    obligation.predicate,
+                    &mut nested_obligations,
+                );
+                if predicate != obligation.predicate {
+                    let mut nested_result = EvaluationResult::EvaluatedToOk;
+                    for obligation in nested_obligations {
+                        nested_result = cmp::max(
+                            this.evaluate_predicate_recursively(previous_stack, obligation)?,
+                            nested_result,
+                        );
+                    }
+
+                    if nested_result.must_apply_modulo_regions() {
+                        let obligation = obligation.with(this.tcx(), predicate);
+                        result = cmp::max(
+                            nested_result,
+                            this.evaluate_trait_predicate_recursively(previous_stack, obligation)?,
+                        );
+                    }
+                }
+            }
+
+            Ok::<_, OverflowError>(result)
+        });
+
         let result = result?;
 
         if !result.must_apply_modulo_regions() {
@@ -1323,7 +1406,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
     /// The weird return type of this function allows it to be used with the `try` (`?`)
     /// operator within certain functions.
     #[inline(always)]
-    fn check_recursion_limit<T: Display + TypeFoldable<'tcx>, V>(
+    fn check_recursion_limit<T: Display + TypeFoldable<TyCtxt<'tcx>>, V>(
         &self,
         obligation: &Obligation<'tcx, T>,
         error_obligation: &Obligation<'tcx, V>,
@@ -1589,7 +1672,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
     ) -> smallvec::SmallVec<[(usize, ty::BoundConstness); 2]> {
         let poly_trait_predicate = self.infcx.resolve_vars_if_possible(obligation.predicate);
         let placeholder_trait_predicate =
-            self.infcx.replace_bound_vars_with_placeholders(poly_trait_predicate);
+            self.infcx.instantiate_binder_with_placeholders(poly_trait_predicate);
         debug!(?placeholder_trait_predicate);
 
         let tcx = self.infcx.tcx;
@@ -1669,7 +1752,6 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         });
         self.infcx
             .at(&obligation.cause, obligation.param_env)
-            .define_opaque_types(false)
             .sup(ty::Binder::dummy(placeholder_trait_ref), trait_bound)
             .map(|InferOk { obligations: _, value: () }| {
                 // This method is called within a probe, so we can't have
@@ -1709,7 +1791,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         potentially_unnormalized_candidates: bool,
     ) -> ProjectionMatchesProjection {
         let mut nested_obligations = Vec::new();
-        let infer_predicate = self.infcx.replace_bound_vars_with_fresh_vars(
+        let infer_predicate = self.infcx.instantiate_binder_with_fresh_vars(
             obligation.cause.span,
             LateBoundRegionConversionTime::HigherRankedType,
             env_predicate,
@@ -1732,7 +1814,6 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         let is_match = self
             .infcx
             .at(&obligation.cause, obligation.param_env)
-            .define_opaque_types(false)
             .sup(obligation.predicate, infer_projection)
             .map_or(false, |InferOk { obligations, value: () }| {
                 self.evaluate_predicates_recursively(
@@ -2037,6 +2118,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             | ty::Ref(..)
             | ty::Generator(..)
             | ty::GeneratorWitness(..)
+            | ty::GeneratorWitnessMIR(..)
             | ty::Array(..)
             | ty::Closure(..)
             | ty::Never
@@ -2064,12 +2146,13 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 }))
             }
 
-            ty::Alias(..) | ty::Param(_) => None,
+            ty::Alias(..) | ty::Param(_) | ty::Placeholder(..) => None,
             ty::Infer(ty::TyVar(_)) => Ambiguous,
 
-            ty::Placeholder(..)
-            | ty::Bound(..)
-            | ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => {
+            // We can make this an ICE if/once we actually instantiate the trait obligation.
+            ty::Bound(..) => None,
+
+            ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => {
                 bug!("asked to assemble builtin bounds of unexpected type: {:?}", self_ty);
             }
         }
@@ -2147,12 +2230,22 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     }
                 }
                 // (*) binder moved here
-                let all_vars = self.tcx().mk_bound_variable_kinds(
+                let all_vars = self.tcx().mk_bound_variable_kinds_from_iter(
                     obligation.predicate.bound_vars().iter().chain(binder.bound_vars().iter()),
                 );
                 Where(ty::Binder::bind_with_vars(witness_tys.to_vec(), all_vars))
             }
 
+            ty::GeneratorWitnessMIR(def_id, ref substs) => {
+                let hidden_types = bind_generator_hidden_types_above(
+                    self.infcx,
+                    def_id,
+                    substs,
+                    obligation.predicate.bound_vars(),
+                );
+                Where(hidden_types)
+            }
+
             ty::Closure(_, substs) => {
                 // (*) binder moved here
                 let ty = self.infcx.shallow_resolve(substs.as_closure().tupled_upvars_ty());
@@ -2207,12 +2300,14 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             | ty::Float(_)
             | ty::FnDef(..)
             | ty::FnPtr(_)
-            | ty::Str
             | ty::Error(_)
             | ty::Infer(ty::IntVar(_) | ty::FloatVar(_))
             | ty::Never
             | ty::Char => ty::Binder::dummy(Vec::new()),
 
+            // Treat this like `struct str([u8]);`
+            ty::Str => ty::Binder::dummy(vec![self.tcx().mk_slice(self.tcx().types.u8)]),
+
             ty::Placeholder(..)
             | ty::Dynamic(..)
             | ty::Param(..)
@@ -2250,6 +2345,10 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 types.map_bound(|types| types.to_vec())
             }
 
+            ty::GeneratorWitnessMIR(def_id, ref substs) => {
+                bind_generator_hidden_types_above(self.infcx, def_id, substs, t.bound_vars())
+            }
+
             // For `PhantomData<T>`, we pass `T`.
             ty::Adt(def, substs) if def.is_phantom_data() => t.rebind(substs.types().collect()),
 
@@ -2261,7 +2360,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 // We can resolve the `impl Trait` to its concrete type,
                 // which enforces a DAG between the functions requiring
                 // the auto trait bounds in question.
-                t.rebind(vec![self.tcx().bound_type_of(def_id).subst(self.tcx(), substs)])
+                t.rebind(vec![self.tcx().type_of(def_id).subst(self.tcx(), substs)])
             }
         }
     }
@@ -2295,7 +2394,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             .flat_map(|ty| {
                 let ty: ty::Binder<'tcx, Ty<'tcx>> = types.rebind(*ty); // <----/
 
-                let placeholder_ty = self.infcx.replace_bound_vars_with_placeholders(ty);
+                let placeholder_ty = self.infcx.instantiate_binder_with_placeholders(ty);
                 let Normalized { value: normalized_ty, mut obligations } =
                     ensure_sufficient_stack(|| {
                         project::normalize_with_depth(
@@ -2346,7 +2445,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 // the placeholder trait ref may fail due the Generalizer relation
                 // raising a CyclicalTy error due to a sub_root_var relation
                 // for a variable being generalized...
-                self.infcx.tcx.sess.delay_span_bug(
+                let guar = self.infcx.tcx.sess.delay_span_bug(
                     obligation.cause.span,
                     &format!(
                         "Impl {:?} was matchable against {:?} but now is not",
@@ -2354,7 +2453,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     ),
                 );
                 let value = self.infcx.fresh_substs_for_item(obligation.cause.span, impl_def_id);
-                let err = self.tcx().ty_error();
+                let err = self.tcx().ty_error(guar);
                 let value = value.fold_with(&mut BottomUpFolder {
                     tcx: self.tcx(),
                     ty_op: |_| err,
@@ -2374,7 +2473,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         obligation: &TraitObligation<'tcx>,
     ) -> Result<Normalized<'tcx, SubstsRef<'tcx>>, ()> {
         let placeholder_obligation =
-            self.infcx.replace_bound_vars_with_placeholders(obligation.predicate);
+            self.infcx.instantiate_binder_with_placeholders(obligation.predicate);
         let placeholder_obligation_trait_ref = placeholder_obligation.trait_ref;
 
         let impl_substs = self.infcx.fresh_substs_for_item(obligation.cause.span, impl_def_id);
@@ -2408,9 +2507,10 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         let InferOk { obligations, .. } = self
             .infcx
             .at(&cause, obligation.param_env)
-            .define_opaque_types(false)
             .eq(placeholder_obligation_trait_ref, impl_trait_ref)
-            .map_err(|e| debug!("match_impl: failed eq_trait_refs due to `{e}`"))?;
+            .map_err(|e| {
+                debug!("match_impl: failed eq_trait_refs due to `{}`", e.to_string(self.tcx()))
+            })?;
         nested_obligations.extend(obligations);
 
         if !self.is_intercrate()
@@ -2457,11 +2557,6 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
     ) -> Result<Vec<PredicateObligation<'tcx>>, ()> {
         self.infcx
             .at(&obligation.cause, obligation.param_env)
-            // We don't want predicates for opaque types to just match all other types,
-            // if there is an obligation on the opaque type, then that obligation must be met
-            // opaquely. Otherwise we'd match any obligation to the opaque type and then error
-            // out later.
-            .define_opaque_types(false)
             .sup(obligation.predicate.to_poly_trait_ref(), poly_trait_ref)
             .map(|InferOk { obligations, .. }| obligations)
             .map_err(|_| ())
@@ -2562,11 +2657,12 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         assert_eq!(predicates.parent, None);
         let predicates = predicates.instantiate_own(tcx, substs);
         let mut obligations = Vec::with_capacity(predicates.len());
-        for (predicate, span) in predicates {
+        for (index, (predicate, span)) in predicates.into_iter().enumerate() {
             let cause = cause.clone().derived_cause(parent_trait_pred, |derived| {
                 ImplDerivedObligation(Box::new(ImplDerivedObligationCause {
                     derived,
-                    impl_def_id: def_id,
+                    impl_or_alias_def_id: def_id,
+                    impl_def_predicate_index: Some(index),
                     span,
                 }))
             });
@@ -2892,3 +2988,56 @@ pub enum ProjectionMatchesProjection {
     Ambiguous,
     No,
 }
+
+/// Replace all regions inside the generator interior with late bound regions.
+/// Note that each region slot in the types gets a new fresh late bound region, which means that
+/// none of the regions inside relate to any other, even if typeck had previously found constraints
+/// that would cause them to be related.
+#[instrument(level = "trace", skip(infcx), ret)]
+fn bind_generator_hidden_types_above<'tcx>(
+    infcx: &InferCtxt<'tcx>,
+    def_id: DefId,
+    substs: ty::SubstsRef<'tcx>,
+    bound_vars: &ty::List<ty::BoundVariableKind>,
+) -> ty::Binder<'tcx, Vec<Ty<'tcx>>> {
+    let tcx = infcx.tcx;
+    let mut seen_tys = FxHashSet::default();
+
+    let considering_regions = infcx.considering_regions;
+
+    let num_bound_variables = bound_vars.len() as u32;
+    let mut counter = num_bound_variables;
+
+    let hidden_types: Vec<_> = tcx
+        .generator_hidden_types(def_id)
+        // Deduplicate tys to avoid repeated work.
+        .filter(|bty| seen_tys.insert(*bty))
+        .map(|bty| {
+            let mut ty = bty.subst(tcx, substs);
+
+            // Only remap erased regions if we use them.
+            if considering_regions {
+                ty = tcx.fold_regions(ty, |mut r, current_depth| {
+                    if let ty::ReErased = r.kind() {
+                        let br = ty::BoundRegion {
+                            var: ty::BoundVar::from_u32(counter),
+                            kind: ty::BrAnon(counter, None),
+                        };
+                        counter += 1;
+                        r = tcx.mk_re_late_bound(current_depth, br);
+                    }
+                    r
+                })
+            }
+
+            ty
+        })
+        .collect();
+    if considering_regions {
+        debug_assert!(!hidden_types.has_erased_regions());
+    }
+    let bound_vars = tcx.mk_bound_variable_kinds_from_iter(bound_vars.iter().chain(
+        (num_bound_variables..counter).map(|i| ty::BoundVariableKind::Region(ty::BrAnon(i, None))),
+    ));
+    ty::Binder::bind_with_vars(hidden_types, bound_vars)
+}
diff --git a/compiler/rustc_trait_selection/src/traits/specialize/mod.rs b/compiler/rustc_trait_selection/src/traits/specialize/mod.rs
index 3b796c623..d1d6a7a90 100644
--- a/compiler/rustc_trait_selection/src/traits/specialize/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/specialize/mod.rs
@@ -455,7 +455,13 @@ pub(crate) fn to_pretty_impl_header(tcx: TyCtxt<'_>, impl_def_id: DefId) -> Opti
         w.push('>');
     }
 
-    write!(w, " {} for {}", trait_ref.print_only_trait_path(), tcx.type_of(impl_def_id)).unwrap();
+    write!(
+        w,
+        " {} for {}",
+        trait_ref.print_only_trait_path(),
+        tcx.type_of(impl_def_id).subst_identity()
+    )
+    .unwrap();
 
     // The predicates will contain default bounds like `T: Sized`. We need to
     // remove these bounds, and add `T: ?Sized` to any untouched type parameters.
diff --git a/compiler/rustc_trait_selection/src/traits/specialize/specialization_graph.rs b/compiler/rustc_trait_selection/src/traits/specialize/specialization_graph.rs
index 0f9196de4..61ed9ef2e 100644
--- a/compiler/rustc_trait_selection/src/traits/specialize/specialization_graph.rs
+++ b/compiler/rustc_trait_selection/src/traits/specialize/specialization_graph.rs
@@ -4,7 +4,7 @@ use crate::traits;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def_id::DefId;
 use rustc_middle::ty::fast_reject::{self, SimplifiedType, TreatParams};
-use rustc_middle::ty::{self, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, TyCtxt, TypeVisitableExt};
 
 pub use rustc_middle::traits::specialization_graph::*;
 
@@ -113,7 +113,7 @@ impl<'tcx> ChildrenExt<'tcx> for Children {
                     // Only report the `Self` type if it has at least
                     // some outer concrete shell; otherwise, it's
                     // not adding much information.
-                    self_ty: if self_ty.has_concrete_skeleton() { Some(self_ty) } else { None },
+                    self_ty: self_ty.has_concrete_skeleton().then_some(self_ty),
                     intercrate_ambiguity_causes: overlap.intercrate_ambiguity_causes,
                     involves_placeholder: overlap.involves_placeholder,
                 }
@@ -399,7 +399,7 @@ pub(crate) fn assoc_def(
     // If there is no such item in that impl, this function will fail with a
     // cycle error if the specialization graph is currently being built.
     if let Some(&impl_item_id) = tcx.impl_item_implementor_ids(impl_def_id).get(&assoc_def_id) {
-        let &item = tcx.associated_item(impl_item_id);
+        let item = tcx.associated_item(impl_item_id);
         let impl_node = Node::Impl(impl_def_id);
         return Ok(LeafDef {
             item,
diff --git a/compiler/rustc_trait_selection/src/traits/structural_match.rs b/compiler/rustc_trait_selection/src/traits/structural_match.rs
index f398fb06c..e38ae9381 100644
--- a/compiler/rustc_trait_selection/src/traits/structural_match.rs
+++ b/compiler/rustc_trait_selection/src/traits/structural_match.rs
@@ -78,7 +78,7 @@ impl<'tcx> Search<'tcx> {
     }
 }
 
-impl<'tcx> TypeVisitor<'tcx> for Search<'tcx> {
+impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for Search<'tcx> {
     type BreakTy = Ty<'tcx>;
 
     fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
@@ -101,7 +101,7 @@ impl<'tcx> TypeVisitor<'tcx> for Search<'tcx> {
             ty::Closure(..) => {
                 return ControlFlow::Break(ty);
             }
-            ty::Generator(..) | ty::GeneratorWitness(..) => {
+            ty::Generator(..) | ty::GeneratorWitness(..) | ty::GeneratorWitnessMIR(..) => {
                 return ControlFlow::Break(ty);
             }
             ty::FnDef(..) => {
@@ -110,7 +110,7 @@ impl<'tcx> TypeVisitor<'tcx> for Search<'tcx> {
                 return ControlFlow::Continue(());
             }
             ty::Array(_, n)
-                if { n.try_eval_usize(self.tcx, ty::ParamEnv::reveal_all()) == Some(0) } =>
+                if { n.try_eval_target_usize(self.tcx, ty::ParamEnv::reveal_all()) == Some(0) } =>
             {
                 // rust-lang/rust#62336: ignore type of contents
                 // for empty array.
diff --git a/compiler/rustc_trait_selection/src/traits/util.rs b/compiler/rustc_trait_selection/src/traits/util.rs
index b5df583e3..bcf63d5a6 100644
--- a/compiler/rustc_trait_selection/src/traits/util.rs
+++ b/compiler/rustc_trait_selection/src/traits/util.rs
@@ -4,7 +4,7 @@ use smallvec::SmallVec;
 
 use rustc_data_structures::fx::FxHashSet;
 use rustc_hir::def_id::DefId;
-use rustc_middle::ty::{self, ImplSubject, ToPredicate, Ty, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, ImplSubject, ToPredicate, Ty, TyCtxt, TypeVisitableExt};
 use rustc_middle::ty::{GenericArg, SubstsRef};
 
 use super::NormalizeExt;
@@ -239,7 +239,7 @@ pub fn predicate_for_trait_def<'tcx>(
     cause: ObligationCause<'tcx>,
     trait_def_id: DefId,
     recursion_depth: usize,
-    params: impl IntoIterator<Item = impl Into<GenericArg<'tcx>>>,
+    params: impl IntoIterator<Item: Into<GenericArg<'tcx>>>,
 ) -> PredicateObligation<'tcx> {
     let trait_ref = tcx.mk_trait_ref(trait_def_id, params);
     predicate_for_trait_ref(tcx, cause, param_env, trait_ref, recursion_depth)
@@ -292,7 +292,7 @@ pub fn closure_trait_ref_and_return_type<'tcx>(
     assert!(!self_ty.has_escaping_bound_vars());
     let arguments_tuple = match tuple_arguments {
         TupleArgumentsFlag::No => sig.skip_binder().inputs()[0],
-        TupleArgumentsFlag::Yes => tcx.intern_tup(sig.skip_binder().inputs()),
+        TupleArgumentsFlag::Yes => tcx.mk_tup(sig.skip_binder().inputs()),
     };
     let trait_ref = tcx.mk_trait_ref(fn_trait_def_id, [self_ty, arguments_tuple]);
     sig.map_bound(|sig| (trait_ref, sig.output()))
diff --git a/compiler/rustc_trait_selection/src/traits/vtable.rs b/compiler/rustc_trait_selection/src/traits/vtable.rs
index 64daca714..a4e9928f8 100644
--- a/compiler/rustc_trait_selection/src/traits/vtable.rs
+++ b/compiler/rustc_trait_selection/src/traits/vtable.rs
@@ -4,7 +4,7 @@ use rustc_hir::def_id::DefId;
 use rustc_hir::lang_items::LangItem;
 use rustc_infer::traits::util::PredicateSet;
 use rustc_infer::traits::ImplSource;
-use rustc_middle::ty::visit::TypeVisitable;
+use rustc_middle::ty::visit::TypeVisitableExt;
 use rustc_middle::ty::InternalSubsts;
 use rustc_middle::ty::{self, GenericParamDefKind, ToPredicate, Ty, TyCtxt, VtblEntry};
 use rustc_span::{sym, Span};
@@ -197,12 +197,12 @@ fn own_existential_vtable_entries(tcx: TyCtxt<'_>, trait_def_id: DefId) -> &[Def
         .in_definition_order()
         .filter(|item| item.kind == ty::AssocKind::Fn);
     // Now list each method's DefId (for within its trait).
-    let own_entries = trait_methods.filter_map(move |trait_method| {
+    let own_entries = trait_methods.filter_map(move |&trait_method| {
         debug!("own_existential_vtable_entry: trait_method={:?}", trait_method);
         let def_id = trait_method.def_id;
 
         // Some methods cannot be called on an object; skip those.
-        if !is_vtable_safe_method(tcx, trait_def_id, &trait_method) {
+        if !is_vtable_safe_method(tcx, trait_def_id, trait_method) {
             debug!("own_existential_vtable_entry: not vtable safe");
             return None;
         }
diff --git a/compiler/rustc_trait_selection/src/traits/wf.rs b/compiler/rustc_trait_selection/src/traits/wf.rs
index 12d4cb4fc..d498af359 100644
--- a/compiler/rustc_trait_selection/src/traits/wf.rs
+++ b/compiler/rustc_trait_selection/src/traits/wf.rs
@@ -1,11 +1,11 @@
 use crate::infer::InferCtxt;
 use crate::traits;
 use rustc_hir as hir;
-use rustc_hir::def_id::DefId;
 use rustc_hir::lang_items::LangItem;
 use rustc_middle::ty::subst::{GenericArg, GenericArgKind, SubstsRef};
-use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitable};
-use rustc_span::Span;
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitableExt};
+use rustc_span::def_id::{DefId, LocalDefId, CRATE_DEF_ID};
+use rustc_span::{Span, DUMMY_SP};
 
 use std::iter;
 /// Returns the set of obligations needed to make `arg` well-formed.
@@ -17,7 +17,7 @@ use std::iter;
 pub fn obligations<'tcx>(
     infcx: &InferCtxt<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
-    body_id: hir::HirId,
+    body_id: LocalDefId,
     recursion_depth: usize,
     arg: GenericArg<'tcx>,
     span: Span,
@@ -75,6 +75,34 @@ pub fn obligations<'tcx>(
     Some(result)
 }
 
+/// Compute the predicates that are required for a type to be well-formed.
+///
+/// This is only intended to be used in the new solver, since it does not
+/// take into account recursion depth or proper error-reporting spans.
+pub fn unnormalized_obligations<'tcx>(
+    infcx: &InferCtxt<'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
+    arg: GenericArg<'tcx>,
+) -> Option<Vec<traits::PredicateObligation<'tcx>>> {
+    if let ty::GenericArgKind::Lifetime(..) = arg.unpack() {
+        return Some(vec![]);
+    }
+
+    debug_assert_eq!(arg, infcx.resolve_vars_if_possible(arg));
+
+    let mut wf = WfPredicates {
+        tcx: infcx.tcx,
+        param_env,
+        body_id: CRATE_DEF_ID,
+        span: DUMMY_SP,
+        out: vec![],
+        recursion_depth: 0,
+        item: None,
+    };
+    wf.compute(arg);
+    Some(wf.out)
+}
+
 /// Returns the obligations that make this trait reference
 /// well-formed. For example, if there is a trait `Set` defined like
 /// `trait Set<K:Eq>`, then the trait reference `Foo: Set<Bar>` is WF
@@ -82,7 +110,7 @@ pub fn obligations<'tcx>(
 pub fn trait_obligations<'tcx>(
     infcx: &InferCtxt<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
-    body_id: hir::HirId,
+    body_id: LocalDefId,
     trait_pred: &ty::TraitPredicate<'tcx>,
     span: Span,
     item: &'tcx hir::Item<'tcx>,
@@ -105,7 +133,7 @@ pub fn trait_obligations<'tcx>(
 pub fn predicate_obligations<'tcx>(
     infcx: &InferCtxt<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
-    body_id: hir::HirId,
+    body_id: LocalDefId,
     predicate: ty::Predicate<'tcx>,
     span: Span,
 ) -> Vec<traits::PredicateObligation<'tcx>> {
@@ -135,6 +163,10 @@ pub fn predicate_obligations<'tcx>(
                 ty::TermKind::Const(c) => c.into(),
             })
         }
+        ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(ct, ty)) => {
+            wf.compute(ct.into());
+            wf.compute(ty.into());
+        }
         ty::PredicateKind::WellFormed(arg) => {
             wf.compute(arg);
         }
@@ -159,6 +191,9 @@ pub fn predicate_obligations<'tcx>(
         ty::PredicateKind::TypeWellFormedFromEnv(..) => {
             bug!("TypeWellFormedFromEnv is only used for Chalk")
         }
+        ty::PredicateKind::AliasEq(..) => {
+            bug!("We should only wf check where clauses and `AliasEq` is not a `Clause`")
+        }
     }
 
     wf.normalize(infcx)
@@ -167,7 +202,7 @@ pub fn predicate_obligations<'tcx>(
 struct WfPredicates<'tcx> {
     tcx: TyCtxt<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
-    body_id: hir::HirId,
+    body_id: LocalDefId,
     span: Span,
     out: Vec<traits::PredicateObligation<'tcx>>,
     recursion_depth: usize,
@@ -523,6 +558,7 @@ impl<'tcx> WfPredicates<'tcx> {
                 | ty::Error(_)
                 | ty::Str
                 | ty::GeneratorWitness(..)
+                | ty::GeneratorWitnessMIR(..)
                 | ty::Never
                 | ty::Param(_)
                 | ty::Bound(..)
@@ -847,7 +883,7 @@ pub fn object_region_bounds<'tcx>(
     // Since we don't actually *know* the self type for an object,
     // this "open(err)" serves as a kind of dummy standin -- basically
     // a placeholder type.
-    let open_ty = tcx.mk_ty_infer(ty::FreshTy(0));
+    let open_ty = tcx.mk_fresh_ty(0);
 
     let predicates = existential_predicates.iter().filter_map(|predicate| {
         if let ty::ExistentialPredicate::Projection(_) = predicate.skip_binder() {
@@ -890,6 +926,7 @@ pub(crate) fn required_region_bounds<'tcx>(
             match obligation.predicate.kind().skip_binder() {
                 ty::PredicateKind::Clause(ty::Clause::Projection(..))
                 | ty::PredicateKind::Clause(ty::Clause::Trait(..))
+                | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
                 | ty::PredicateKind::Subtype(..)
                 | ty::PredicateKind::Coerce(..)
                 | ty::PredicateKind::WellFormed(..)
@@ -899,6 +936,7 @@ pub(crate) fn required_region_bounds<'tcx>(
                 | ty::PredicateKind::ConstEvaluatable(..)
                 | ty::PredicateKind::ConstEquate(..)
                 | ty::PredicateKind::Ambiguous
+                | ty::PredicateKind::AliasEq(..)
                 | ty::PredicateKind::TypeWellFormedFromEnv(..) => None,
                 ty::PredicateKind::Clause(ty::Clause::TypeOutlives(ty::OutlivesPredicate(
                     ref t,
diff --git a/compiler/rustc_traits/Cargo.toml b/compiler/rustc_traits/Cargo.toml
index a432498ab..eff6fb26d 100644
--- a/compiler/rustc_traits/Cargo.toml
+++ b/compiler/rustc_traits/Cargo.toml
@@ -5,7 +5,6 @@ edition = "2021"
 
 [dependencies]
 tracing = "0.1"
-rustc_attr = { path = "../rustc_attr" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_hir = { path = "../rustc_hir" }
diff --git a/compiler/rustc_traits/src/chalk/db.rs b/compiler/rustc_traits/src/chalk/db.rs
index f146de396..f8c8f744e 100644
--- a/compiler/rustc_traits/src/chalk/db.rs
+++ b/compiler/rustc_traits/src/chalk/db.rs
@@ -246,7 +246,7 @@ impl<'tcx> chalk_solve::RustIrDatabase<RustInterner<'tcx>> for RustIrDatabase<'t
 
         // Grab the ADT and the param we might need to calculate its layout
         let param_env = tcx.param_env(did);
-        let adt_ty = tcx.type_of(did);
+        let adt_ty = tcx.type_of(did).subst_identity();
 
         // The ADT is a 1-zst if it's a ZST and its alignment is 1.
         // Mark the ADT as _not_ a 1-zst if there was a layout error.
@@ -269,7 +269,7 @@ impl<'tcx> chalk_solve::RustIrDatabase<RustInterner<'tcx>> for RustIrDatabase<'t
 
         let where_clauses = self.where_clauses_for(def_id, bound_vars);
 
-        let sig = self.interner.tcx.bound_fn_sig(def_id);
+        let sig = self.interner.tcx.fn_sig(def_id);
         let (inputs_and_output, iobinders, _) = crate::chalk::lowering::collect_bound_vars(
             self.interner,
             self.interner.tcx,
@@ -468,7 +468,7 @@ impl<'tcx> chalk_solve::RustIrDatabase<RustInterner<'tcx>> for RustIrDatabase<'t
         let ty = self
             .interner
             .tcx
-            .bound_type_of(def_id)
+            .type_of(def_id)
             .subst(self.interner.tcx, bound_vars)
             .lower_into(self.interner);
 
@@ -580,7 +580,7 @@ impl<'tcx> chalk_solve::RustIrDatabase<RustInterner<'tcx>> for RustIrDatabase<'t
     }
 
     fn is_object_safe(&self, trait_id: chalk_ir::TraitId<RustInterner<'tcx>>) -> bool {
-        self.interner.tcx.is_object_safe(trait_id.0)
+        self.interner.tcx.check_is_object_safe(trait_id.0)
     }
 
     fn hidden_opaque_type(
@@ -588,10 +588,7 @@ impl<'tcx> chalk_solve::RustIrDatabase<RustInterner<'tcx>> for RustIrDatabase<'t
         _id: chalk_ir::OpaqueTyId<RustInterner<'tcx>>,
     ) -> chalk_ir::Ty<RustInterner<'tcx>> {
         // FIXME(chalk): actually get hidden ty
-        self.interner
-            .tcx
-            .mk_ty(ty::Tuple(self.interner.tcx.intern_type_list(&[])))
-            .lower_into(self.interner)
+        self.interner.tcx.types.unit.lower_into(self.interner)
     }
 
     fn closure_kind(
@@ -721,13 +718,13 @@ impl<'tcx> chalk_ir::UnificationDatabase<RustInterner<'tcx>> for RustIrDatabase<
 fn bound_vars_for_item(tcx: TyCtxt<'_>, def_id: DefId) -> SubstsRef<'_> {
     InternalSubsts::for_item(tcx, def_id, |param, substs| match param.kind {
         ty::GenericParamDefKind::Type { .. } => tcx
-            .mk_ty(ty::Bound(
+            .mk_bound(
                 ty::INNERMOST,
                 ty::BoundTy {
                     var: ty::BoundVar::from(param.index),
-                    kind: ty::BoundTyKind::Param(param.name),
+                    kind: ty::BoundTyKind::Param(param.def_id, param.name),
                 },
-            ))
+            )
             .into(),
 
         ty::GenericParamDefKind::Lifetime => {
@@ -735,13 +732,13 @@ fn bound_vars_for_item(tcx: TyCtxt<'_>, def_id: DefId) -> SubstsRef<'_> {
                 var: ty::BoundVar::from_usize(substs.len()),
                 kind: ty::BrAnon(substs.len() as u32, None),
             };
-            tcx.mk_region(ty::ReLateBound(ty::INNERMOST, br)).into()
+            tcx.mk_re_late_bound(ty::INNERMOST, br).into()
         }
 
         ty::GenericParamDefKind::Const { .. } => tcx
             .mk_const(
                 ty::ConstKind::Bound(ty::INNERMOST, ty::BoundVar::from(param.index)),
-                tcx.type_of(param.def_id),
+                tcx.type_of(param.def_id).subst_identity(),
             )
             .into(),
     })
@@ -772,12 +769,12 @@ struct ReplaceOpaqueTyFolder<'tcx> {
     binder_index: ty::DebruijnIndex,
 }
 
-impl<'tcx> ty::TypeFolder<'tcx> for ReplaceOpaqueTyFolder<'tcx> {
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+impl<'tcx> ty::TypeFolder<TyCtxt<'tcx>> for ReplaceOpaqueTyFolder<'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
-    fn fold_binder<T: TypeFoldable<'tcx>>(
+    fn fold_binder<T: TypeFoldable<TyCtxt<'tcx>>>(
         &mut self,
         t: ty::Binder<'tcx, T>,
     ) -> ty::Binder<'tcx, T> {
@@ -790,10 +787,9 @@ impl<'tcx> ty::TypeFolder<'tcx> for ReplaceOpaqueTyFolder<'tcx> {
     fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
         if let ty::Alias(ty::Opaque, ty::AliasTy { def_id, substs, .. }) = *ty.kind() {
             if def_id == self.opaque_ty_id.0 && substs == self.identity_substs {
-                return self.tcx.mk_ty(ty::Bound(
-                    self.binder_index,
-                    ty::BoundTy::from(ty::BoundVar::from_u32(0)),
-                ));
+                return self
+                    .tcx
+                    .mk_bound(self.binder_index, ty::BoundTy::from(ty::BoundVar::from_u32(0)));
             }
         }
         ty
diff --git a/compiler/rustc_traits/src/chalk/lowering.rs b/compiler/rustc_traits/src/chalk/lowering.rs
index 9712abb70..60e22d100 100644
--- a/compiler/rustc_traits/src/chalk/lowering.rs
+++ b/compiler/rustc_traits/src/chalk/lowering.rs
@@ -62,7 +62,9 @@ impl<'tcx> LowerInto<'tcx, chalk_ir::Substitution<RustInterner<'tcx>>> for Subst
 
 impl<'tcx> LowerInto<'tcx, SubstsRef<'tcx>> for &chalk_ir::Substitution<RustInterner<'tcx>> {
     fn lower_into(self, interner: RustInterner<'tcx>) -> SubstsRef<'tcx> {
-        interner.tcx.mk_substs(self.iter(interner).map(|subst| subst.lower_into(interner)))
+        interner
+            .tcx
+            .mk_substs_from_iter(self.iter(interner).map(|subst| subst.lower_into(interner)))
     }
 }
 
@@ -116,6 +118,8 @@ impl<'tcx> LowerInto<'tcx, chalk_ir::InEnvironment<chalk_ir::Goal<RustInterner<'
                     )),
                 },
                 ty::PredicateKind::ObjectSafe(..)
+                | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
+                | ty::PredicateKind::AliasEq(..)
                 | ty::PredicateKind::ClosureKind(..)
                 | ty::PredicateKind::Subtype(..)
                 | ty::PredicateKind::Coerce(..)
@@ -210,6 +214,8 @@ impl<'tcx> LowerInto<'tcx, chalk_ir::GoalData<RustInterner<'tcx>>> for ty::Predi
             // We can defer this, but ultimately we'll want to express
             // some of these in terms of chalk operations.
             ty::PredicateKind::ClosureKind(..)
+            | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
+            | ty::PredicateKind::AliasEq(..)
             | ty::PredicateKind::Coerce(..)
             | ty::PredicateKind::ConstEvaluatable(..)
             | ty::PredicateKind::Ambiguous
@@ -343,6 +349,7 @@ impl<'tcx> LowerInto<'tcx, chalk_ir::Ty<RustInterner<'tcx>>> for Ty<'tcx> {
                 substs.lower_into(interner),
             ),
             ty::GeneratorWitness(_) => unimplemented!(),
+            ty::GeneratorWitnessMIR(..) => unimplemented!(),
             ty::Never => chalk_ir::TyKind::Never,
             ty::Tuple(types) => {
                 chalk_ir::TyKind::Tuple(types.len(), types.as_substs().lower_into(interner))
@@ -369,7 +376,7 @@ impl<'tcx> LowerInto<'tcx, chalk_ir::Ty<RustInterner<'tcx>>> for Ty<'tcx> {
             ty::Placeholder(_placeholder) => {
                 chalk_ir::TyKind::Placeholder(chalk_ir::PlaceholderIndex {
                     ui: chalk_ir::UniverseIndex { counter: _placeholder.universe.as_usize() },
-                    idx: _placeholder.name.as_usize(),
+                    idx: _placeholder.name.expect_anon() as usize,
                 })
             }
             ty::Infer(_infer) => unimplemented!(),
@@ -450,11 +457,7 @@ impl<'tcx> LowerInto<'tcx, Ty<'tcx>> for &chalk_ir::Ty<RustInterner<'tcx>> {
                 interner.tcx.mk_alias_ty(assoc_ty.0, substitution.lower_into(interner)),
             ),
             TyKind::Foreign(def_id) => ty::Foreign(def_id.0),
-            TyKind::Error => return interner.tcx.ty_error(),
-            TyKind::Placeholder(placeholder) => ty::Placeholder(ty::Placeholder {
-                universe: ty::UniverseIndex::from_usize(placeholder.ui.counter),
-                name: ty::BoundVar::from_usize(placeholder.idx),
-            }),
+            TyKind::Error => return interner.tcx.ty_error_misc(),
             TyKind::Alias(alias_ty) => match alias_ty {
                 chalk_ir::AliasTy::Projection(projection) => ty::Alias(
                     ty::Projection,
@@ -472,17 +475,21 @@ impl<'tcx> LowerInto<'tcx, Ty<'tcx>> for &chalk_ir::Ty<RustInterner<'tcx>> {
                 ),
             },
             TyKind::Function(_quantified_ty) => unimplemented!(),
-            TyKind::BoundVar(_bound) => ty::Bound(
-                ty::DebruijnIndex::from_usize(_bound.debruijn.depth() as usize),
+            TyKind::BoundVar(bound) => ty::Bound(
+                ty::DebruijnIndex::from_usize(bound.debruijn.depth() as usize),
                 ty::BoundTy {
-                    var: ty::BoundVar::from_usize(_bound.index),
-                    kind: ty::BoundTyKind::Anon,
+                    var: ty::BoundVar::from_usize(bound.index),
+                    kind: ty::BoundTyKind::Anon(bound.index as u32),
                 },
             ),
+            TyKind::Placeholder(placeholder) => ty::Placeholder(ty::Placeholder {
+                universe: ty::UniverseIndex::from_usize(placeholder.ui.counter),
+                name: ty::BoundTyKind::Anon(placeholder.idx as u32),
+            }),
             TyKind::InferenceVar(_, _) => unimplemented!(),
             TyKind::Dyn(_) => unimplemented!(),
         };
-        interner.tcx.mk_ty(kind)
+        interner.tcx.mk_ty_from_kind(kind)
     }
 }
 
@@ -492,6 +499,9 @@ impl<'tcx> LowerInto<'tcx, chalk_ir::Lifetime<RustInterner<'tcx>>> for Region<'t
             ty::ReEarlyBound(_) => {
                 panic!("Should have already been substituted.");
             }
+            ty::ReError(_) => {
+                panic!("Error lifetime should not have already been lowered.");
+            }
             ty::ReLateBound(db, br) => chalk_ir::LifetimeData::BoundVar(chalk_ir::BoundVar::new(
                 chalk_ir::DebruijnIndex::new(db.as_u32()),
                 br.var.as_usize(),
@@ -503,7 +513,7 @@ impl<'tcx> LowerInto<'tcx, chalk_ir::Lifetime<RustInterner<'tcx>>> for Region<'t
             ty::RePlaceholder(placeholder_region) => {
                 chalk_ir::LifetimeData::Placeholder(chalk_ir::PlaceholderIndex {
                     ui: chalk_ir::UniverseIndex { counter: placeholder_region.universe.index() },
-                    idx: 0,
+                    idx: 0, // FIXME: This `idx: 0` is sus.
                 })
                 .intern(interner)
             }
@@ -514,8 +524,9 @@ impl<'tcx> LowerInto<'tcx, chalk_ir::Lifetime<RustInterner<'tcx>>> for Region<'t
 
 impl<'tcx> LowerInto<'tcx, Region<'tcx>> for &chalk_ir::Lifetime<RustInterner<'tcx>> {
     fn lower_into(self, interner: RustInterner<'tcx>) -> Region<'tcx> {
-        let kind = match self.data(interner) {
-            chalk_ir::LifetimeData::BoundVar(var) => ty::ReLateBound(
+        let tcx = interner.tcx;
+        match self.data(interner) {
+            chalk_ir::LifetimeData::BoundVar(var) => tcx.mk_re_late_bound(
                 ty::DebruijnIndex::from_u32(var.debruijn.depth()),
                 ty::BoundRegion {
                     var: ty::BoundVar::from_usize(var.index),
@@ -523,15 +534,14 @@ impl<'tcx> LowerInto<'tcx, Region<'tcx>> for &chalk_ir::Lifetime<RustInterner<'t
                 },
             ),
             chalk_ir::LifetimeData::InferenceVar(_var) => unimplemented!(),
-            chalk_ir::LifetimeData::Placeholder(p) => ty::RePlaceholder(ty::Placeholder {
+            chalk_ir::LifetimeData::Placeholder(p) => tcx.mk_re_placeholder(ty::Placeholder {
                 universe: ty::UniverseIndex::from_usize(p.ui.counter),
                 name: ty::BoundRegionKind::BrAnon(p.idx as u32, None),
             }),
-            chalk_ir::LifetimeData::Static => return interner.tcx.lifetimes.re_static,
-            chalk_ir::LifetimeData::Erased => return interner.tcx.lifetimes.re_erased,
+            chalk_ir::LifetimeData::Static => tcx.lifetimes.re_static,
+            chalk_ir::LifetimeData::Erased => tcx.lifetimes.re_erased,
             chalk_ir::LifetimeData::Phantom(void, _) => match *void {},
-        };
-        interner.tcx.mk_region(kind)
+        }
     }
 }
 
@@ -639,8 +649,10 @@ impl<'tcx> LowerInto<'tcx, Option<chalk_ir::QuantifiedWhereClause<RustInterner<'
                 Some(chalk_ir::WhereClause::AliasEq(predicate.lower_into(interner)))
             }
             ty::PredicateKind::WellFormed(_ty) => None,
+            ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..)) => None,
 
             ty::PredicateKind::ObjectSafe(..)
+            | ty::PredicateKind::AliasEq(..)
             | ty::PredicateKind::ClosureKind(..)
             | ty::PredicateKind::Subtype(..)
             | ty::PredicateKind::Coerce(..)
@@ -670,11 +682,11 @@ impl<'tcx> LowerInto<'tcx, chalk_ir::Binders<chalk_ir::QuantifiedWhereClauses<Ru
         // shifted in by one so that they are still escaping.
         let predicates = ty::fold::shift_vars(interner.tcx, self, 1);
 
-        let self_ty = interner.tcx.mk_ty(ty::Bound(
+        let self_ty = interner.tcx.mk_bound(
             // This is going to be wrapped in a binder
             ty::DebruijnIndex::from_usize(1),
-            ty::BoundTy { var: ty::BoundVar::from_usize(0), kind: ty::BoundTyKind::Anon },
-        ));
+            ty::BoundTy { var: ty::BoundVar::from_usize(0), kind: ty::BoundTyKind::Anon(0) },
+        );
         let where_clauses = predicates.into_iter().map(|predicate| {
             let (predicate, binders, _named_regions) =
                 collect_bound_vars(interner, interner.tcx, predicate);
@@ -772,8 +784,10 @@ impl<'tcx> LowerInto<'tcx, Option<chalk_solve::rust_ir::QuantifiedInlineBound<Ru
             }
             ty::PredicateKind::Clause(ty::Clause::TypeOutlives(_predicate)) => None,
             ty::PredicateKind::WellFormed(_ty) => None,
+            ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..)) => None,
 
             ty::PredicateKind::Clause(ty::Clause::RegionOutlives(..))
+            | ty::PredicateKind::AliasEq(..)
             | ty::PredicateKind::ObjectSafe(..)
             | ty::PredicateKind::ClosureKind(..)
             | ty::PredicateKind::Subtype(..)
@@ -867,7 +881,7 @@ impl<'tcx> LowerInto<'tcx, chalk_solve::rust_ir::AliasEqBound<RustInterner<'tcx>
 /// It's important to note that because of prior substitution, we may have
 /// late-bound regions, even outside of fn contexts, since this is the best way
 /// to prep types for chalk lowering.
-pub(crate) fn collect_bound_vars<'tcx, T: TypeFoldable<'tcx>>(
+pub(crate) fn collect_bound_vars<'tcx, T: TypeFoldable<TyCtxt<'tcx>>>(
     interner: RustInterner<'tcx>,
     tcx: TyCtxt<'tcx>,
     ty: Binder<'tcx, T>,
@@ -917,8 +931,8 @@ impl<'tcx> BoundVarsCollector<'tcx> {
     }
 }
 
-impl<'tcx> TypeVisitor<'tcx> for BoundVarsCollector<'tcx> {
-    fn visit_binder<T: TypeVisitable<'tcx>>(
+impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for BoundVarsCollector<'tcx> {
+    fn visit_binder<T: TypeVisitable<TyCtxt<'tcx>>>(
         &mut self,
         t: &Binder<'tcx, T>,
     ) -> ControlFlow<Self::BreakTy> {
@@ -998,12 +1012,15 @@ impl<'a, 'tcx> NamedBoundVarSubstitutor<'a, 'tcx> {
     }
 }
 
-impl<'a, 'tcx> TypeFolder<'tcx> for NamedBoundVarSubstitutor<'a, 'tcx> {
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for NamedBoundVarSubstitutor<'a, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
-    fn fold_binder<T: TypeFoldable<'tcx>>(&mut self, t: Binder<'tcx, T>) -> Binder<'tcx, T> {
+    fn fold_binder<T: TypeFoldable<TyCtxt<'tcx>>>(
+        &mut self,
+        t: Binder<'tcx, T>,
+    ) -> Binder<'tcx, T> {
         self.binder_index.shift_in(1);
         let result = t.super_fold_with(self);
         self.binder_index.shift_out(1);
@@ -1016,7 +1033,7 @@ impl<'a, 'tcx> TypeFolder<'tcx> for NamedBoundVarSubstitutor<'a, 'tcx> {
                 ty::BrNamed(def_id, _name) => match self.named_parameters.get(&def_id) {
                     Some(idx) => {
                         let new_br = ty::BoundRegion { var: br.var, kind: ty::BrAnon(*idx, None) };
-                        return self.tcx.mk_region(ty::ReLateBound(index, new_br));
+                        return self.tcx.mk_re_late_bound(index, new_br);
                     }
                     None => panic!("Missing `BrNamed`."),
                 },
@@ -1037,7 +1054,7 @@ pub(crate) struct ParamsSubstitutor<'tcx> {
     binder_index: ty::DebruijnIndex,
     list: Vec<rustc_middle::ty::ParamTy>,
     next_ty_placeholder: usize,
-    pub(crate) params: rustc_data_structures::fx::FxHashMap<usize, rustc_middle::ty::ParamTy>,
+    pub(crate) params: rustc_data_structures::fx::FxHashMap<u32, rustc_middle::ty::ParamTy>,
     pub(crate) named_regions: BTreeMap<DefId, u32>,
 }
 
@@ -1054,12 +1071,15 @@ impl<'tcx> ParamsSubstitutor<'tcx> {
     }
 }
 
-impl<'tcx> TypeFolder<'tcx> for ParamsSubstitutor<'tcx> {
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ParamsSubstitutor<'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
-    fn fold_binder<T: TypeFoldable<'tcx>>(&mut self, t: Binder<'tcx, T>) -> Binder<'tcx, T> {
+    fn fold_binder<T: TypeFoldable<TyCtxt<'tcx>>>(
+        &mut self,
+        t: Binder<'tcx, T>,
+    ) -> Binder<'tcx, T> {
         self.binder_index.shift_in(1);
         let result = t.super_fold_with(self);
         self.binder_index.shift_out(1);
@@ -1069,18 +1089,18 @@ impl<'tcx> TypeFolder<'tcx> for ParamsSubstitutor<'tcx> {
     fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
         match *t.kind() {
             ty::Param(param) => match self.list.iter().position(|r| r == &param) {
-                Some(idx) => self.tcx.mk_ty(ty::Placeholder(ty::PlaceholderType {
+                Some(idx) => self.tcx.mk_placeholder(ty::PlaceholderType {
                     universe: ty::UniverseIndex::from_usize(0),
-                    name: ty::BoundVar::from_usize(idx),
-                })),
+                    name: ty::BoundTyKind::Anon(idx as u32),
+                }),
                 None => {
                     self.list.push(param);
                     let idx = self.list.len() - 1 + self.next_ty_placeholder;
-                    self.params.insert(idx, param);
-                    self.tcx.mk_ty(ty::Placeholder(ty::PlaceholderType {
+                    self.params.insert(idx as u32, param);
+                    self.tcx.mk_placeholder(ty::PlaceholderType {
                         universe: ty::UniverseIndex::from_usize(0),
-                        name: ty::BoundVar::from_usize(idx),
-                    }))
+                        name: ty::BoundTyKind::Anon(idx as u32),
+                    })
                 }
             },
             _ => t.super_fold_with(self),
@@ -1098,7 +1118,7 @@ impl<'tcx> TypeFolder<'tcx> for ParamsSubstitutor<'tcx> {
                         var: ty::BoundVar::from_u32(*idx),
                         kind: ty::BrAnon(*idx, None),
                     };
-                    self.tcx.mk_region(ty::ReLateBound(self.binder_index, br))
+                    self.tcx.mk_re_late_bound(self.binder_index, br)
                 }
                 None => {
                     let idx = self.named_regions.len() as u32;
@@ -1107,7 +1127,7 @@ impl<'tcx> TypeFolder<'tcx> for ParamsSubstitutor<'tcx> {
                         kind: ty::BrAnon(idx, None),
                     };
                     self.named_regions.insert(_re.def_id, idx);
-                    self.tcx.mk_region(ty::ReLateBound(self.binder_index, br))
+                    self.tcx.mk_re_late_bound(self.binder_index, br)
                 }
             },
 
@@ -1118,28 +1138,28 @@ impl<'tcx> TypeFolder<'tcx> for ParamsSubstitutor<'tcx> {
 
 pub(crate) struct ReverseParamsSubstitutor<'tcx> {
     tcx: TyCtxt<'tcx>,
-    params: rustc_data_structures::fx::FxHashMap<usize, rustc_middle::ty::ParamTy>,
+    params: rustc_data_structures::fx::FxHashMap<u32, rustc_middle::ty::ParamTy>,
 }
 
 impl<'tcx> ReverseParamsSubstitutor<'tcx> {
     pub(crate) fn new(
         tcx: TyCtxt<'tcx>,
-        params: rustc_data_structures::fx::FxHashMap<usize, rustc_middle::ty::ParamTy>,
+        params: rustc_data_structures::fx::FxHashMap<u32, rustc_middle::ty::ParamTy>,
     ) -> Self {
         Self { tcx, params }
     }
 }
 
-impl<'tcx> TypeFolder<'tcx> for ReverseParamsSubstitutor<'tcx> {
-    fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ReverseParamsSubstitutor<'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
     fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
         match *t.kind() {
             ty::Placeholder(ty::PlaceholderType { universe: ty::UniverseIndex::ROOT, name }) => {
-                match self.params.get(&name.as_usize()) {
-                    Some(param) => self.tcx.mk_ty(ty::Param(*param)),
+                match self.params.get(&name.expect_anon()) {
+                    Some(&ty::ParamTy { index, name }) => self.tcx.mk_ty_param(index, name),
                     None => t,
                 }
             }
@@ -1166,11 +1186,12 @@ impl PlaceholdersCollector {
     }
 }
 
-impl<'tcx> TypeVisitor<'tcx> for PlaceholdersCollector {
+impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for PlaceholdersCollector {
     fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
         match t.kind() {
             ty::Placeholder(p) if p.universe == self.universe_index => {
-                self.next_ty_placeholder = self.next_ty_placeholder.max(p.name.as_usize() + 1);
+                self.next_ty_placeholder =
+                    self.next_ty_placeholder.max(p.name.expect_anon() as usize + 1);
             }
 
             _ => (),
@@ -1185,6 +1206,7 @@ impl<'tcx> TypeVisitor<'tcx> for PlaceholdersCollector {
                 if let ty::BoundRegionKind::BrAnon(anon, _) = p.name {
                     self.next_anon_region_placeholder = self.next_anon_region_placeholder.max(anon);
                 }
+                // FIXME: This doesn't seem to handle BrNamed at all?
             }
 
             _ => (),
diff --git a/compiler/rustc_traits/src/chalk/mod.rs b/compiler/rustc_traits/src/chalk/mod.rs
index f76386fa7..a5ebc26a8 100644
--- a/compiler/rustc_traits/src/chalk/mod.rs
+++ b/compiler/rustc_traits/src/chalk/mod.rs
@@ -6,15 +6,10 @@
 pub(crate) mod db;
 pub(crate) mod lowering;
 
-use rustc_data_structures::fx::FxHashMap;
-
-use rustc_index::vec::IndexVec;
-
 use rustc_middle::infer::canonical::{CanonicalTyVarKind, CanonicalVarKind};
 use rustc_middle::traits::ChalkRustInterner;
 use rustc_middle::ty::query::Providers;
-use rustc_middle::ty::subst::GenericArg;
-use rustc_middle::ty::{self, BoundVar, ParamTy, TyCtxt, TypeFoldable, TypeVisitable};
+use rustc_middle::ty::{self, TyCtxt, TypeFoldable, TypeVisitable};
 
 use rustc_infer::infer::canonical::{
     Canonical, CanonicalVarValues, Certainty, QueryRegionConstraints, QueryResponse,
@@ -44,7 +39,7 @@ pub(crate) fn evaluate_goal<'tcx>(
     let mut params_substitutor =
         ParamsSubstitutor::new(tcx, placeholders_collector.next_ty_placeholder);
     let obligation = obligation.fold_with(&mut params_substitutor);
-    let params: FxHashMap<usize, ParamTy> = params_substitutor.params;
+    let params = params_substitutor.params;
 
     let max_universe = obligation.max_universe.index();
 
@@ -100,36 +95,35 @@ pub(crate) fn evaluate_goal<'tcx>(
                          binders: chalk_ir::CanonicalVarKinds<_>| {
         use rustc_middle::infer::canonical::CanonicalVarInfo;
 
-        let mut var_values: IndexVec<BoundVar, GenericArg<'tcx>> = IndexVec::new();
         let mut reverse_param_substitutor = ReverseParamsSubstitutor::new(tcx, params);
-        subst.as_slice(interner).iter().for_each(|p| {
-            var_values.push(p.lower_into(interner).fold_with(&mut reverse_param_substitutor));
-        });
-        let variables: Vec<_> = binders
-            .iter(interner)
-            .map(|var| {
-                let kind = match var.kind {
-                    chalk_ir::VariableKind::Ty(ty_kind) => CanonicalVarKind::Ty(match ty_kind {
-                        chalk_ir::TyVariableKind::General => CanonicalTyVarKind::General(
-                            ty::UniverseIndex::from_usize(var.skip_kind().counter),
-                        ),
-                        chalk_ir::TyVariableKind::Integer => CanonicalTyVarKind::Int,
-                        chalk_ir::TyVariableKind::Float => CanonicalTyVarKind::Float,
-                    }),
-                    chalk_ir::VariableKind::Lifetime => CanonicalVarKind::Region(
+        let var_values = tcx.mk_substs_from_iter(
+            subst
+                .as_slice(interner)
+                .iter()
+                .map(|p| p.lower_into(interner).fold_with(&mut reverse_param_substitutor)),
+        );
+        let variables = binders.iter(interner).map(|var| {
+            let kind = match var.kind {
+                chalk_ir::VariableKind::Ty(ty_kind) => CanonicalVarKind::Ty(match ty_kind {
+                    chalk_ir::TyVariableKind::General => CanonicalTyVarKind::General(
                         ty::UniverseIndex::from_usize(var.skip_kind().counter),
                     ),
-                    // FIXME(compiler-errors): We don't currently have a way of turning
-                    // a Chalk ty back into a rustc ty, right?
-                    chalk_ir::VariableKind::Const(_) => todo!(),
-                };
-                CanonicalVarInfo { kind }
-            })
-            .collect();
+                    chalk_ir::TyVariableKind::Integer => CanonicalTyVarKind::Int,
+                    chalk_ir::TyVariableKind::Float => CanonicalTyVarKind::Float,
+                }),
+                chalk_ir::VariableKind::Lifetime => {
+                    CanonicalVarKind::Region(ty::UniverseIndex::from_usize(var.skip_kind().counter))
+                }
+                // FIXME(compiler-errors): We don't currently have a way of turning
+                // a Chalk ty back into a rustc ty, right?
+                chalk_ir::VariableKind::Const(_) => todo!(),
+            };
+            CanonicalVarInfo { kind }
+        });
         let max_universe = binders.iter(interner).map(|v| v.skip_kind().counter).max().unwrap_or(0);
         let sol = Canonical {
             max_universe: ty::UniverseIndex::from_usize(max_universe),
-            variables: tcx.intern_canonical_var_infos(&variables),
+            variables: tcx.mk_canonical_var_infos_from_iter(variables),
             value: QueryResponse {
                 var_values: CanonicalVarValues { var_values },
                 region_constraints: QueryRegionConstraints::default(),
@@ -159,8 +153,7 @@ pub(crate) fn evaluate_goal<'tcx>(
                             max_universe: ty::UniverseIndex::from_usize(0),
                             variables: obligation.variables,
                             value: QueryResponse {
-                                var_values: CanonicalVarValues { var_values: IndexVec::new() }
-                                    .make_identity(tcx),
+                                var_values: CanonicalVarValues::dummy(),
                                 region_constraints: QueryRegionConstraints::default(),
                                 certainty: Certainty::Ambiguous,
                                 opaque_types: vec![],
diff --git a/compiler/rustc_traits/src/codegen.rs b/compiler/rustc_traits/src/codegen.rs
index c0da8a816..6f81d343e 100644
--- a/compiler/rustc_traits/src/codegen.rs
+++ b/compiler/rustc_traits/src/codegen.rs
@@ -4,7 +4,7 @@
 // general routines.
 
 use rustc_infer::infer::{DefiningAnchor, TyCtxtInferExt};
-use rustc_infer::traits::FulfillmentErrorCode;
+use rustc_infer::traits::{FulfillmentErrorCode, TraitEngineExt as _};
 use rustc_middle::traits::CodegenObligationError;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt;
diff --git a/compiler/rustc_traits/src/dropck_outlives.rs b/compiler/rustc_traits/src/dropck_outlives.rs
index 481b56e11..b5924e949 100644
--- a/compiler/rustc_traits/src/dropck_outlives.rs
+++ b/compiler/rustc_traits/src/dropck_outlives.rs
@@ -164,7 +164,8 @@ fn dtorck_constraint_for_ty<'tcx>(
         | ty::Ref(..)
         | ty::FnDef(..)
         | ty::FnPtr(_)
-        | ty::GeneratorWitness(..) => {
+        | ty::GeneratorWitness(..)
+        | ty::GeneratorWitnessMIR(..) => {
             // these types never have a destructor
         }
 
@@ -307,7 +308,7 @@ pub(crate) fn adt_dtorck_constraint(
 
     let mut result = DropckConstraint::empty();
     for field in def.all_fields() {
-        let fty = tcx.type_of(field.did);
+        let fty = tcx.type_of(field.did).subst_identity();
         dtorck_constraint_for_ty(tcx, span, fty, 0, fty, &mut result)?;
     }
     result.outlives.extend(tcx.destructor_constraints(def));
diff --git a/compiler/rustc_traits/src/implied_outlives_bounds.rs b/compiler/rustc_traits/src/implied_outlives_bounds.rs
index 7d2d8433c..ddd4ca143 100644
--- a/compiler/rustc_traits/src/implied_outlives_bounds.rs
+++ b/compiler/rustc_traits/src/implied_outlives_bounds.rs
@@ -2,13 +2,13 @@
 //! Do not call this query directory. See
 //! [`rustc_trait_selection::traits::query::type_op::implied_outlives_bounds`].
 
-use rustc_hir as hir;
 use rustc_infer::infer::canonical::{self, Canonical};
 use rustc_infer::infer::outlives::components::{push_outlives_components, Component};
 use rustc_infer::infer::TyCtxtInferExt;
 use rustc_infer::traits::query::OutlivesBound;
 use rustc_middle::ty::query::Providers;
-use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitableExt};
+use rustc_span::def_id::CRATE_DEF_ID;
 use rustc_span::source_map::DUMMY_SP;
 use rustc_trait_selection::infer::InferCtxtBuilderExt;
 use rustc_trait_selection::traits::query::{CanonicalTyGoal, Fallible, NoSolution};
@@ -67,51 +67,69 @@ fn compute_implied_outlives_bounds<'tcx>(
         // FIXME(@lcnr): It's not really "always fine", having fewer implied
         // bounds can be backward incompatible, e.g. #101951 was caused by
         // us not dealing with inference vars in `TypeOutlives` predicates.
-        let obligations =
-            wf::obligations(ocx.infcx, param_env, hir::CRATE_HIR_ID, 0, arg, DUMMY_SP)
-                .unwrap_or_default();
-
-        // While these predicates should all be implied by other parts of
-        // the program, they are still relevant as they may constrain
-        // inference variables, which is necessary to add the correct
-        // implied bounds in some cases, mostly when dealing with projections.
-        ocx.register_obligations(
-            obligations.iter().filter(|o| o.predicate.has_non_region_infer()).cloned(),
-        );
-
-        // From the full set of obligations, just filter down to the
-        // region relationships.
-        outlives_bounds.extend(obligations.into_iter().filter_map(|obligation| {
+        let obligations = wf::obligations(ocx.infcx, param_env, CRATE_DEF_ID, 0, arg, DUMMY_SP)
+            .unwrap_or_default();
+
+        for obligation in obligations {
+            debug!(?obligation);
             assert!(!obligation.has_escaping_bound_vars());
-            match obligation.predicate.kind().no_bound_vars() {
-                None => None,
-                Some(pred) => match pred {
-                    ty::PredicateKind::Clause(ty::Clause::Trait(..))
-                    | ty::PredicateKind::Subtype(..)
-                    | ty::PredicateKind::Coerce(..)
-                    | ty::PredicateKind::Clause(ty::Clause::Projection(..))
-                    | ty::PredicateKind::ClosureKind(..)
-                    | ty::PredicateKind::ObjectSafe(..)
-                    | ty::PredicateKind::ConstEvaluatable(..)
-                    | ty::PredicateKind::ConstEquate(..)
-                    | ty::PredicateKind::Ambiguous
-                    | ty::PredicateKind::TypeWellFormedFromEnv(..) => None,
-                    ty::PredicateKind::WellFormed(arg) => {
-                        wf_args.push(arg);
-                        None
+
+            // While these predicates should all be implied by other parts of
+            // the program, they are still relevant as they may constrain
+            // inference variables, which is necessary to add the correct
+            // implied bounds in some cases, mostly when dealing with projections.
+            //
+            // Another important point here: we only register `Projection`
+            // predicates, since otherwise we might register outlives
+            // predicates containing inference variables, and we don't
+            // learn anything new from those.
+            if obligation.predicate.has_non_region_infer() {
+                match obligation.predicate.kind().skip_binder() {
+                    ty::PredicateKind::Clause(ty::Clause::Projection(..))
+                    | ty::PredicateKind::AliasEq(..) => {
+                        ocx.register_obligation(obligation.clone());
                     }
+                    _ => {}
+                }
+            }
 
-                    ty::PredicateKind::Clause(ty::Clause::RegionOutlives(
-                        ty::OutlivesPredicate(r_a, r_b),
-                    )) => Some(ty::OutlivesPredicate(r_a.into(), r_b)),
+            let pred = match obligation.predicate.kind().no_bound_vars() {
+                None => continue,
+                Some(pred) => pred,
+            };
+            match pred {
+                ty::PredicateKind::Clause(ty::Clause::Trait(..))
+                // FIXME(const_generics): Make sure that `<'a, 'b, const N: &'a &'b u32>` is sound
+                // if we ever support that
+                | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
+                | ty::PredicateKind::Subtype(..)
+                | ty::PredicateKind::Coerce(..)
+                | ty::PredicateKind::Clause(ty::Clause::Projection(..))
+                | ty::PredicateKind::ClosureKind(..)
+                | ty::PredicateKind::ObjectSafe(..)
+                | ty::PredicateKind::ConstEvaluatable(..)
+                | ty::PredicateKind::ConstEquate(..)
+                | ty::PredicateKind::Ambiguous
+                | ty::PredicateKind::AliasEq(..)
+                | ty::PredicateKind::TypeWellFormedFromEnv(..) => {}
+
+                // We need to search through *all* WellFormed predicates
+                ty::PredicateKind::WellFormed(arg) => {
+                    wf_args.push(arg);
+                }
+
+                // We need to register region relationships
+                ty::PredicateKind::Clause(ty::Clause::RegionOutlives(ty::OutlivesPredicate(
+                    r_a,
+                    r_b,
+                ))) => outlives_bounds.push(ty::OutlivesPredicate(r_a.into(), r_b)),
 
-                    ty::PredicateKind::Clause(ty::Clause::TypeOutlives(ty::OutlivesPredicate(
-                        ty_a,
-                        r_b,
-                    ))) => Some(ty::OutlivesPredicate(ty_a.into(), r_b)),
-                },
+                ty::PredicateKind::Clause(ty::Clause::TypeOutlives(ty::OutlivesPredicate(
+                    ty_a,
+                    r_b,
+                ))) => outlives_bounds.push(ty::OutlivesPredicate(ty_a.into(), r_b)),
             }
-        }));
+        }
     }
 
     // This call to `select_all_or_error` is necessary to constrain inference variables, which we
diff --git a/compiler/rustc_traits/src/lib.rs b/compiler/rustc_traits/src/lib.rs
index 9aa26667e..8bea5588a 100644
--- a/compiler/rustc_traits/src/lib.rs
+++ b/compiler/rustc_traits/src/lib.rs
@@ -4,6 +4,7 @@
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 #![feature(let_chains)]
+#![feature(drain_filter)]
 #![recursion_limit = "256"]
 
 #[macro_use]
diff --git a/compiler/rustc_traits/src/normalize_erasing_regions.rs b/compiler/rustc_traits/src/normalize_erasing_regions.rs
index 5cad2c2cc..f0597f192 100644
--- a/compiler/rustc_traits/src/normalize_erasing_regions.rs
+++ b/compiler/rustc_traits/src/normalize_erasing_regions.rs
@@ -1,7 +1,7 @@
 use rustc_infer::infer::TyCtxtInferExt;
 use rustc_middle::traits::query::NoSolution;
 use rustc_middle::ty::query::Providers;
-use rustc_middle::ty::{self, ParamEnvAnd, TyCtxt, TypeFoldable};
+use rustc_middle::ty::{self, ParamEnvAnd, TyCtxt, TypeFoldable, TypeVisitableExt};
 use rustc_trait_selection::traits::query::normalize::QueryNormalizeExt;
 use rustc_trait_selection::traits::{Normalized, ObligationCause};
 use std::sync::atomic::Ordering;
@@ -22,7 +22,7 @@ pub(crate) fn provide(p: &mut Providers) {
     };
 }
 
-fn try_normalize_after_erasing_regions<'tcx, T: TypeFoldable<'tcx> + PartialEq + Copy>(
+fn try_normalize_after_erasing_regions<'tcx, T: TypeFoldable<TyCtxt<'tcx>> + PartialEq + Copy>(
     tcx: TyCtxt<'tcx>,
     goal: ParamEnvAnd<'tcx, T>,
 ) -> Result<T, NoSolution> {
@@ -60,6 +60,8 @@ fn not_outlives_predicate(p: ty::Predicate<'_>) -> bool {
         | ty::PredicateKind::Clause(ty::Clause::TypeOutlives(..)) => false,
         ty::PredicateKind::Clause(ty::Clause::Trait(..))
         | ty::PredicateKind::Clause(ty::Clause::Projection(..))
+        | ty::PredicateKind::Clause(ty::Clause::ConstArgHasType(..))
+        | ty::PredicateKind::AliasEq(..)
         | ty::PredicateKind::WellFormed(..)
         | ty::PredicateKind::ObjectSafe(..)
         | ty::PredicateKind::ClosureKind(..)
diff --git a/compiler/rustc_traits/src/type_op.rs b/compiler/rustc_traits/src/type_op.rs
index f35c5e448..e0fd487b3 100644
--- a/compiler/rustc_traits/src/type_op.rs
+++ b/compiler/rustc_traits/src/type_op.rs
@@ -6,6 +6,7 @@ use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, FnSig, Lift, PolyFnSig, Ty, TyCtxt, TypeFoldable};
 use rustc_middle::ty::{ParamEnvAnd, Predicate};
 use rustc_middle::ty::{UserSelfTy, UserSubsts, UserType};
+use rustc_span::def_id::CRATE_DEF_ID;
 use rustc_span::{Span, DUMMY_SP};
 use rustc_trait_selection::infer::InferCtxtBuilderExt;
 use rustc_trait_selection::traits::query::normalize::QueryNormalizeExt;
@@ -76,7 +77,6 @@ fn relate_mir_and_user_ty<'tcx>(
     // FIXME(#104764): We should check well-formedness before normalization.
     let predicate = ty::Binder::dummy(ty::PredicateKind::WellFormed(user_ty.into()));
     ocx.register_obligation(Obligation::new(ocx.infcx.tcx, cause, param_env, predicate));
-
     Ok(())
 }
 
@@ -93,7 +93,7 @@ fn relate_mir_and_user_substs<'tcx>(
     let tcx = ocx.infcx.tcx;
     let cause = ObligationCause::dummy_with_span(span);
 
-    let ty = tcx.bound_type_of(def_id).subst(tcx, substs);
+    let ty = tcx.type_of(def_id).subst(tcx, substs);
     let ty = ocx.normalize(&cause, param_env, ty);
     debug!("relate_type_and_user_type: ty of def-id is {:?}", ty);
 
@@ -111,7 +111,7 @@ fn relate_mir_and_user_substs<'tcx>(
         let span = if span == DUMMY_SP { predicate_span } else { span };
         let cause = ObligationCause::new(
             span,
-            hir::CRATE_HIR_ID,
+            CRATE_DEF_ID,
             ObligationCauseCode::AscribeUserTypeProvePredicate(predicate_span),
         );
         let instantiated_predicate =
@@ -122,11 +122,10 @@ fn relate_mir_and_user_substs<'tcx>(
 
     if let Some(UserSelfTy { impl_def_id, self_ty }) = user_self_ty {
         let self_ty = ocx.normalize(&cause, param_env, self_ty);
-        let impl_self_ty = tcx.bound_type_of(impl_def_id).subst(tcx, substs);
+        let impl_self_ty = tcx.type_of(impl_def_id).subst(tcx, substs);
         let impl_self_ty = ocx.normalize(&cause, param_env, impl_self_ty);
 
         ocx.eq(&cause, param_env, self_ty, impl_self_ty)?;
-
         let predicate = ty::Binder::dummy(ty::PredicateKind::WellFormed(impl_self_ty.into()));
         ocx.register_obligation(Obligation::new(tcx, cause.clone(), param_env, predicate));
     }
@@ -162,7 +161,7 @@ fn type_op_normalize<'tcx, T>(
     key: ParamEnvAnd<'tcx, Normalize<T>>,
 ) -> Fallible<T>
 where
-    T: fmt::Debug + TypeFoldable<'tcx> + Lift<'tcx>,
+    T: fmt::Debug + TypeFoldable<TyCtxt<'tcx>> + Lift<'tcx>,
 {
     let (param_env, Normalize { value }) = key.into_parts();
     let Normalized { value, obligations } =
diff --git a/compiler/rustc_transmute/src/layout/tree.rs b/compiler/rustc_transmute/src/layout/tree.rs
index 30e20ba6f..295b65c2c 100644
--- a/compiler/rustc_transmute/src/layout/tree.rs
+++ b/compiler/rustc_transmute/src/layout/tree.rs
@@ -284,8 +284,9 @@ pub(crate) mod rustc {
                 }
 
                 ty::Array(ty, len) => {
-                    let len =
-                        len.try_eval_usize(tcx, ParamEnv::reveal_all()).ok_or(Err::Unspecified)?;
+                    let len = len
+                        .try_eval_target_usize(tcx, ParamEnv::reveal_all())
+                        .ok_or(Err::Unspecified)?;
                     let elt = Tree::from_ty(*ty, tcx)?;
                     Ok(std::iter::repeat(elt)
                         .take(len as usize)
diff --git a/compiler/rustc_transmute/src/lib.rs b/compiler/rustc_transmute/src/lib.rs
index b3b9a67b2..4b4a8ebd0 100644
--- a/compiler/rustc_transmute/src/lib.rs
+++ b/compiler/rustc_transmute/src/lib.rs
@@ -117,7 +117,7 @@ mod rustc {
             c: Const<'tcx>,
         ) -> Option<Self> {
             use rustc_middle::ty::ScalarInt;
-            use rustc_middle::ty::TypeVisitable;
+            use rustc_middle::ty::TypeVisitableExt;
             use rustc_span::symbol::sym;
 
             let c = c.eval(tcx, param_env);
diff --git a/compiler/rustc_error_messages/locales/en-US/ty_utils.ftl b/compiler/rustc_ty_utils/locales/en-US.ftl
index abe65a0e3..abe65a0e3 100644
--- a/compiler/rustc_error_messages/locales/en-US/ty_utils.ftl
+++ b/compiler/rustc_ty_utils/locales/en-US.ftl
diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs
index 91a505a72..35c9f95eb 100644
--- a/compiler/rustc_ty_utils/src/abi.rs
+++ b/compiler/rustc_ty_utils/src/abi.rs
@@ -41,7 +41,7 @@ fn fn_sig_for_fn_abi<'tcx>(
             // We normalize the `fn_sig` again after substituting at a later point.
             let mut sig = match *ty.kind() {
                 ty::FnDef(def_id, substs) => tcx
-                    .bound_fn_sig(def_id)
+                    .fn_sig(def_id)
                     .map_bound(|fn_sig| {
                         tcx.normalize_erasing_regions(tcx.param_env(def_id), fn_sig)
                     })
@@ -54,7 +54,7 @@ fn fn_sig_for_fn_abi<'tcx>(
                 sig = sig.map_bound(|mut sig| {
                     let mut inputs_and_output = sig.inputs_and_output.to_vec();
                     inputs_and_output[0] = tcx.mk_mut_ptr(inputs_and_output[0]);
-                    sig.inputs_and_output = tcx.intern_type_list(&inputs_and_output);
+                    sig.inputs_and_output = tcx.mk_type_list(&inputs_and_output);
                     sig
                 });
             }
@@ -63,14 +63,14 @@ fn fn_sig_for_fn_abi<'tcx>(
         ty::Closure(def_id, substs) => {
             let sig = substs.as_closure().sig();
 
-            let bound_vars = tcx.mk_bound_variable_kinds(
+            let bound_vars = tcx.mk_bound_variable_kinds_from_iter(
                 sig.bound_vars().iter().chain(iter::once(ty::BoundVariableKind::Region(ty::BrEnv))),
             );
             let br = ty::BoundRegion {
                 var: ty::BoundVar::from_usize(bound_vars.len() - 1),
                 kind: ty::BoundRegionKind::BrEnv,
             };
-            let env_region = ty::ReLateBound(ty::INNERMOST, br);
+            let env_region = tcx.mk_re_late_bound(ty::INNERMOST, br);
             let env_ty = tcx.closure_env_ty(def_id, substs, env_region).unwrap();
 
             let sig = sig.skip_binder();
@@ -88,19 +88,18 @@ fn fn_sig_for_fn_abi<'tcx>(
         ty::Generator(did, substs, _) => {
             let sig = substs.as_generator().poly_sig();
 
-            let bound_vars = tcx.mk_bound_variable_kinds(
+            let bound_vars = tcx.mk_bound_variable_kinds_from_iter(
                 sig.bound_vars().iter().chain(iter::once(ty::BoundVariableKind::Region(ty::BrEnv))),
             );
             let br = ty::BoundRegion {
                 var: ty::BoundVar::from_usize(bound_vars.len() - 1),
                 kind: ty::BoundRegionKind::BrEnv,
             };
-            let env_region = ty::ReLateBound(ty::INNERMOST, br);
-            let env_ty = tcx.mk_mut_ref(tcx.mk_region(env_region), ty);
+            let env_ty = tcx.mk_mut_ref(tcx.mk_re_late_bound(ty::INNERMOST, br), ty);
 
             let pin_did = tcx.require_lang_item(LangItem::Pin, None);
             let pin_adt_ref = tcx.adt_def(pin_did);
-            let pin_substs = tcx.intern_substs(&[env_ty.into()]);
+            let pin_substs = tcx.mk_substs(&[env_ty.into()]);
             let env_ty = tcx.mk_adt(pin_adt_ref, pin_substs);
 
             let sig = sig.skip_binder();
@@ -112,7 +111,7 @@ fn fn_sig_for_fn_abi<'tcx>(
                 // The signature should be `Future::poll(_, &mut Context<'_>) -> Poll<Output>`
                 let poll_did = tcx.require_lang_item(LangItem::Poll, None);
                 let poll_adt_ref = tcx.adt_def(poll_did);
-                let poll_substs = tcx.intern_substs(&[sig.return_ty.into()]);
+                let poll_substs = tcx.mk_substs(&[sig.return_ty.into()]);
                 let ret_ty = tcx.mk_adt(poll_adt_ref, poll_substs);
 
                 // We have to replace the `ResumeTy` that is used for type and borrow checking
@@ -134,7 +133,7 @@ fn fn_sig_for_fn_abi<'tcx>(
                 // The signature should be `Generator::resume(_, Resume) -> GeneratorState<Yield, Return>`
                 let state_did = tcx.require_lang_item(LangItem::GeneratorState, None);
                 let state_adt_ref = tcx.adt_def(state_did);
-                let state_substs = tcx.intern_substs(&[sig.yield_ty.into(), sig.return_ty.into()]);
+                let state_substs = tcx.mk_substs(&[sig.yield_ty.into(), sig.return_ty.into()]);
                 let ret_ty = tcx.mk_adt(state_adt_ref, state_substs);
 
                 (sig.resume_ty, ret_ty)
@@ -142,8 +141,8 @@ fn fn_sig_for_fn_abi<'tcx>(
 
             ty::Binder::bind_with_vars(
                 tcx.mk_fn_sig(
-                    [env_ty, resume_ty].iter(),
-                    &ret_ty,
+                    [env_ty, resume_ty],
+                    ret_ty,
                     false,
                     hir::Unsafety::Normal,
                     rustc_target::spec::abi::Abi::Rust,
@@ -207,11 +206,8 @@ fn fn_abi_of_instance<'tcx>(
 
     let sig = fn_sig_for_fn_abi(tcx, instance, param_env);
 
-    let caller_location = if instance.def.requires_caller_location(tcx) {
-        Some(tcx.caller_location_ty())
-    } else {
-        None
-    };
+    let caller_location =
+        instance.def.requires_caller_location(tcx).then(|| tcx.caller_location_ty());
 
     fn_abi_new_uncached(
         &LayoutCx { tcx, param_env },
@@ -244,7 +240,7 @@ fn adjust_for_rust_scalar<'tcx>(
     }
 
     // Only pointer types handled below.
-    let Scalar::Initialized { value: Pointer, valid_range} = scalar else { return };
+    let Scalar::Initialized { value: Pointer(_), valid_range} = scalar else { return };
 
     if !valid_range.contains(0) {
         attrs.set(ArgAttribute::NonNull);
@@ -254,15 +250,18 @@ fn adjust_for_rust_scalar<'tcx>(
         if let Some(kind) = pointee.safe {
             attrs.pointee_align = Some(pointee.align);
 
-            // `Box` (`UniqueBorrowed`) are not necessarily dereferenceable
-            // for the entire duration of the function as they can be deallocated
-            // at any time. Same for shared mutable references. If LLVM had a
-            // way to say "dereferenceable on entry" we could use it here.
+            // `Box` are not necessarily dereferenceable for the entire duration of the function as
+            // they can be deallocated at any time. Same for non-frozen shared references (see
+            // <https://github.com/rust-lang/rust/pull/98017>), and for mutable references to
+            // potentially self-referential types (see
+            // <https://github.com/rust-lang/unsafe-code-guidelines/issues/381>). If LLVM had a way
+            // to say "dereferenceable on entry" we could use it here.
             attrs.pointee_size = match kind {
-                PointerKind::UniqueBorrowed
-                | PointerKind::UniqueBorrowedPinned
-                | PointerKind::Frozen => pointee.size,
-                PointerKind::SharedMutable | PointerKind::UniqueOwned => Size::ZERO,
+                PointerKind::Box { .. }
+                | PointerKind::SharedRef { frozen: false }
+                | PointerKind::MutableRef { unpin: false } => Size::ZERO,
+                PointerKind::SharedRef { frozen: true }
+                | PointerKind::MutableRef { unpin: true } => pointee.size,
             };
 
             // The aliasing rules for `Box<T>` are still not decided, but currently we emit
@@ -275,18 +274,16 @@ fn adjust_for_rust_scalar<'tcx>(
             // versions at all anymore. We still support turning it off using -Zmutable-noalias.
             let noalias_mut_ref = cx.tcx.sess.opts.unstable_opts.mutable_noalias;
 
-            // `&mut` pointer parameters never alias other parameters,
-            // or mutable global data
+            // `&T` where `T` contains no `UnsafeCell<U>` is immutable, and can be marked as both
+            // `readonly` and `noalias`, as LLVM's definition of `noalias` is based solely on memory
+            // dependencies rather than pointer equality. However this only applies to arguments,
+            // not return values.
             //
-            // `&T` where `T` contains no `UnsafeCell<U>` is immutable,
-            // and can be marked as both `readonly` and `noalias`, as
-            // LLVM's definition of `noalias` is based solely on memory
-            // dependencies rather than pointer equality
+            // `&mut T` and `Box<T>` where `T: Unpin` are unique and hence `noalias`.
             let no_alias = match kind {
-                PointerKind::SharedMutable | PointerKind::UniqueBorrowedPinned => false,
-                PointerKind::UniqueBorrowed => noalias_mut_ref,
-                PointerKind::UniqueOwned => noalias_for_box,
-                PointerKind::Frozen => true,
+                PointerKind::SharedRef { frozen } => frozen,
+                PointerKind::MutableRef { unpin } => unpin && noalias_mut_ref,
+                PointerKind::Box { unpin } => unpin && noalias_for_box,
             };
             // We can never add `noalias` in return position; that LLVM attribute has some very surprising semantics
             // (see <https://github.com/rust-lang/unsafe-code-guidelines/issues/385#issuecomment-1368055745>).
@@ -294,7 +291,7 @@ fn adjust_for_rust_scalar<'tcx>(
                 attrs.set(ArgAttribute::NoAlias);
             }
 
-            if kind == PointerKind::Frozen && !is_return {
+            if matches!(kind, PointerKind::SharedRef { frozen: true }) && !is_return {
                 attrs.set(ArgAttribute::ReadOnly);
             }
         }
@@ -479,7 +476,7 @@ fn fn_abi_adjust_for_abi<'tcx>(
             }
 
             let size = arg.layout.size;
-            if arg.layout.is_unsized() || size > Pointer.size(cx) {
+            if arg.layout.is_unsized() || size > Pointer(AddressSpace::DATA).size(cx) {
                 arg.make_indirect();
             } else {
                 // We want to pass small aggregates as immediates, but using
diff --git a/compiler/rustc_ty_utils/src/assoc.rs b/compiler/rustc_ty_utils/src/assoc.rs
index 424b52309..0648784b2 100644
--- a/compiler/rustc_ty_utils/src/assoc.rs
+++ b/compiler/rustc_ty_utils/src/assoc.rs
@@ -1,13 +1,19 @@
 use rustc_data_structures::fx::FxHashMap;
 use rustc_hir as hir;
-use rustc_hir::def_id::DefId;
-use rustc_middle::ty::{self, TyCtxt};
+use rustc_hir::def::DefKind;
+use rustc_hir::def_id::{DefId, LocalDefId};
+use rustc_hir::definitions::DefPathData;
+use rustc_hir::intravisit::{self, Visitor};
+use rustc_middle::ty::{self, DefIdTree, ImplTraitInTraitData, InternalSubsts, TyCtxt};
+use rustc_span::symbol::kw;
 
 pub fn provide(providers: &mut ty::query::Providers) {
     *providers = ty::query::Providers {
         associated_item,
         associated_item_def_ids,
         associated_items,
+        associated_items_for_impl_trait_in_trait,
+        associated_item_for_impl_trait_in_trait,
         impl_item_implementor_ids,
         ..*providers
     };
@@ -16,20 +22,79 @@ pub fn provide(providers: &mut ty::query::Providers) {
 fn associated_item_def_ids(tcx: TyCtxt<'_>, def_id: DefId) -> &[DefId] {
     let item = tcx.hir().expect_item(def_id.expect_local());
     match item.kind {
-        hir::ItemKind::Trait(.., ref trait_item_refs) => tcx.arena.alloc_from_iter(
-            trait_item_refs.iter().map(|trait_item_ref| trait_item_ref.id.owner_id.to_def_id()),
-        ),
-        hir::ItemKind::Impl(ref impl_) => tcx.arena.alloc_from_iter(
-            impl_.items.iter().map(|impl_item_ref| impl_item_ref.id.owner_id.to_def_id()),
-        ),
-        hir::ItemKind::TraitAlias(..) => &[],
+        hir::ItemKind::Trait(.., ref trait_item_refs) => {
+            if tcx.sess.opts.unstable_opts.lower_impl_trait_in_trait_to_assoc_ty {
+                // We collect RPITITs for each trait method's return type and create a
+                // corresponding associated item using associated_items_for_impl_trait_in_trait
+                // query.
+                tcx.arena.alloc_from_iter(
+                    trait_item_refs
+                        .iter()
+                        .map(|trait_item_ref| trait_item_ref.id.owner_id.to_def_id())
+                        .chain(
+                            trait_item_refs
+                                .iter()
+                                .filter(|trait_item_ref| {
+                                    matches!(trait_item_ref.kind, hir::AssocItemKind::Fn { .. })
+                                })
+                                .flat_map(|trait_item_ref| {
+                                    let trait_fn_def_id =
+                                        trait_item_ref.id.owner_id.def_id.to_def_id();
+                                    tcx.associated_items_for_impl_trait_in_trait(trait_fn_def_id)
+                                })
+                                .map(|def_id| *def_id),
+                        ),
+                )
+            } else {
+                tcx.arena.alloc_from_iter(
+                    trait_item_refs
+                        .iter()
+                        .map(|trait_item_ref| trait_item_ref.id.owner_id.to_def_id()),
+                )
+            }
+        }
+        hir::ItemKind::Impl(ref impl_) => {
+            if tcx.sess.opts.unstable_opts.lower_impl_trait_in_trait_to_assoc_ty {
+                // We collect RPITITs for each trait method's return type, on the impl side too and
+                // create a corresponding associated item using
+                // associated_items_for_impl_trait_in_trait query.
+                tcx.arena.alloc_from_iter(
+                    impl_
+                        .items
+                        .iter()
+                        .map(|impl_item_ref| impl_item_ref.id.owner_id.to_def_id())
+                        .chain(impl_.of_trait.iter().flat_map(|_| {
+                            impl_
+                                .items
+                                .iter()
+                                .filter(|impl_item_ref| {
+                                    matches!(impl_item_ref.kind, hir::AssocItemKind::Fn { .. })
+                                })
+                                .flat_map(|impl_item_ref| {
+                                    let impl_fn_def_id =
+                                        impl_item_ref.id.owner_id.def_id.to_def_id();
+                                    tcx.associated_items_for_impl_trait_in_trait(impl_fn_def_id)
+                                })
+                                .map(|def_id| *def_id)
+                        })),
+                )
+            } else {
+                tcx.arena.alloc_from_iter(
+                    impl_.items.iter().map(|impl_item_ref| impl_item_ref.id.owner_id.to_def_id()),
+                )
+            }
+        }
         _ => span_bug!(item.span, "associated_item_def_ids: not impl or trait"),
     }
 }
 
-fn associated_items(tcx: TyCtxt<'_>, def_id: DefId) -> ty::AssocItems<'_> {
-    let items = tcx.associated_item_def_ids(def_id).iter().map(|did| tcx.associated_item(*did));
-    ty::AssocItems::new(items)
+fn associated_items(tcx: TyCtxt<'_>, def_id: DefId) -> ty::AssocItems {
+    if tcx.is_trait_alias(def_id) {
+        ty::AssocItems::new(Vec::new())
+    } else {
+        let items = tcx.associated_item_def_ids(def_id).iter().map(|did| tcx.associated_item(*did));
+        ty::AssocItems::new(items)
+    }
 }
 
 fn impl_item_implementor_ids(tcx: TyCtxt<'_>, impl_id: DefId) -> FxHashMap<DefId, DefId> {
@@ -109,3 +174,184 @@ fn associated_item_from_impl_item_ref(impl_item_ref: &hir::ImplItemRef) -> ty::A
         fn_has_self_parameter: has_self,
     }
 }
+
+/// Given an `fn_def_id` of a trait or of an impl that implements a given trait:
+/// if `fn_def_id` is the def id of a function defined inside a trait, then it creates and returns
+/// the associated items that correspond to each impl trait in return position for that trait.
+/// if `fn_def_id` is the def id of a function defined inside an impl that implements a trait, then it
+/// creates and returns the associated items that correspond to each impl trait in return position
+/// of the implemented trait.
+fn associated_items_for_impl_trait_in_trait(tcx: TyCtxt<'_>, fn_def_id: DefId) -> &'_ [DefId] {
+    let parent_def_id = tcx.parent(fn_def_id);
+
+    match tcx.def_kind(parent_def_id) {
+        DefKind::Trait => {
+            struct RPITVisitor {
+                rpits: Vec<LocalDefId>,
+            }
+
+            impl<'v> Visitor<'v> for RPITVisitor {
+                fn visit_ty(&mut self, ty: &'v hir::Ty<'v>) {
+                    if let hir::TyKind::OpaqueDef(item_id, _, _) = ty.kind {
+                        self.rpits.push(item_id.owner_id.def_id)
+                    }
+                    intravisit::walk_ty(self, ty)
+                }
+            }
+
+            let mut visitor = RPITVisitor { rpits: Vec::new() };
+
+            if let Some(output) = tcx.hir().get_fn_output(fn_def_id.expect_local()) {
+                visitor.visit_fn_ret_ty(output);
+
+                tcx.arena.alloc_from_iter(visitor.rpits.iter().map(|opaque_ty_def_id| {
+                    tcx.associated_item_for_impl_trait_in_trait(opaque_ty_def_id).to_def_id()
+                }))
+            } else {
+                &[]
+            }
+        }
+
+        DefKind::Impl { .. } => {
+            let Some(trait_fn_def_id) = tcx.associated_item(fn_def_id).trait_item_def_id else { return &[] };
+
+            tcx.arena.alloc_from_iter(
+                tcx.associated_items_for_impl_trait_in_trait(trait_fn_def_id).iter().map(
+                    move |trait_assoc_def_id| {
+                        impl_associated_item_for_impl_trait_in_trait(
+                            tcx,
+                            trait_assoc_def_id.expect_local(),
+                            fn_def_id.expect_local(),
+                        )
+                        .to_def_id()
+                    },
+                ),
+            )
+        }
+
+        def_kind => bug!(
+            "associated_items_for_impl_trait_in_trait: {:?} should be Trait or Impl but is {:?}",
+            parent_def_id,
+            def_kind
+        ),
+    }
+}
+
+/// Given an `opaque_ty_def_id` corresponding to an impl trait in trait, create and return the
+/// corresponding associated item.
+fn associated_item_for_impl_trait_in_trait(
+    tcx: TyCtxt<'_>,
+    opaque_ty_def_id: LocalDefId,
+) -> LocalDefId {
+    let fn_def_id = tcx.impl_trait_in_trait_parent(opaque_ty_def_id.to_def_id());
+    let trait_def_id = tcx.parent(fn_def_id);
+    assert_eq!(tcx.def_kind(trait_def_id), DefKind::Trait);
+
+    let span = tcx.def_span(opaque_ty_def_id);
+    let trait_assoc_ty =
+        tcx.at(span).create_def(trait_def_id.expect_local(), DefPathData::ImplTraitAssocTy);
+
+    let local_def_id = trait_assoc_ty.def_id();
+    let def_id = local_def_id.to_def_id();
+
+    trait_assoc_ty.opt_def_kind(Some(DefKind::AssocTy));
+
+    // There's no HIR associated with this new synthesized `def_id`, so feed
+    // `opt_local_def_id_to_hir_id` with `None`.
+    trait_assoc_ty.opt_local_def_id_to_hir_id(None);
+
+    // Copy span of the opaque.
+    trait_assoc_ty.def_ident_span(Some(span));
+
+    // Add the def_id of the function and opaque that generated this synthesized associated type.
+    trait_assoc_ty.opt_rpitit_info(Some(ImplTraitInTraitData::Trait {
+        fn_def_id,
+        opaque_def_id: opaque_ty_def_id.to_def_id(),
+    }));
+
+    trait_assoc_ty.associated_item(ty::AssocItem {
+        name: kw::Empty,
+        kind: ty::AssocKind::Type,
+        def_id,
+        trait_item_def_id: None,
+        container: ty::TraitContainer,
+        fn_has_self_parameter: false,
+    });
+
+    // Copy visility of the containing function.
+    trait_assoc_ty.visibility(tcx.visibility(fn_def_id));
+
+    // Copy impl_defaultness of the containing function.
+    trait_assoc_ty.impl_defaultness(tcx.impl_defaultness(fn_def_id));
+
+    // Copy type_of of the opaque.
+    trait_assoc_ty.type_of(ty::EarlyBinder(tcx.mk_opaque(
+        opaque_ty_def_id.to_def_id(),
+        InternalSubsts::identity_for_item(tcx, opaque_ty_def_id.to_def_id()),
+    )));
+
+    // Copy generics_of of the opaque.
+    trait_assoc_ty.generics_of(tcx.generics_of(opaque_ty_def_id).clone());
+
+    // There are no predicates for the synthesized associated type.
+    trait_assoc_ty.explicit_predicates_of(ty::GenericPredicates {
+        parent: Some(trait_def_id),
+        predicates: &[],
+    });
+
+    // There are no inferred outlives for the synthesized associated type.
+    trait_assoc_ty.inferred_outlives_of(&[]);
+
+    // FIXME implement this.
+    trait_assoc_ty.explicit_item_bounds(&[]);
+
+    local_def_id
+}
+
+/// Given an `trait_assoc_def_id` that corresponds to a previously synthesized impl trait in trait
+/// into an associated type and an `impl_def_id` corresponding to an impl block, create and return
+/// the corresponding associated item inside the impl block.
+fn impl_associated_item_for_impl_trait_in_trait(
+    tcx: TyCtxt<'_>,
+    trait_assoc_def_id: LocalDefId,
+    impl_fn_def_id: LocalDefId,
+) -> LocalDefId {
+    let impl_def_id = tcx.local_parent(impl_fn_def_id);
+
+    // FIXME fix the span, we probably want the def_id of the return type of the function
+    let span = tcx.def_span(impl_fn_def_id);
+    let impl_assoc_ty = tcx.at(span).create_def(impl_def_id, DefPathData::ImplTraitAssocTy);
+
+    let local_def_id = impl_assoc_ty.def_id();
+    let def_id = local_def_id.to_def_id();
+
+    impl_assoc_ty.opt_def_kind(Some(DefKind::AssocTy));
+
+    // There's no HIR associated with this new synthesized `def_id`, so feed
+    // `opt_local_def_id_to_hir_id` with `None`.
+    impl_assoc_ty.opt_local_def_id_to_hir_id(None);
+
+    // Add the def_id of the function that generated this synthesized associated type.
+    impl_assoc_ty.opt_rpitit_info(Some(ImplTraitInTraitData::Impl {
+        fn_def_id: impl_fn_def_id.to_def_id(),
+    }));
+
+    impl_assoc_ty.associated_item(ty::AssocItem {
+        name: kw::Empty,
+        kind: ty::AssocKind::Type,
+        def_id,
+        trait_item_def_id: Some(trait_assoc_def_id.to_def_id()),
+        container: ty::ImplContainer,
+        fn_has_self_parameter: false,
+    });
+
+    // Copy impl_defaultness of the containing function.
+    impl_assoc_ty.impl_defaultness(tcx.impl_defaultness(impl_fn_def_id));
+
+    // Copy generics_of the trait's associated item.
+    // FIXME: This is not correct, in particular the parent is going to be wrong. So we would need
+    // to copy from trait_assoc_def_id and adjust things.
+    impl_assoc_ty.generics_of(tcx.generics_of(trait_assoc_def_id).clone());
+
+    local_def_id
+}
diff --git a/compiler/rustc_ty_utils/src/consts.rs b/compiler/rustc_ty_utils/src/consts.rs
index a9fbad55d..f26352716 100644
--- a/compiler/rustc_ty_utils/src/consts.rs
+++ b/compiler/rustc_ty_utils/src/consts.rs
@@ -5,7 +5,7 @@ use rustc_middle::mir::interpret::{LitToConstError, LitToConstInput};
 use rustc_middle::thir::visit;
 use rustc_middle::thir::visit::Visitor;
 use rustc_middle::ty::abstract_const::CastKind;
-use rustc_middle::ty::{self, Expr, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, Expr, TyCtxt, TypeVisitableExt};
 use rustc_middle::{mir, thir};
 use rustc_span::Span;
 use rustc_target::abi::VariantIdx;
@@ -144,7 +144,7 @@ fn recurse_build<'tcx>(
             for &id in args.iter() {
                 new_args.push(recurse_build(tcx, body, id, root_span)?);
             }
-            let new_args = tcx.mk_const_list(new_args.iter());
+            let new_args = tcx.mk_const_list(&new_args);
             tcx.mk_const(Expr::FunctionCall(fun, new_args), node.ty)
         }
         &ExprKind::Binary { op, lhs, rhs } if check_binop(op) => {
diff --git a/compiler/rustc_ty_utils/src/errors.rs b/compiler/rustc_ty_utils/src/errors.rs
index c05eeb353..ab3e62f04 100644
--- a/compiler/rustc_ty_utils/src/errors.rs
+++ b/compiler/rustc_ty_utils/src/errors.rs
@@ -16,7 +16,7 @@ pub struct NeedsDropOverflow<'tcx> {
 pub struct GenericConstantTooComplex {
     #[primary_span]
     pub span: Span,
-    #[note(maybe_supported)]
+    #[note(ty_utils_maybe_supported)]
     pub maybe_supported: Option<()>,
     #[subdiagnostic]
     pub sub: GenericConstantTooComplexSub,
diff --git a/compiler/rustc_ty_utils/src/implied_bounds.rs b/compiler/rustc_ty_utils/src/implied_bounds.rs
index 7a2464580..7fecee2a3 100644
--- a/compiler/rustc_ty_utils/src/implied_bounds.rs
+++ b/compiler/rustc_ty_utils/src/implied_bounds.rs
@@ -9,26 +9,26 @@ pub fn provide(providers: &mut ty::query::Providers) {
 fn assumed_wf_types(tcx: TyCtxt<'_>, def_id: DefId) -> &ty::List<Ty<'_>> {
     match tcx.def_kind(def_id) {
         DefKind::Fn => {
-            let sig = tcx.fn_sig(def_id);
+            let sig = tcx.fn_sig(def_id).subst_identity();
             let liberated_sig = tcx.liberate_late_bound_regions(def_id, sig);
             liberated_sig.inputs_and_output
         }
         DefKind::AssocFn => {
-            let sig = tcx.fn_sig(def_id);
+            let sig = tcx.fn_sig(def_id).subst_identity();
             let liberated_sig = tcx.liberate_late_bound_regions(def_id, sig);
             let mut assumed_wf_types: Vec<_> =
                 tcx.assumed_wf_types(tcx.parent(def_id)).as_slice().into();
             assumed_wf_types.extend(liberated_sig.inputs_and_output);
-            tcx.intern_type_list(&assumed_wf_types)
+            tcx.mk_type_list(&assumed_wf_types)
         }
-        DefKind::Impl => {
+        DefKind::Impl { .. } => {
             match tcx.impl_trait_ref(def_id) {
                 Some(trait_ref) => {
                     let types: Vec<_> = trait_ref.skip_binder().substs.types().collect();
-                    tcx.intern_type_list(&types)
+                    tcx.mk_type_list(&types)
                 }
                 // Only the impl self type
-                None => tcx.intern_type_list(&[tcx.type_of(def_id)]),
+                None => tcx.mk_type_list(&[tcx.type_of(def_id).subst_identity()]),
             }
         }
         DefKind::AssocConst | DefKind::AssocTy => tcx.assumed_wf_types(tcx.parent(def_id)),
diff --git a/compiler/rustc_ty_utils/src/instance.rs b/compiler/rustc_ty_utils/src/instance.rs
index 8d46ba320..2eaeca73d 100644
--- a/compiler/rustc_ty_utils/src/instance.rs
+++ b/compiler/rustc_ty_utils/src/instance.rs
@@ -3,7 +3,7 @@ use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_infer::infer::TyCtxtInferExt;
 use rustc_middle::traits::CodegenObligationError;
 use rustc_middle::ty::subst::SubstsRef;
-use rustc_middle::ty::{self, Instance, TyCtxt, TypeVisitable};
+use rustc_middle::ty::{self, Instance, TyCtxt, TypeVisitableExt};
 use rustc_span::sym;
 use rustc_trait_selection::traits;
 use traits::{translate_substs, Reveal};
@@ -53,7 +53,8 @@ fn inner_resolve_instance<'tcx>(
         )
     } else {
         let ty = tcx.type_of(def.def_id_for_type_of());
-        let item_type = tcx.subst_and_normalize_erasing_regions(substs, param_env, ty);
+        let item_type =
+            tcx.subst_and_normalize_erasing_regions(substs, param_env, ty.skip_binder());
 
         let def = match *item_type.kind() {
             ty::FnDef(def_id, ..) if tcx.is_intrinsic(def_id) => {
diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs
index 6aa016133..e3132fcc4 100644
--- a/compiler/rustc_ty_utils/src/layout.rs
+++ b/compiler/rustc_ty_utils/src/layout.rs
@@ -7,9 +7,9 @@ use rustc_middle::ty::layout::{
     IntegerExt, LayoutCx, LayoutError, LayoutOf, TyAndLayout, MAX_SIMD_LANES,
 };
 use rustc_middle::ty::{
-    self, subst::SubstsRef, AdtDef, EarlyBinder, ReprOptions, Ty, TyCtxt, TypeVisitable,
+    self, subst::SubstsRef, AdtDef, EarlyBinder, ReprOptions, Ty, TyCtxt, TypeVisitableExt,
 };
-use rustc_session::{DataTypeKind, FieldInfo, SizeKind, VariantInfo};
+use rustc_session::{DataTypeKind, FieldInfo, FieldKind, SizeKind, VariantInfo};
 use rustc_span::symbol::Symbol;
 use rustc_span::DUMMY_SP;
 use rustc_target::abi::*;
@@ -78,10 +78,10 @@ fn invert_mapping(map: &[u32]) -> Vec<u32> {
 fn univariant_uninterned<'tcx>(
     cx: &LayoutCx<'tcx, TyCtxt<'tcx>>,
     ty: Ty<'tcx>,
-    fields: &[TyAndLayout<'_>],
+    fields: &[Layout<'_>],
     repr: &ReprOptions,
     kind: StructKind,
-) -> Result<LayoutS<VariantIdx>, LayoutError<'tcx>> {
+) -> Result<LayoutS, LayoutError<'tcx>> {
     let dl = cx.data_layout();
     let pack = repr.pack;
     if pack.is_some() && repr.align.is_some() {
@@ -104,23 +104,23 @@ fn layout_of_uncached<'tcx>(
         assert!(size.bits() <= 128);
         Scalar::Initialized { value, valid_range: WrappingRange::full(size) }
     };
-    let scalar = |value: Primitive| tcx.intern_layout(LayoutS::scalar(cx, scalar_unit(value)));
+    let scalar = |value: Primitive| tcx.mk_layout(LayoutS::scalar(cx, scalar_unit(value)));
 
-    let univariant = |fields: &[TyAndLayout<'_>], repr: &ReprOptions, kind| {
-        Ok(tcx.intern_layout(univariant_uninterned(cx, ty, fields, repr, kind)?))
+    let univariant = |fields: &[Layout<'_>], repr: &ReprOptions, kind| {
+        Ok(tcx.mk_layout(univariant_uninterned(cx, ty, fields, repr, kind)?))
     };
     debug_assert!(!ty.has_non_region_infer());
 
     Ok(match *ty.kind() {
         // Basic scalars.
-        ty::Bool => tcx.intern_layout(LayoutS::scalar(
+        ty::Bool => tcx.mk_layout(LayoutS::scalar(
             cx,
             Scalar::Initialized {
                 value: Int(I8, false),
                 valid_range: WrappingRange { start: 0, end: 1 },
             },
         )),
-        ty::Char => tcx.intern_layout(LayoutS::scalar(
+        ty::Char => tcx.mk_layout(LayoutS::scalar(
             cx,
             Scalar::Initialized {
                 value: Int(I32, false),
@@ -134,24 +134,24 @@ fn layout_of_uncached<'tcx>(
             ty::FloatTy::F64 => F64,
         }),
         ty::FnPtr(_) => {
-            let mut ptr = scalar_unit(Pointer);
+            let mut ptr = scalar_unit(Pointer(dl.instruction_address_space));
             ptr.valid_range_mut().start = 1;
-            tcx.intern_layout(LayoutS::scalar(cx, ptr))
+            tcx.mk_layout(LayoutS::scalar(cx, ptr))
         }
 
         // The never type.
-        ty::Never => tcx.intern_layout(cx.layout_of_never_type()),
+        ty::Never => tcx.mk_layout(cx.layout_of_never_type()),
 
         // Potentially-wide pointers.
         ty::Ref(_, pointee, _) | ty::RawPtr(ty::TypeAndMut { ty: pointee, .. }) => {
-            let mut data_ptr = scalar_unit(Pointer);
+            let mut data_ptr = scalar_unit(Pointer(AddressSpace::DATA));
             if !ty.is_unsafe_ptr() {
                 data_ptr.valid_range_mut().start = 1;
             }
 
             let pointee = tcx.normalize_erasing_regions(param_env, pointee);
             if pointee.is_sized(tcx, param_env) {
-                return Ok(tcx.intern_layout(LayoutS::scalar(cx, data_ptr)));
+                return Ok(tcx.mk_layout(LayoutS::scalar(cx, data_ptr)));
             }
 
             let unsized_part = tcx.struct_tail_erasing_lifetimes(pointee, param_env);
@@ -164,7 +164,7 @@ fn layout_of_uncached<'tcx>(
                 let metadata_layout = cx.layout_of(metadata_ty)?;
                 // If the metadata is a 1-zst, then the pointer is thin.
                 if metadata_layout.is_zst() && metadata_layout.align.abi.bytes() == 1 {
-                    return Ok(tcx.intern_layout(LayoutS::scalar(cx, data_ptr)));
+                    return Ok(tcx.mk_layout(LayoutS::scalar(cx, data_ptr)));
                 }
 
                 let Abi::Scalar(metadata) = metadata_layout.abi else {
@@ -174,11 +174,11 @@ fn layout_of_uncached<'tcx>(
             } else {
                 match unsized_part.kind() {
                     ty::Foreign(..) => {
-                        return Ok(tcx.intern_layout(LayoutS::scalar(cx, data_ptr)));
+                        return Ok(tcx.mk_layout(LayoutS::scalar(cx, data_ptr)));
                     }
                     ty::Slice(_) | ty::Str => scalar_unit(Int(dl.ptr_sized_integer(), false)),
                     ty::Dynamic(..) => {
-                        let mut vtable = scalar_unit(Pointer);
+                        let mut vtable = scalar_unit(Pointer(AddressSpace::DATA));
                         vtable.valid_range_mut().start = 1;
                         vtable
                     }
@@ -189,15 +189,15 @@ fn layout_of_uncached<'tcx>(
             };
 
             // Effectively a (ptr, meta) tuple.
-            tcx.intern_layout(cx.scalar_pair(data_ptr, metadata))
+            tcx.mk_layout(cx.scalar_pair(data_ptr, metadata))
         }
 
         ty::Dynamic(_, _, ty::DynStar) => {
-            let mut data = scalar_unit(Int(dl.ptr_sized_integer(), false));
+            let mut data = scalar_unit(Pointer(AddressSpace::DATA));
             data.valid_range_mut().start = 0;
-            let mut vtable = scalar_unit(Pointer);
+            let mut vtable = scalar_unit(Pointer(AddressSpace::DATA));
             vtable.valid_range_mut().start = 1;
-            tcx.intern_layout(cx.scalar_pair(data, vtable))
+            tcx.mk_layout(cx.scalar_pair(data, vtable))
         }
 
         // Arrays and slices.
@@ -209,7 +209,8 @@ fn layout_of_uncached<'tcx>(
                 }
             }
 
-            let count = count.try_eval_usize(tcx, param_env).ok_or(LayoutError::Unknown(ty))?;
+            let count =
+                count.try_eval_target_usize(tcx, param_env).ok_or(LayoutError::Unknown(ty))?;
             let element = cx.layout_of(element)?;
             let size = element.size.checked_mul(count, dl).ok_or(LayoutError::SizeOverflow(ty))?;
 
@@ -221,7 +222,7 @@ fn layout_of_uncached<'tcx>(
 
             let largest_niche = if count != 0 { element.largest_niche } else { None };
 
-            tcx.intern_layout(LayoutS {
+            tcx.mk_layout(LayoutS {
                 variants: Variants::Single { index: VariantIdx::new(0) },
                 fields: FieldsShape::Array { stride: element.size, count },
                 abi,
@@ -232,7 +233,7 @@ fn layout_of_uncached<'tcx>(
         }
         ty::Slice(element) => {
             let element = cx.layout_of(element)?;
-            tcx.intern_layout(LayoutS {
+            tcx.mk_layout(LayoutS {
                 variants: Variants::Single { index: VariantIdx::new(0) },
                 fields: FieldsShape::Array { stride: element.size, count: 0 },
                 abi: Abi::Aggregate { sized: false },
@@ -241,7 +242,7 @@ fn layout_of_uncached<'tcx>(
                 size: Size::ZERO,
             })
         }
-        ty::Str => tcx.intern_layout(LayoutS {
+        ty::Str => tcx.mk_layout(LayoutS {
             variants: Variants::Single { index: VariantIdx::new(0) },
             fields: FieldsShape::Array { stride: Size::from_bytes(1), count: 0 },
             abi: Abi::Aggregate { sized: false },
@@ -264,7 +265,7 @@ fn layout_of_uncached<'tcx>(
                 Abi::Aggregate { ref mut sized } => *sized = false,
                 _ => bug!(),
             }
-            tcx.intern_layout(unit)
+            tcx.mk_layout(unit)
         }
 
         ty::Generator(def_id, substs, _) => generator_layout(cx, ty, def_id, substs)?,
@@ -272,7 +273,7 @@ fn layout_of_uncached<'tcx>(
         ty::Closure(_, ref substs) => {
             let tys = substs.as_closure().upvar_tys();
             univariant(
-                &tys.map(|ty| cx.layout_of(ty)).collect::<Result<Vec<_>, _>>()?,
+                &tys.map(|ty| Ok(cx.layout_of(ty)?.layout)).collect::<Result<Vec<_>, _>>()?,
                 &ReprOptions::default(),
                 StructKind::AlwaysSized,
             )?
@@ -283,7 +284,7 @@ fn layout_of_uncached<'tcx>(
                 if tys.len() == 0 { StructKind::AlwaysSized } else { StructKind::MaybeUnsized };
 
             univariant(
-                &tys.iter().map(|k| cx.layout_of(k)).collect::<Result<Vec<_>, _>>()?,
+                &tys.iter().map(|k| Ok(cx.layout_of(k)?.layout)).collect::<Result<Vec<_>, _>>()?,
                 &ReprOptions::default(),
                 kind,
             )?
@@ -393,7 +394,7 @@ fn layout_of_uncached<'tcx>(
                 FieldsShape::Array { stride: e_ly.size, count: e_len }
             };
 
-            tcx.intern_layout(LayoutS {
+            tcx.mk_layout(LayoutS {
                 variants: Variants::Single { index: VariantIdx::new(0) },
                 fields,
                 abi: Abi::Vector { element: e_abi, count: e_len },
@@ -412,7 +413,7 @@ fn layout_of_uncached<'tcx>(
                 .map(|v| {
                     v.fields
                         .iter()
-                        .map(|field| cx.layout_of(field.ty(tcx, substs)))
+                        .map(|field| Ok(cx.layout_of(field.ty(tcx, substs))?.layout))
                         .collect::<Result<Vec<_>, _>>()
                 })
                 .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
@@ -426,12 +427,12 @@ fn layout_of_uncached<'tcx>(
                     return Err(LayoutError::Unknown(ty));
                 }
 
-                return Ok(tcx.intern_layout(
+                return Ok(tcx.mk_layout(
                     cx.layout_of_union(&def.repr(), &variants).ok_or(LayoutError::Unknown(ty))?,
                 ));
             }
 
-            tcx.intern_layout(
+            tcx.mk_layout(
                 cx.layout_of_struct_or_enum(
                     &def.repr(),
                     &variants,
@@ -452,9 +453,10 @@ fn layout_of_uncached<'tcx>(
                         let param_env = tcx.param_env(def.did());
                         def.is_struct()
                             && match def.variants().iter().next().and_then(|x| x.fields.last()) {
-                                Some(last_field) => {
-                                    tcx.type_of(last_field.did).is_sized(tcx, param_env)
-                                }
+                                Some(last_field) => tcx
+                                    .type_of(last_field.did)
+                                    .subst_identity()
+                                    .is_sized(tcx, param_env),
                                 None => false,
                             }
                     },
@@ -470,11 +472,11 @@ fn layout_of_uncached<'tcx>(
             return Err(LayoutError::Unknown(ty));
         }
 
-        ty::Placeholder(..) | ty::GeneratorWitness(..) | ty::Infer(_) => {
+        ty::Bound(..) | ty::GeneratorWitness(..) | ty::GeneratorWitnessMIR(..) | ty::Infer(_) => {
             bug!("Layout::compute: unexpected type `{}`", ty)
         }
 
-        ty::Bound(..) | ty::Param(_) | ty::Error(_) => {
+        ty::Placeholder(..) | ty::Param(_) | ty::Error(_) => {
             return Err(LayoutError::Unknown(ty));
         }
     })
@@ -630,23 +632,21 @@ fn generator_layout<'tcx>(
     // `info.variant_fields` already accounts for the reserved variants, so no need to add them.
     let max_discr = (info.variant_fields.len() - 1) as u128;
     let discr_int = Integer::fit_unsigned(max_discr);
-    let discr_int_ty = discr_int.to_ty(tcx, false);
     let tag = Scalar::Initialized {
         value: Primitive::Int(discr_int, false),
         valid_range: WrappingRange { start: 0, end: max_discr },
     };
-    let tag_layout = cx.tcx.intern_layout(LayoutS::scalar(cx, tag));
-    let tag_layout = TyAndLayout { ty: discr_int_ty, layout: tag_layout };
+    let tag_layout = cx.tcx.mk_layout(LayoutS::scalar(cx, tag));
 
     let promoted_layouts = ineligible_locals
         .iter()
-        .map(|local| subst_field(info.field_tys[local]))
+        .map(|local| subst_field(info.field_tys[local].ty))
         .map(|ty| tcx.mk_maybe_uninit(ty))
-        .map(|ty| cx.layout_of(ty));
+        .map(|ty| Ok(cx.layout_of(ty)?.layout));
     let prefix_layouts = substs
         .as_generator()
         .prefix_tys()
-        .map(|ty| cx.layout_of(ty))
+        .map(|ty| Ok(cx.layout_of(ty)?.layout))
         .chain(iter::once(Ok(tag_layout)))
         .chain(promoted_layouts)
         .collect::<Result<Vec<_>, _>>()?;
@@ -710,12 +710,14 @@ fn generator_layout<'tcx>(
                     Assigned(_) => bug!("assignment does not match variant"),
                     Ineligible(_) => false,
                 })
-                .map(|local| subst_field(info.field_tys[*local]));
+                .map(|local| subst_field(info.field_tys[*local].ty));
 
             let mut variant = univariant_uninterned(
                 cx,
                 ty,
-                &variant_only_tys.map(|ty| cx.layout_of(ty)).collect::<Result<Vec<_>, _>>()?,
+                &variant_only_tys
+                    .map(|ty| Ok(cx.layout_of(ty)?.layout))
+                    .collect::<Result<Vec<_>, _>>()?,
                 &ReprOptions::default(),
                 StructKind::Prefixed(prefix_size, prefix_align.abi),
             )?;
@@ -782,7 +784,7 @@ fn generator_layout<'tcx>(
         Abi::Aggregate { sized: true }
     };
 
-    let layout = tcx.intern_layout(LayoutS {
+    let layout = tcx.mk_layout(LayoutS {
         variants: Variants::Multiple {
             tag,
             tag_encoding: TagEncoding::Direct,
@@ -878,6 +880,7 @@ fn variant_info_for_adt<'tcx>(
                 let offset = layout.fields.offset(i);
                 min_size = min_size.max(offset + field_layout.size);
                 FieldInfo {
+                    kind: FieldKind::AdtField,
                     name,
                     offset: offset.bytes(),
                     size: field_layout.size.bytes(),
@@ -957,6 +960,7 @@ fn variant_info_for_generator<'tcx>(
             let offset = layout.fields.offset(field_idx);
             upvars_size = upvars_size.max(offset + field_layout.size);
             FieldInfo {
+                kind: FieldKind::Upvar,
                 name: Symbol::intern(&name),
                 offset: offset.bytes(),
                 size: field_layout.size.bytes(),
@@ -965,7 +969,7 @@ fn variant_info_for_generator<'tcx>(
         })
         .collect();
 
-    let variant_infos: Vec<_> = generator
+    let mut variant_infos: Vec<_> = generator
         .variant_fields
         .iter_enumerated()
         .map(|(variant_idx, variant_def)| {
@@ -980,6 +984,7 @@ fn variant_info_for_generator<'tcx>(
                     // The struct is as large as the last field's end
                     variant_size = variant_size.max(offset + field_layout.size);
                     FieldInfo {
+                        kind: FieldKind::GeneratorLocal,
                         name: state_specific_names.get(*local).copied().flatten().unwrap_or(
                             Symbol::intern(&format!(".generator_field{}", local.as_usize())),
                         ),
@@ -1027,6 +1032,15 @@ fn variant_info_for_generator<'tcx>(
             }
         })
         .collect();
+
+    // The first three variants are hardcoded to be `UNRESUMED`, `RETURNED` and `POISONED`.
+    // We will move the `RETURNED` and `POISONED` elements to the end so we
+    // are left with a sorting order according to the generators yield points:
+    // First `Unresumed`, then the `SuspendN` followed by `Returned` and `Panicked` (POISONED).
+    let end_states = variant_infos.drain(1..=2);
+    let end_states: Vec<_> = end_states.collect();
+    variant_infos.extend(end_states);
+
     (
         variant_infos,
         match tag_encoding {
diff --git a/compiler/rustc_ty_utils/src/lib.rs b/compiler/rustc_ty_utils/src/lib.rs
index 0853de601..35f468aa9 100644
--- a/compiler/rustc_ty_utils/src/lib.rs
+++ b/compiler/rustc_ty_utils/src/lib.rs
@@ -15,6 +15,8 @@ extern crate rustc_middle;
 #[macro_use]
 extern crate tracing;
 
+use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
+use rustc_macros::fluent_messages;
 use rustc_middle::ty::query::Providers;
 
 mod abi;
@@ -31,6 +33,8 @@ pub mod representability;
 mod structural_match;
 mod ty;
 
+fluent_messages! { "../locales/en-US.ftl" }
+
 pub fn provide(providers: &mut Providers) {
     abi::provide(providers);
     assoc::provide(providers);
diff --git a/compiler/rustc_ty_utils/src/needs_drop.rs b/compiler/rustc_ty_utils/src/needs_drop.rs
index 0df060fc5..de7fd0031 100644
--- a/compiler/rustc_ty_utils/src/needs_drop.rs
+++ b/compiler/rustc_ty_utils/src/needs_drop.rs
@@ -109,6 +109,13 @@ where
 
             for component in components {
                 match *component.kind() {
+                    // The information required to determine whether a generator has drop is
+                    // computed on MIR, while this very method is used to build MIR.
+                    // To avoid cycles, we consider that generators always require drop.
+                    ty::Generator(..) if tcx.sess.opts.unstable_opts.drop_tracking_mir => {
+                        return Some(Err(AlwaysRequiresDrop));
+                    }
+
                     _ if component.is_copy_modulo_regions(tcx, self.param_env) => (),
 
                     ty::Closure(_, substs) => {
@@ -235,7 +242,7 @@ fn drop_tys_helper<'tcx>(
             Ok(Vec::new())
         } else {
             let field_tys = adt_def.all_fields().map(|field| {
-                let r = tcx.bound_type_of(field.did).subst(tcx, substs);
+                let r = tcx.type_of(field.did).subst(tcx, substs);
                 debug!("drop_tys_helper: Subst into {:?} with {:?} gettng {:?}", field, substs, r);
                 r
             });
@@ -288,9 +295,15 @@ fn adt_drop_tys<'tcx>(
     let adt_has_dtor =
         |adt_def: ty::AdtDef<'tcx>| adt_def.destructor(tcx).map(|_| DtorType::Significant);
     // `tcx.type_of(def_id)` identical to `tcx.make_adt(def, identity_substs)`
-    drop_tys_helper(tcx, tcx.type_of(def_id), tcx.param_env(def_id), adt_has_dtor, false)
-        .collect::<Result<Vec<_>, _>>()
-        .map(|components| tcx.intern_type_list(&components))
+    drop_tys_helper(
+        tcx,
+        tcx.type_of(def_id).subst_identity(),
+        tcx.param_env(def_id),
+        adt_has_dtor,
+        false,
+    )
+    .collect::<Result<Vec<_>, _>>()
+    .map(|components| tcx.mk_type_list(&components))
 }
 // If `def_id` refers to a generic ADT, the queries above and below act as if they had been handed
 // a `tcx.make_ty(def, identity_substs)` and as such it is legal to substitute the generic parameters
@@ -301,13 +314,13 @@ fn adt_significant_drop_tys(
 ) -> Result<&ty::List<Ty<'_>>, AlwaysRequiresDrop> {
     drop_tys_helper(
         tcx,
-        tcx.type_of(def_id), // identical to `tcx.make_adt(def, identity_substs)`
+        tcx.type_of(def_id).subst_identity(), // identical to `tcx.make_adt(def, identity_substs)`
         tcx.param_env(def_id),
         adt_consider_insignificant_dtor(tcx),
         true,
     )
     .collect::<Result<Vec<_>, _>>()
-    .map(|components| tcx.intern_type_list(&components))
+    .map(|components| tcx.mk_type_list(&components))
 }
 
 pub(crate) fn provide(providers: &mut ty::query::Providers) {
diff --git a/compiler/rustc_ty_utils/src/representability.rs b/compiler/rustc_ty_utils/src/representability.rs
index 7f48fb804..591017eec 100644
--- a/compiler/rustc_ty_utils/src/representability.rs
+++ b/compiler/rustc_ty_utils/src/representability.rs
@@ -31,7 +31,7 @@ fn representability(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Representability {
             }
             Representability::Representable
         }
-        DefKind::Field => representability_ty(tcx, tcx.type_of(def_id)),
+        DefKind::Field => representability_ty(tcx, tcx.type_of(def_id).subst_identity()),
         def_kind => bug!("unexpected {def_kind:?}"),
     }
 }
@@ -91,7 +91,7 @@ fn params_in_repr(tcx: TyCtxt<'_>, def_id: DefId) -> BitSet<u32> {
     let mut params_in_repr = BitSet::new_empty(generics.params.len());
     for variant in adt_def.variants() {
         for field in variant.fields.iter() {
-            params_in_repr_ty(tcx, tcx.type_of(field.did), &mut params_in_repr);
+            params_in_repr_ty(tcx, tcx.type_of(field.did).subst_identity(), &mut params_in_repr);
         }
     }
     params_in_repr
diff --git a/compiler/rustc_ty_utils/src/ty.rs b/compiler/rustc_ty_utils/src/ty.rs
index eb5454bf2..18159778a 100644
--- a/compiler/rustc_ty_utils/src/ty.rs
+++ b/compiler/rustc_ty_utils/src/ty.rs
@@ -1,8 +1,13 @@
-use rustc_data_structures::fx::FxIndexSet;
+use rustc_data_structures::fx::{FxHashSet, FxIndexSet};
 use rustc_hir as hir;
-use rustc_hir::def_id::DefId;
-use rustc_middle::ty::{self, Binder, Predicate, PredicateKind, ToPredicate, Ty, TyCtxt};
+use rustc_hir::def::DefKind;
+use rustc_index::bit_set::BitSet;
+use rustc_middle::ty::{
+    self, Binder, EarlyBinder, Predicate, PredicateKind, ToPredicate, Ty, TyCtxt,
+    TypeSuperVisitable, TypeVisitable, TypeVisitor,
+};
 use rustc_session::config::TraitSolver;
+use rustc_span::def_id::{DefId, CRATE_DEF_ID};
 use rustc_trait_selection::traits;
 
 fn sized_constraint_for_ty<'tcx>(
@@ -16,7 +21,13 @@ fn sized_constraint_for_ty<'tcx>(
         Bool | Char | Int(..) | Uint(..) | Float(..) | RawPtr(..) | Ref(..) | FnDef(..)
         | FnPtr(_) | Array(..) | Closure(..) | Generator(..) | Never => vec![],
 
-        Str | Dynamic(..) | Slice(_) | Foreign(..) | Error(_) | GeneratorWitness(..) => {
+        Str
+        | Dynamic(..)
+        | Slice(_)
+        | Foreign(..)
+        | Error(_)
+        | GeneratorWitness(..)
+        | GeneratorWitnessMIR(..) => {
             // these are never sized - return the target type
             vec![ty]
         }
@@ -87,16 +98,16 @@ fn impl_defaultness(tcx: TyCtxt<'_>, def_id: DefId) -> hir::Defaultness {
 fn adt_sized_constraint(tcx: TyCtxt<'_>, def_id: DefId) -> &[Ty<'_>] {
     if let Some(def_id) = def_id.as_local() {
         if matches!(tcx.representability(def_id), ty::Representability::Infinite) {
-            return tcx.intern_type_list(&[tcx.ty_error()]);
+            return tcx.mk_type_list(&[tcx.ty_error_misc()]);
         }
     }
     let def = tcx.adt_def(def_id);
 
-    let result = tcx.mk_type_list(
+    let result = tcx.mk_type_list_from_iter(
         def.variants()
             .iter()
             .flat_map(|v| v.fields.last())
-            .flat_map(|f| sized_constraint_for_ty(tcx, def, tcx.type_of(f.did))),
+            .flat_map(|f| sized_constraint_for_ty(tcx, def, tcx.type_of(f.did).subst_identity())),
     );
 
     debug!("adt_sized_constraint: {:?} => {:?}", def, result);
@@ -127,6 +138,19 @@ fn param_env(tcx: TyCtxt<'_>, def_id: DefId) -> ty::ParamEnv<'_> {
         predicates.extend(environment);
     }
 
+    if tcx.def_kind(def_id) == DefKind::AssocFn
+        && tcx.associated_item(def_id).container == ty::AssocItemContainer::TraitContainer
+    {
+        let sig = tcx.fn_sig(def_id).subst_identity();
+        sig.visit_with(&mut ImplTraitInTraitFinder {
+            tcx,
+            fn_def_id: def_id,
+            bound_vars: sig.bound_vars(),
+            predicates: &mut predicates,
+            seen: FxHashSet::default(),
+        });
+    }
+
     let local_did = def_id.as_local();
     let hir_id = local_did.map(|def_id| tcx.hir().local_def_id_to_hir_id(def_id));
 
@@ -202,24 +226,54 @@ fn param_env(tcx: TyCtxt<'_>, def_id: DefId) -> ty::ParamEnv<'_> {
         None => hir::Constness::NotConst,
     };
 
-    let unnormalized_env = ty::ParamEnv::new(
-        tcx.intern_predicates(&predicates),
-        traits::Reveal::UserFacing,
-        constness,
-    );
-
-    let body_id =
-        local_did.and_then(|id| tcx.hir().maybe_body_owned_by(id).map(|body| body.hir_id));
-    let body_id = match body_id {
-        Some(id) => id,
-        None if hir_id.is_some() => hir_id.unwrap(),
-        _ => hir::CRATE_HIR_ID,
-    };
+    let unnormalized_env =
+        ty::ParamEnv::new(tcx.mk_predicates(&predicates), traits::Reveal::UserFacing, constness);
 
+    let body_id = local_did.unwrap_or(CRATE_DEF_ID);
     let cause = traits::ObligationCause::misc(tcx.def_span(def_id), body_id);
     traits::normalize_param_env_or_error(tcx, unnormalized_env, cause)
 }
 
+/// Walk through a function type, gathering all RPITITs and installing a
+/// `NormalizesTo(Projection(RPITIT) -> Opaque(RPITIT))` predicate into the
+/// predicates list. This allows us to observe that an RPITIT projects to
+/// its corresponding opaque within the body of a default-body trait method.
+struct ImplTraitInTraitFinder<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    predicates: &'a mut Vec<Predicate<'tcx>>,
+    fn_def_id: DefId,
+    bound_vars: &'tcx ty::List<ty::BoundVariableKind>,
+    seen: FxHashSet<DefId>,
+}
+
+impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for ImplTraitInTraitFinder<'_, 'tcx> {
+    fn visit_ty(&mut self, ty: Ty<'tcx>) -> std::ops::ControlFlow<Self::BreakTy> {
+        if let ty::Alias(ty::Projection, alias_ty) = *ty.kind()
+            && self.tcx.def_kind(alias_ty.def_id) == DefKind::ImplTraitPlaceholder
+            && self.tcx.impl_trait_in_trait_parent(alias_ty.def_id) == self.fn_def_id
+            && self.seen.insert(alias_ty.def_id)
+        {
+            self.predicates.push(
+                ty::Binder::bind_with_vars(
+                    ty::ProjectionPredicate {
+                        projection_ty: alias_ty,
+                        term: self.tcx.mk_alias(ty::Opaque, alias_ty).into(),
+                    },
+                    self.bound_vars,
+                )
+                .to_predicate(self.tcx),
+            );
+
+            for bound in self.tcx.item_bounds(alias_ty.def_id).subst_iter(self.tcx, alias_ty.substs)
+            {
+                bound.visit_with(self);
+            }
+        }
+
+        ty.super_visit_with(self)
+    }
+}
+
 /// Elaborate the environment.
 ///
 /// Collect a list of `Predicate`'s used for building the `ParamEnv`. Adds `TypeWellFormedFromEnv`'s
@@ -299,14 +353,14 @@ fn well_formed_types_in_env(tcx: TyCtxt<'_>, def_id: DefId) -> &ty::List<Predica
         // In an inherent impl, we assume that the receiver type and all its
         // constituents are well-formed.
         NodeKind::InherentImpl => {
-            let self_ty = tcx.type_of(def_id);
+            let self_ty = tcx.type_of(def_id).subst_identity();
             inputs.extend(self_ty.walk());
         }
 
         // In an fn, we assume that the arguments and all their constituents are
         // well-formed.
         NodeKind::Fn => {
-            let fn_sig = tcx.fn_sig(def_id);
+            let fn_sig = tcx.fn_sig(def_id).subst_identity();
             let fn_sig = tcx.liberate_late_bound_regions(def_id, fn_sig);
 
             inputs.extend(fn_sig.inputs().iter().flat_map(|ty| ty.walk()));
@@ -329,7 +383,7 @@ fn well_formed_types_in_env(tcx: TyCtxt<'_>, def_id: DefId) -> &ty::List<Predica
         }
     });
 
-    tcx.mk_predicates(clauses.chain(input_clauses))
+    tcx.mk_predicates_from_iter(clauses.chain(input_clauses))
 }
 
 fn param_env_reveal_all_normalized(tcx: TyCtxt<'_>, def_id: DefId) -> ty::ParamEnv<'_> {
@@ -355,7 +409,7 @@ fn instance_def_size_estimate<'tcx>(
 /// If `def_id` is an issue 33140 hack impl, returns its self type; otherwise, returns `None`.
 ///
 /// See [`ty::ImplOverlapKind::Issue33140`] for more details.
-fn issue33140_self_ty(tcx: TyCtxt<'_>, def_id: DefId) -> Option<Ty<'_>> {
+fn issue33140_self_ty(tcx: TyCtxt<'_>, def_id: DefId) -> Option<EarlyBinder<Ty<'_>>> {
     debug!("issue33140_self_ty({:?})", def_id);
 
     let trait_ref = tcx
@@ -394,7 +448,7 @@ fn issue33140_self_ty(tcx: TyCtxt<'_>, def_id: DefId) -> Option<Ty<'_>> {
 
     if self_ty_matches {
         debug!("issue33140_self_ty - MATCHES!");
-        Some(self_ty)
+        Some(EarlyBinder(self_ty))
     } else {
         debug!("issue33140_self_ty - non-matching self type");
         None
@@ -407,6 +461,52 @@ fn asyncness(tcx: TyCtxt<'_>, def_id: DefId) -> hir::IsAsync {
     node.fn_sig().map_or(hir::IsAsync::NotAsync, |sig| sig.header.asyncness)
 }
 
+fn unsizing_params_for_adt<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> BitSet<u32> {
+    let def = tcx.adt_def(def_id);
+    let num_params = tcx.generics_of(def_id).count();
+
+    let maybe_unsizing_param_idx = |arg: ty::GenericArg<'tcx>| match arg.unpack() {
+        ty::GenericArgKind::Type(ty) => match ty.kind() {
+            ty::Param(p) => Some(p.index),
+            _ => None,
+        },
+
+        // We can't unsize a lifetime
+        ty::GenericArgKind::Lifetime(_) => None,
+
+        ty::GenericArgKind::Const(ct) => match ct.kind() {
+            ty::ConstKind::Param(p) => Some(p.index),
+            _ => None,
+        },
+    };
+
+    // The last field of the structure has to exist and contain type/const parameters.
+    let Some((tail_field, prefix_fields)) =
+        def.non_enum_variant().fields.split_last() else
+    {
+        return BitSet::new_empty(num_params);
+    };
+
+    let mut unsizing_params = BitSet::new_empty(num_params);
+    for arg in tcx.type_of(tail_field.did).subst_identity().walk() {
+        if let Some(i) = maybe_unsizing_param_idx(arg) {
+            unsizing_params.insert(i);
+        }
+    }
+
+    // Ensure none of the other fields mention the parameters used
+    // in unsizing.
+    for field in prefix_fields {
+        for arg in tcx.type_of(field.did).subst_identity().walk() {
+            if let Some(i) = maybe_unsizing_param_idx(arg) {
+                unsizing_params.remove(i);
+            }
+        }
+    }
+
+    unsizing_params
+}
+
 pub fn provide(providers: &mut ty::query::Providers) {
     *providers = ty::query::Providers {
         asyncness,
@@ -416,6 +516,7 @@ pub fn provide(providers: &mut ty::query::Providers) {
         instance_def_size_estimate,
         issue33140_self_ty,
         impl_defaultness,
+        unsizing_params_for_adt,
         ..*providers
     };
 }
diff --git a/compiler/rustc_type_ir/src/fold.rs b/compiler/rustc_type_ir/src/fold.rs
new file mode 100644
index 000000000..ee4ef57c3
--- /dev/null
+++ b/compiler/rustc_type_ir/src/fold.rs
@@ -0,0 +1,239 @@
+//! A folding traversal mechanism for complex data structures that contain type
+//! information.
+//!
+//! This is a modifying traversal. It consumes the data structure, producing a
+//! (possibly) modified version of it. Both fallible and infallible versions are
+//! available. The name is potentially confusing, because this traversal is more
+//! like `Iterator::map` than `Iterator::fold`.
+//!
+//! This traversal has limited flexibility. Only a small number of "types of
+//! interest" within the complex data structures can receive custom
+//! modification. These are the ones containing the most important type-related
+//! information, such as `Ty`, `Predicate`, `Region`, and `Const`.
+//!
+//! There are three groups of traits involved in each traversal.
+//! - `TypeFoldable`. This is implemented once for many types, including:
+//!   - Types of interest, for which the methods delegate to the folder.
+//!   - All other types, including generic containers like `Vec` and `Option`.
+//!     It defines a "skeleton" of how they should be folded.
+//! - `TypeSuperFoldable`. This is implemented only for each type of interest,
+//!   and defines the folding "skeleton" for these types.
+//! - `TypeFolder`/`FallibleTypeFolder. One of these is implemented for each
+//!   folder. This defines how types of interest are folded.
+//!
+//! This means each fold is a mixture of (a) generic folding operations, and (b)
+//! custom fold operations that are specific to the folder.
+//! - The `TypeFoldable` impls handle most of the traversal, and call into
+//!   `TypeFolder`/`FallibleTypeFolder` when they encounter a type of interest.
+//! - A `TypeFolder`/`FallibleTypeFolder` may call into another `TypeFoldable`
+//!   impl, because some of the types of interest are recursive and can contain
+//!   other types of interest.
+//! - A `TypeFolder`/`FallibleTypeFolder` may also call into a `TypeSuperFoldable`
+//!   impl, because each folder might provide custom handling only for some types
+//!   of interest, or only for some variants of each type of interest, and then
+//!   use default traversal for the remaining cases.
+//!
+//! For example, if you have `struct S(Ty, U)` where `S: TypeFoldable` and `U:
+//! TypeFoldable`, and an instance `s = S(ty, u)`, it would be folded like so:
+//! ```text
+//! s.fold_with(folder) calls
+//! - ty.fold_with(folder) calls
+//!   - folder.fold_ty(ty) may call
+//!     - ty.super_fold_with(folder)
+//! - u.fold_with(folder)
+//! ```
+use crate::{visit::TypeVisitable, Interner};
+
+/// This trait is implemented for every type that can be folded,
+/// providing the skeleton of the traversal.
+///
+/// To implement this conveniently, use the derive macro located in
+/// `rustc_macros`.
+pub trait TypeFoldable<I: Interner>: TypeVisitable<I> {
+    /// The entry point for folding. To fold a value `t` with a folder `f`
+    /// call: `t.try_fold_with(f)`.
+    ///
+    /// For most types, this just traverses the value, calling `try_fold_with`
+    /// on each field/element.
+    ///
+    /// For types of interest (such as `Ty`), the implementation of method
+    /// calls a folder method specifically for that type (such as
+    /// `F::try_fold_ty`). This is where control transfers from `TypeFoldable`
+    /// to `TypeFolder`.
+    fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error>;
+
+    /// A convenient alternative to `try_fold_with` for use with infallible
+    /// folders. Do not override this method, to ensure coherence with
+    /// `try_fold_with`.
+    fn fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self {
+        self.try_fold_with(folder).into_ok()
+    }
+}
+
+// This trait is implemented for types of interest.
+pub trait TypeSuperFoldable<I: Interner>: TypeFoldable<I> {
+    /// Provides a default fold for a type of interest. This should only be
+    /// called within `TypeFolder` methods, when a non-custom traversal is
+    /// desired for the value of the type of interest passed to that method.
+    /// For example, in `MyFolder::try_fold_ty(ty)`, it is valid to call
+    /// `ty.try_super_fold_with(self)`, but any other folding should be done
+    /// with `xyz.try_fold_with(self)`.
+    fn try_super_fold_with<F: FallibleTypeFolder<I>>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self, F::Error>;
+
+    /// A convenient alternative to `try_super_fold_with` for use with
+    /// infallible folders. Do not override this method, to ensure coherence
+    /// with `try_super_fold_with`.
+    fn super_fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self {
+        self.try_super_fold_with(folder).into_ok()
+    }
+}
+
+/// This trait is implemented for every infallible folding traversal. There is
+/// a fold method defined for every type of interest. Each such method has a
+/// default that does an "identity" fold. Implementations of these methods
+/// often fall back to a `super_fold_with` method if the primary argument
+/// doesn't satisfy a particular condition.
+///
+/// A blanket implementation of [`FallibleTypeFolder`] will defer to
+/// the infallible methods of this trait to ensure that the two APIs
+/// are coherent.
+pub trait TypeFolder<I: Interner>: FallibleTypeFolder<I, Error = !> {
+    fn interner(&self) -> I;
+
+    fn fold_binder<T>(&mut self, t: I::Binder<T>) -> I::Binder<T>
+    where
+        T: TypeFoldable<I>,
+        I::Binder<T>: TypeSuperFoldable<I>,
+    {
+        t.super_fold_with(self)
+    }
+
+    fn fold_ty(&mut self, t: I::Ty) -> I::Ty
+    where
+        I::Ty: TypeSuperFoldable<I>,
+    {
+        t.super_fold_with(self)
+    }
+
+    fn fold_region(&mut self, r: I::Region) -> I::Region
+    where
+        I::Region: TypeSuperFoldable<I>,
+    {
+        r.super_fold_with(self)
+    }
+
+    fn fold_const(&mut self, c: I::Const) -> I::Const
+    where
+        I::Const: TypeSuperFoldable<I>,
+    {
+        c.super_fold_with(self)
+    }
+
+    fn fold_predicate(&mut self, p: I::Predicate) -> I::Predicate
+    where
+        I::Predicate: TypeSuperFoldable<I>,
+    {
+        p.super_fold_with(self)
+    }
+}
+
+/// This trait is implemented for every folding traversal. There is a fold
+/// method defined for every type of interest. Each such method has a default
+/// that does an "identity" fold.
+///
+/// A blanket implementation of this trait (that defers to the relevant
+/// method of [`TypeFolder`]) is provided for all infallible folders in
+/// order to ensure the two APIs are coherent.
+pub trait FallibleTypeFolder<I: Interner>: Sized {
+    type Error;
+
+    fn interner(&self) -> I;
+
+    fn try_fold_binder<T>(&mut self, t: I::Binder<T>) -> Result<I::Binder<T>, Self::Error>
+    where
+        T: TypeFoldable<I>,
+        I::Binder<T>: TypeSuperFoldable<I>,
+    {
+        t.try_super_fold_with(self)
+    }
+
+    fn try_fold_ty(&mut self, t: I::Ty) -> Result<I::Ty, Self::Error>
+    where
+        I::Ty: TypeSuperFoldable<I>,
+    {
+        t.try_super_fold_with(self)
+    }
+
+    fn try_fold_region(&mut self, r: I::Region) -> Result<I::Region, Self::Error>
+    where
+        I::Region: TypeSuperFoldable<I>,
+    {
+        r.try_super_fold_with(self)
+    }
+
+    fn try_fold_const(&mut self, c: I::Const) -> Result<I::Const, Self::Error>
+    where
+        I::Const: TypeSuperFoldable<I>,
+    {
+        c.try_super_fold_with(self)
+    }
+
+    fn try_fold_predicate(&mut self, p: I::Predicate) -> Result<I::Predicate, Self::Error>
+    where
+        I::Predicate: TypeSuperFoldable<I>,
+    {
+        p.try_super_fold_with(self)
+    }
+}
+
+// This blanket implementation of the fallible trait for infallible folders
+// delegates to infallible methods to ensure coherence.
+impl<I: Interner, F> FallibleTypeFolder<I> for F
+where
+    F: TypeFolder<I>,
+{
+    type Error = !;
+
+    fn interner(&self) -> I {
+        TypeFolder::interner(self)
+    }
+
+    fn try_fold_binder<T>(&mut self, t: I::Binder<T>) -> Result<I::Binder<T>, !>
+    where
+        T: TypeFoldable<I>,
+        I::Binder<T>: TypeSuperFoldable<I>,
+    {
+        Ok(self.fold_binder(t))
+    }
+
+    fn try_fold_ty(&mut self, t: I::Ty) -> Result<I::Ty, !>
+    where
+        I::Ty: TypeSuperFoldable<I>,
+    {
+        Ok(self.fold_ty(t))
+    }
+
+    fn try_fold_region(&mut self, r: I::Region) -> Result<I::Region, !>
+    where
+        I::Region: TypeSuperFoldable<I>,
+    {
+        Ok(self.fold_region(r))
+    }
+
+    fn try_fold_const(&mut self, c: I::Const) -> Result<I::Const, !>
+    where
+        I::Const: TypeSuperFoldable<I>,
+    {
+        Ok(self.fold_const(c))
+    }
+
+    fn try_fold_predicate(&mut self, p: I::Predicate) -> Result<I::Predicate, !>
+    where
+        I::Predicate: TypeSuperFoldable<I>,
+    {
+        Ok(self.fold_predicate(p))
+    }
+}
diff --git a/compiler/rustc_type_ir/src/lib.rs b/compiler/rustc_type_ir/src/lib.rs
index 44004cb0b..5a991e03d 100644
--- a/compiler/rustc_type_ir/src/lib.rs
+++ b/compiler/rustc_type_ir/src/lib.rs
@@ -1,6 +1,9 @@
+#![feature(associated_type_defaults)]
 #![feature(fmt_helpers_for_derive)]
 #![feature(min_specialization)]
+#![feature(never_type)]
 #![feature(rustc_attrs)]
+#![feature(unwrap_infallible)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 
@@ -18,8 +21,14 @@ use std::hash::Hash;
 use std::mem::discriminant;
 
 pub mod codec;
+pub mod fold;
 pub mod sty;
 pub mod ty_info;
+pub mod visit;
+
+#[macro_use]
+mod macros;
+mod structural_impls;
 
 pub use codec::*;
 pub use sty::*;
@@ -28,68 +37,69 @@ pub use ty_info::*;
 /// Needed so we can use #[derive(HashStable_Generic)]
 pub trait HashStableContext {}
 
-pub trait Interner {
-    type AdtDef: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type SubstsRef: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type DefId: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type Ty: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type Const: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type Region: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type TypeAndMut: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type Mutability: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type Movability: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type PolyFnSig: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type ListBinderExistentialPredicate: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type BinderListTy: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type ListTy: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type AliasTy: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type ParamTy: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type BoundTy: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type PlaceholderType: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type InferTy: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type ErrorGuaranteed: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
+pub trait Interner: Sized {
+    type AdtDef: Clone + Debug + Hash + Ord;
+    type SubstsRef: Clone + Debug + Hash + Ord;
+    type DefId: Clone + Debug + Hash + Ord;
+    type Binder<T>;
+    type Ty: Clone + Debug + Hash + Ord;
+    type Const: Clone + Debug + Hash + Ord;
+    type Region: Clone + Debug + Hash + Ord;
+    type Predicate;
+    type TypeAndMut: Clone + Debug + Hash + Ord;
+    type Mutability: Clone + Debug + Hash + Ord;
+    type Movability: Clone + Debug + Hash + Ord;
+    type PolyFnSig: Clone + Debug + Hash + Ord;
+    type ListBinderExistentialPredicate: Clone + Debug + Hash + Ord;
+    type BinderListTy: Clone + Debug + Hash + Ord;
+    type ListTy: Clone + Debug + Hash + Ord;
+    type AliasTy: Clone + Debug + Hash + Ord;
+    type ParamTy: Clone + Debug + Hash + Ord;
+    type BoundTy: Clone + Debug + Hash + Ord;
+    type PlaceholderType: Clone + Debug + Hash + Ord;
+    type InferTy: Clone + Debug + Hash + Ord;
+    type ErrorGuaranteed: Clone + Debug + Hash + Ord;
     type PredicateKind: Clone + Debug + Hash + PartialEq + Eq;
-    type AllocId: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
+    type AllocId: Clone + Debug + Hash + Ord;
 
-    type EarlyBoundRegion: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type BoundRegion: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type FreeRegion: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type RegionVid: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
-    type PlaceholderRegion: Clone + Debug + Hash + PartialEq + Eq + PartialOrd + Ord;
+    type EarlyBoundRegion: Clone + Debug + Hash + Ord;
+    type BoundRegion: Clone + Debug + Hash + Ord;
+    type FreeRegion: Clone + Debug + Hash + Ord;
+    type RegionVid: Clone + Debug + Hash + Ord;
+    type PlaceholderRegion: Clone + Debug + Hash + Ord;
 }
 
-pub trait InternAs<T: ?Sized, R> {
+/// Imagine you have a function `F: FnOnce(&[T]) -> R`, plus an iterator `iter`
+/// that produces `T` items. You could combine them with
+/// `f(&iter.collect::<Vec<_>>())`, but this requires allocating memory for the
+/// `Vec`.
+///
+/// This trait allows for faster implementations, intended for cases where the
+/// number of items produced by the iterator is small. There is a blanket impl
+/// for `T` items, but there is also a fallible impl for `Result<T, E>` items.
+pub trait CollectAndApply<T, R>: Sized {
     type Output;
-    fn intern_with<F>(self, f: F) -> Self::Output
+
+    /// Produce a result of type `Self::Output` from `iter`. The result will
+    /// typically be produced by applying `f` on the elements produced by
+    /// `iter`, though this may not happen in some impls, e.g. if an error
+    /// occured during iteration.
+    fn collect_and_apply<I, F>(iter: I, f: F) -> Self::Output
     where
+        I: Iterator<Item = Self>,
         F: FnOnce(&[T]) -> R;
 }
 
-impl<I, T, R, E> InternAs<T, R> for I
-where
-    E: InternIteratorElement<T, R>,
-    I: Iterator<Item = E>,
-{
-    type Output = E::Output;
-    fn intern_with<F>(self, f: F) -> Self::Output
+/// The blanket impl that always collects all elements and applies `f`.
+impl<T, R> CollectAndApply<T, R> for T {
+    type Output = R;
+
+    /// Equivalent to `f(&iter.collect::<Vec<_>>())`.
+    fn collect_and_apply<I, F>(mut iter: I, f: F) -> R
     where
+        I: Iterator<Item = T>,
         F: FnOnce(&[T]) -> R,
     {
-        E::intern_with(self, f)
-    }
-}
-
-pub trait InternIteratorElement<T, R>: Sized {
-    type Output;
-    fn intern_with<I: Iterator<Item = Self>, F: FnOnce(&[T]) -> R>(iter: I, f: F) -> Self::Output;
-}
-
-impl<T, R> InternIteratorElement<T, R> for T {
-    type Output = R;
-    fn intern_with<I: Iterator<Item = Self>, F: FnOnce(&[T]) -> R>(
-        mut iter: I,
-        f: F,
-    ) -> Self::Output {
         // This code is hot enough that it's worth specializing for the most
         // common length lists, to avoid the overhead of `SmallVec` creation.
         // Lengths 0, 1, and 2 typically account for ~95% of cases. If
@@ -116,23 +126,17 @@ impl<T, R> InternIteratorElement<T, R> for T {
     }
 }
 
-impl<'a, T, R> InternIteratorElement<T, R> for &'a T
-where
-    T: Clone + 'a,
-{
-    type Output = R;
-    fn intern_with<I: Iterator<Item = Self>, F: FnOnce(&[T]) -> R>(iter: I, f: F) -> Self::Output {
-        // This code isn't hot.
-        f(&iter.cloned().collect::<SmallVec<[_; 8]>>())
-    }
-}
-
-impl<T, R, E> InternIteratorElement<T, R> for Result<T, E> {
+/// A fallible impl that will fail, without calling `f`, if there are any
+/// errors during collection.
+impl<T, R, E> CollectAndApply<T, R> for Result<T, E> {
     type Output = Result<R, E>;
-    fn intern_with<I: Iterator<Item = Self>, F: FnOnce(&[T]) -> R>(
-        mut iter: I,
-        f: F,
-    ) -> Self::Output {
+
+    /// Equivalent to `Ok(f(&iter.collect::<Result<Vec<_>>>()?))`.
+    fn collect_and_apply<I, F>(mut iter: I, f: F) -> Result<R, E>
+    where
+        I: Iterator<Item = Result<T, E>>,
+        F: FnOnce(&[T]) -> R,
+    {
         // This code is hot enough that it's worth specializing for the most
         // common length lists, to avoid the overhead of `SmallVec` creation.
         // Lengths 0, 1, and 2 typically account for ~95% of cases. If
@@ -220,7 +224,8 @@ bitflags! {
                                           // which is different from how types/const are freshened.
                                           | TypeFlags::HAS_TY_FRESH.bits
                                           | TypeFlags::HAS_CT_FRESH.bits
-                                          | TypeFlags::HAS_FREE_LOCAL_REGIONS.bits;
+                                          | TypeFlags::HAS_FREE_LOCAL_REGIONS.bits
+                                          | TypeFlags::HAS_RE_ERASED.bits;
 
         /// Does this have `Projection`?
         const HAS_TY_PROJECTION           = 1 << 10;
@@ -265,6 +270,9 @@ bitflags! {
 
         /// Does this value have `InferConst::Fresh`?
         const HAS_CT_FRESH                = 1 << 21;
+
+        /// Does this have `Generator` or `GeneratorWitness`?
+        const HAS_TY_GENERATOR            = 1 << 22;
     }
 }
 
diff --git a/compiler/rustc_type_ir/src/macros.rs b/compiler/rustc_type_ir/src/macros.rs
new file mode 100644
index 000000000..6c1810397
--- /dev/null
+++ b/compiler/rustc_type_ir/src/macros.rs
@@ -0,0 +1,176 @@
+/// Used for types that are `Copy` and which **do not care arena
+/// allocated data** (i.e., don't need to be folded).
+macro_rules! TrivialTypeTraversalImpls {
+    ($($ty:ty,)+) => {
+        $(
+            impl<I: $crate::Interner> $crate::fold::TypeFoldable<I> for $ty {
+                fn try_fold_with<F: $crate::fold::FallibleTypeFolder<I>>(
+                    self,
+                    _: &mut F,
+                ) -> ::std::result::Result<Self, F::Error> {
+                    Ok(self)
+                }
+
+                #[inline]
+                fn fold_with<F: $crate::fold::TypeFolder<I>>(
+                    self,
+                    _: &mut F,
+                ) -> Self {
+                    self
+                }
+            }
+
+            impl<I: $crate::Interner> $crate::visit::TypeVisitable<I> for $ty {
+                #[inline]
+                fn visit_with<F: $crate::visit::TypeVisitor<I>>(
+                    &self,
+                    _: &mut F)
+                    -> ::std::ops::ControlFlow<F::BreakTy>
+                {
+                    ::std::ops::ControlFlow::Continue(())
+                }
+            }
+        )+
+    };
+}
+
+macro_rules! EnumTypeTraversalImpl {
+    (impl<$($p:tt),*> TypeFoldable<$tcx:tt> for $s:path {
+        $($variants:tt)*
+    } $(where $($wc:tt)*)*) => {
+        impl<$($p),*> $crate::fold::TypeFoldable<$tcx> for $s
+            $(where $($wc)*)*
+        {
+            fn try_fold_with<V: $crate::fold::FallibleTypeFolder<$tcx>>(
+                self,
+                folder: &mut V,
+            ) -> ::std::result::Result<Self, V::Error> {
+                EnumTypeTraversalImpl!(@FoldVariants(self, folder) input($($variants)*) output())
+            }
+        }
+    };
+
+    (impl<$($p:tt),*> TypeVisitable<$tcx:tt> for $s:path {
+        $($variants:tt)*
+    } $(where $($wc:tt)*)*) => {
+        impl<$($p),*> $crate::visit::TypeVisitable<$tcx> for $s
+            $(where $($wc)*)*
+        {
+            fn visit_with<V: $crate::visit::TypeVisitor<$tcx>>(
+                &self,
+                visitor: &mut V,
+            ) -> ::std::ops::ControlFlow<V::BreakTy> {
+                EnumTypeTraversalImpl!(@VisitVariants(self, visitor) input($($variants)*) output())
+            }
+        }
+    };
+
+    (@FoldVariants($this:expr, $folder:expr) input() output($($output:tt)*)) => {
+        Ok(match $this {
+            $($output)*
+        })
+    };
+
+    (@FoldVariants($this:expr, $folder:expr)
+     input( ($variant:path) ( $($variant_arg:ident),* ) , $($input:tt)*)
+     output( $($output:tt)*) ) => {
+        EnumTypeTraversalImpl!(
+            @FoldVariants($this, $folder)
+                input($($input)*)
+                output(
+                    $variant ( $($variant_arg),* ) => {
+                        $variant (
+                            $($crate::fold::TypeFoldable::try_fold_with($variant_arg, $folder)?),*
+                        )
+                    }
+                    $($output)*
+                )
+        )
+    };
+
+    (@FoldVariants($this:expr, $folder:expr)
+     input( ($variant:path) { $($variant_arg:ident),* $(,)? } , $($input:tt)*)
+     output( $($output:tt)*) ) => {
+        EnumTypeTraversalImpl!(
+            @FoldVariants($this, $folder)
+                input($($input)*)
+                output(
+                    $variant { $($variant_arg),* } => {
+                        $variant {
+                            $($variant_arg: $crate::fold::TypeFoldable::fold_with(
+                                $variant_arg, $folder
+                            )?),* }
+                    }
+                    $($output)*
+                )
+        )
+    };
+
+    (@FoldVariants($this:expr, $folder:expr)
+     input( ($variant:path), $($input:tt)*)
+     output( $($output:tt)*) ) => {
+        EnumTypeTraversalImpl!(
+            @FoldVariants($this, $folder)
+                input($($input)*)
+                output(
+                    $variant => { $variant }
+                    $($output)*
+                )
+        )
+    };
+
+    (@VisitVariants($this:expr, $visitor:expr) input() output($($output:tt)*)) => {
+        match $this {
+            $($output)*
+        }
+    };
+
+    (@VisitVariants($this:expr, $visitor:expr)
+     input( ($variant:path) ( $($variant_arg:ident),* ) , $($input:tt)*)
+     output( $($output:tt)*) ) => {
+        EnumTypeTraversalImpl!(
+            @VisitVariants($this, $visitor)
+                input($($input)*)
+                output(
+                    $variant ( $($variant_arg),* ) => {
+                        $($crate::visit::TypeVisitable::visit_with(
+                            $variant_arg, $visitor
+                        )?;)*
+                        ::std::ops::ControlFlow::Continue(())
+                    }
+                    $($output)*
+                )
+        )
+    };
+
+    (@VisitVariants($this:expr, $visitor:expr)
+     input( ($variant:path) { $($variant_arg:ident),* $(,)? } , $($input:tt)*)
+     output( $($output:tt)*) ) => {
+        EnumTypeTraversalImpl!(
+            @VisitVariants($this, $visitor)
+                input($($input)*)
+                output(
+                    $variant { $($variant_arg),* } => {
+                        $($crate::visit::TypeVisitable::visit_with(
+                            $variant_arg, $visitor
+                        )?;)*
+                        ::std::ops::ControlFlow::Continue(())
+                    }
+                    $($output)*
+                )
+        )
+    };
+
+    (@VisitVariants($this:expr, $visitor:expr)
+     input( ($variant:path), $($input:tt)*)
+     output( $($output:tt)*) ) => {
+        EnumTypeTraversalImpl!(
+            @VisitVariants($this, $visitor)
+                input($($input)*)
+                output(
+                    $variant => { ::std::ops::ControlFlow::Continue(()) }
+                    $($output)*
+                )
+        )
+    };
+}
diff --git a/compiler/rustc_type_ir/src/structural_impls.rs b/compiler/rustc_type_ir/src/structural_impls.rs
new file mode 100644
index 000000000..3ebe24104
--- /dev/null
+++ b/compiler/rustc_type_ir/src/structural_impls.rs
@@ -0,0 +1,175 @@
+//! This module contains implementations of the `TypeFoldable` and `TypeVisitable`
+//! traits for various types in the Rust compiler. Most are written by hand, though
+//! we've recently added some macros and proc-macros to help with the tedium.
+
+use crate::fold::{FallibleTypeFolder, TypeFoldable};
+use crate::visit::{TypeVisitable, TypeVisitor};
+use crate::Interner;
+use rustc_data_structures::functor::IdFunctor;
+use rustc_index::vec::{Idx, IndexVec};
+
+use std::ops::ControlFlow;
+use std::rc::Rc;
+use std::sync::Arc;
+
+///////////////////////////////////////////////////////////////////////////
+// Atomic structs
+//
+// For things that don't carry any arena-allocated data (and are
+// copy...), just add them to this list.
+
+TrivialTypeTraversalImpls! {
+    (),
+    bool,
+    usize,
+    u16,
+    u32,
+    u64,
+    String,
+    crate::DebruijnIndex,
+}
+
+///////////////////////////////////////////////////////////////////////////
+// Traversal implementations.
+
+impl<I: Interner, T: TypeFoldable<I>, U: TypeFoldable<I>> TypeFoldable<I> for (T, U) {
+    fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<(T, U), F::Error> {
+        Ok((self.0.try_fold_with(folder)?, self.1.try_fold_with(folder)?))
+    }
+}
+
+impl<I: Interner, T: TypeVisitable<I>, U: TypeVisitable<I>> TypeVisitable<I> for (T, U) {
+    fn visit_with<V: TypeVisitor<I>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+        self.0.visit_with(visitor)?;
+        self.1.visit_with(visitor)
+    }
+}
+
+impl<I: Interner, A: TypeFoldable<I>, B: TypeFoldable<I>, C: TypeFoldable<I>> TypeFoldable<I>
+    for (A, B, C)
+{
+    fn try_fold_with<F: FallibleTypeFolder<I>>(
+        self,
+        folder: &mut F,
+    ) -> Result<(A, B, C), F::Error> {
+        Ok((
+            self.0.try_fold_with(folder)?,
+            self.1.try_fold_with(folder)?,
+            self.2.try_fold_with(folder)?,
+        ))
+    }
+}
+
+impl<I: Interner, A: TypeVisitable<I>, B: TypeVisitable<I>, C: TypeVisitable<I>> TypeVisitable<I>
+    for (A, B, C)
+{
+    fn visit_with<V: TypeVisitor<I>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+        self.0.visit_with(visitor)?;
+        self.1.visit_with(visitor)?;
+        self.2.visit_with(visitor)
+    }
+}
+
+EnumTypeTraversalImpl! {
+    impl<I, T> TypeFoldable<I> for Option<T> {
+        (Some)(a),
+        (None),
+    } where I: Interner, T: TypeFoldable<I>
+}
+EnumTypeTraversalImpl! {
+    impl<I, T> TypeVisitable<I> for Option<T> {
+        (Some)(a),
+        (None),
+    } where I: Interner, T: TypeVisitable<I>
+}
+
+EnumTypeTraversalImpl! {
+    impl<I, T, E> TypeFoldable<I> for Result<T, E> {
+        (Ok)(a),
+        (Err)(a),
+    } where I: Interner, T: TypeFoldable<I>, E: TypeFoldable<I>,
+}
+EnumTypeTraversalImpl! {
+    impl<I, T, E> TypeVisitable<I> for Result<T, E> {
+        (Ok)(a),
+        (Err)(a),
+    } where I: Interner, T: TypeVisitable<I>, E: TypeVisitable<I>,
+}
+
+impl<I: Interner, T: TypeFoldable<I>> TypeFoldable<I> for Rc<T> {
+    fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error> {
+        self.try_map_id(|value| value.try_fold_with(folder))
+    }
+}
+
+impl<I: Interner, T: TypeVisitable<I>> TypeVisitable<I> for Rc<T> {
+    fn visit_with<V: TypeVisitor<I>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+        (**self).visit_with(visitor)
+    }
+}
+
+impl<I: Interner, T: TypeFoldable<I>> TypeFoldable<I> for Arc<T> {
+    fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error> {
+        self.try_map_id(|value| value.try_fold_with(folder))
+    }
+}
+
+impl<I: Interner, T: TypeVisitable<I>> TypeVisitable<I> for Arc<T> {
+    fn visit_with<V: TypeVisitor<I>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+        (**self).visit_with(visitor)
+    }
+}
+
+impl<I: Interner, T: TypeFoldable<I>> TypeFoldable<I> for Box<T> {
+    fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error> {
+        self.try_map_id(|value| value.try_fold_with(folder))
+    }
+}
+
+impl<I: Interner, T: TypeVisitable<I>> TypeVisitable<I> for Box<T> {
+    fn visit_with<V: TypeVisitor<I>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+        (**self).visit_with(visitor)
+    }
+}
+
+impl<I: Interner, T: TypeFoldable<I>> TypeFoldable<I> for Vec<T> {
+    fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error> {
+        self.try_map_id(|t| t.try_fold_with(folder))
+    }
+}
+
+impl<I: Interner, T: TypeVisitable<I>> TypeVisitable<I> for Vec<T> {
+    fn visit_with<V: TypeVisitor<I>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+        self.iter().try_for_each(|t| t.visit_with(visitor))
+    }
+}
+
+impl<I: Interner, T: TypeVisitable<I>> TypeVisitable<I> for &[T] {
+    fn visit_with<V: TypeVisitor<I>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+        self.iter().try_for_each(|t| t.visit_with(visitor))
+    }
+}
+
+impl<I: Interner, T: TypeFoldable<I>> TypeFoldable<I> for Box<[T]> {
+    fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error> {
+        self.try_map_id(|t| t.try_fold_with(folder))
+    }
+}
+
+impl<I: Interner, T: TypeVisitable<I>> TypeVisitable<I> for Box<[T]> {
+    fn visit_with<V: TypeVisitor<I>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+        self.iter().try_for_each(|t| t.visit_with(visitor))
+    }
+}
+
+impl<I: Interner, T: TypeFoldable<I>, Ix: Idx> TypeFoldable<I> for IndexVec<Ix, T> {
+    fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error> {
+        self.try_map_id(|x| x.try_fold_with(folder))
+    }
+}
+
+impl<I: Interner, T: TypeVisitable<I>, Ix: Idx> TypeVisitable<I> for IndexVec<Ix, T> {
+    fn visit_with<V: TypeVisitor<I>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+        self.iter().try_for_each(|t| t.visit_with(visitor))
+    }
+}
diff --git a/compiler/rustc_type_ir/src/sty.rs b/compiler/rustc_type_ir/src/sty.rs
index 5f29588ae..ebe2b76ae 100644
--- a/compiler/rustc_type_ir/src/sty.rs
+++ b/compiler/rustc_type_ir/src/sty.rs
@@ -26,11 +26,9 @@ pub enum DynKind {
     Dyn,
     /// A sized `dyn* Trait` object
     ///
-    /// These objects are represented as a `(data, vtable)` pair where `data` is a ptr-sized value
-    /// (often a pointer to the real object, but not necessarily) and `vtable` is a pointer to
-    /// the vtable for `dyn* Trait`. The representation is essentially the same as `&dyn Trait`
-    /// or similar, but the drop function included in the vtable is responsible for freeing the
-    /// underlying storage if needed. This allows a `dyn*` object to be treated agnostically with
+    /// These objects are represented as a `(data, vtable)` pair where `data` is a value of some
+    /// ptr-sized and ptr-aligned dynamically determined type `T` and `vtable` is a pointer to the
+    /// vtable of `impl T for Trait`. This allows a `dyn*` object to be treated agnostically with
     /// respect to whether it points to a `Box<T>`, `Rc<T>`, etc.
     DynStar,
 }
@@ -160,6 +158,32 @@ pub enum TyKind<I: Interner> {
     /// ```
     GeneratorWitness(I::BinderListTy),
 
+    /// A type representing the types stored inside a generator.
+    /// This should only appear as part of the `GeneratorSubsts`.
+    ///
+    /// Unlike upvars, the witness can reference lifetimes from
+    /// inside of the generator itself. To deal with them in
+    /// the type of the generator, we convert them to higher ranked
+    /// lifetimes bound by the witness itself.
+    ///
+    /// This variant is only using when `drop_tracking_mir` is set.
+    /// This contains the `DefId` and the `SubstRef` of the generator.
+    /// The actual witness types are computed on MIR by the `mir_generator_witnesses` query.
+    ///
+    /// Looking at the following example, the witness for this generator
+    /// may end up as something like `for<'a> [Vec<i32>, &'a Vec<i32>]`:
+    ///
+    /// ```ignore UNSOLVED (ask @compiler-errors, should this error? can we just swap the yields?)
+    /// #![feature(generators)]
+    /// |a| {
+    ///     let x = &vec![3];
+    ///     yield a;
+    ///     yield x[0];
+    /// }
+    /// # ;
+    /// ```
+    GeneratorWitnessMIR(I::DefId, I::SubstsRef),
+
     /// The never type `!`.
     Never,
 
@@ -241,6 +265,7 @@ const fn tykind_discriminant<I: Interner>(value: &TyKind<I>) -> usize {
         Placeholder(_) => 23,
         Infer(_) => 24,
         Error(_) => 25,
+        GeneratorWitnessMIR(_, _) => 26,
     }
 }
 
@@ -266,6 +291,7 @@ impl<I: Interner> Clone for TyKind<I> {
             Closure(d, s) => Closure(d.clone(), s.clone()),
             Generator(d, s, m) => Generator(d.clone(), s.clone(), m.clone()),
             GeneratorWitness(g) => GeneratorWitness(g.clone()),
+            GeneratorWitnessMIR(d, s) => GeneratorWitnessMIR(d.clone(), s.clone()),
             Never => Never,
             Tuple(t) => Tuple(t.clone()),
             Alias(k, p) => Alias(*k, p.clone()),
@@ -282,43 +308,51 @@ impl<I: Interner> Clone for TyKind<I> {
 impl<I: Interner> PartialEq for TyKind<I> {
     #[inline]
     fn eq(&self, other: &TyKind<I>) -> bool {
-        tykind_discriminant(self) == tykind_discriminant(other)
-            && match (self, other) {
-                (Int(a_i), Int(b_i)) => a_i == b_i,
-                (Uint(a_u), Uint(b_u)) => a_u == b_u,
-                (Float(a_f), Float(b_f)) => a_f == b_f,
-                (Adt(a_d, a_s), Adt(b_d, b_s)) => a_d == b_d && a_s == b_s,
-                (Foreign(a_d), Foreign(b_d)) => a_d == b_d,
-                (Array(a_t, a_c), Array(b_t, b_c)) => a_t == b_t && a_c == b_c,
-                (Slice(a_t), Slice(b_t)) => a_t == b_t,
-                (RawPtr(a_t), RawPtr(b_t)) => a_t == b_t,
-                (Ref(a_r, a_t, a_m), Ref(b_r, b_t, b_m)) => a_r == b_r && a_t == b_t && a_m == b_m,
-                (FnDef(a_d, a_s), FnDef(b_d, b_s)) => a_d == b_d && a_s == b_s,
-                (FnPtr(a_s), FnPtr(b_s)) => a_s == b_s,
-                (Dynamic(a_p, a_r, a_repr), Dynamic(b_p, b_r, b_repr)) => {
-                    a_p == b_p && a_r == b_r && a_repr == b_repr
-                }
-                (Closure(a_d, a_s), Closure(b_d, b_s)) => a_d == b_d && a_s == b_s,
-                (Generator(a_d, a_s, a_m), Generator(b_d, b_s, b_m)) => {
-                    a_d == b_d && a_s == b_s && a_m == b_m
-                }
-                (GeneratorWitness(a_g), GeneratorWitness(b_g)) => a_g == b_g,
-                (Tuple(a_t), Tuple(b_t)) => a_t == b_t,
-                (Alias(a_i, a_p), Alias(b_i, b_p)) => a_i == b_i && a_p == b_p,
-                (Param(a_p), Param(b_p)) => a_p == b_p,
-                (Bound(a_d, a_b), Bound(b_d, b_b)) => a_d == b_d && a_b == b_b,
-                (Placeholder(a_p), Placeholder(b_p)) => a_p == b_p,
-                (Infer(a_t), Infer(b_t)) => a_t == b_t,
-                (Error(a_e), Error(b_e)) => a_e == b_e,
-                (Bool, Bool) | (Char, Char) | (Str, Str) | (Never, Never) => true,
-                _ => {
-                    debug_assert!(
-                        false,
-                        "This branch must be unreachable, maybe the match is missing an arm? self = self = {self:?}, other = {other:?}"
-                    );
-                    true
-                }
+        // You might expect this `match` to be preceded with this:
+        //
+        //   tykind_discriminant(self) == tykind_discriminant(other) &&
+        //
+        // but the data patterns in practice are such that a comparison
+        // succeeds 99%+ of the time, and it's faster to omit it.
+        match (self, other) {
+            (Int(a_i), Int(b_i)) => a_i == b_i,
+            (Uint(a_u), Uint(b_u)) => a_u == b_u,
+            (Float(a_f), Float(b_f)) => a_f == b_f,
+            (Adt(a_d, a_s), Adt(b_d, b_s)) => a_d == b_d && a_s == b_s,
+            (Foreign(a_d), Foreign(b_d)) => a_d == b_d,
+            (Array(a_t, a_c), Array(b_t, b_c)) => a_t == b_t && a_c == b_c,
+            (Slice(a_t), Slice(b_t)) => a_t == b_t,
+            (RawPtr(a_t), RawPtr(b_t)) => a_t == b_t,
+            (Ref(a_r, a_t, a_m), Ref(b_r, b_t, b_m)) => a_r == b_r && a_t == b_t && a_m == b_m,
+            (FnDef(a_d, a_s), FnDef(b_d, b_s)) => a_d == b_d && a_s == b_s,
+            (FnPtr(a_s), FnPtr(b_s)) => a_s == b_s,
+            (Dynamic(a_p, a_r, a_repr), Dynamic(b_p, b_r, b_repr)) => {
+                a_p == b_p && a_r == b_r && a_repr == b_repr
+            }
+            (Closure(a_d, a_s), Closure(b_d, b_s)) => a_d == b_d && a_s == b_s,
+            (Generator(a_d, a_s, a_m), Generator(b_d, b_s, b_m)) => {
+                a_d == b_d && a_s == b_s && a_m == b_m
+            }
+            (GeneratorWitness(a_g), GeneratorWitness(b_g)) => a_g == b_g,
+            (GeneratorWitnessMIR(a_d, a_s), GeneratorWitnessMIR(b_d, b_s)) => {
+                a_d == b_d && a_s == b_s
             }
+            (Tuple(a_t), Tuple(b_t)) => a_t == b_t,
+            (Alias(a_i, a_p), Alias(b_i, b_p)) => a_i == b_i && a_p == b_p,
+            (Param(a_p), Param(b_p)) => a_p == b_p,
+            (Bound(a_d, a_b), Bound(b_d, b_b)) => a_d == b_d && a_b == b_b,
+            (Placeholder(a_p), Placeholder(b_p)) => a_p == b_p,
+            (Infer(a_t), Infer(b_t)) => a_t == b_t,
+            (Error(a_e), Error(b_e)) => a_e == b_e,
+            (Bool, Bool) | (Char, Char) | (Str, Str) | (Never, Never) => true,
+            _ => {
+                debug_assert!(
+                    tykind_discriminant(self) != tykind_discriminant(other),
+                    "This branch must be unreachable, maybe the match is missing an arm? self = self = {self:?}, other = {other:?}"
+                );
+                false
+            }
+        }
     }
 }
 
@@ -360,6 +394,13 @@ impl<I: Interner> Ord for TyKind<I> {
                     a_d.cmp(b_d).then_with(|| a_s.cmp(b_s).then_with(|| a_m.cmp(b_m)))
                 }
                 (GeneratorWitness(a_g), GeneratorWitness(b_g)) => a_g.cmp(b_g),
+                (
+                    GeneratorWitnessMIR(a_d, a_s),
+                    GeneratorWitnessMIR(b_d, b_s),
+                ) => match Ord::cmp(a_d, b_d) {
+                    Ordering::Equal => Ord::cmp(a_s, b_s),
+                    cmp => cmp,
+                },
                 (Tuple(a_t), Tuple(b_t)) => a_t.cmp(b_t),
                 (Alias(a_i, a_p), Alias(b_i, b_p)) => a_i.cmp(b_i).then_with(|| a_p.cmp(b_p)),
                 (Param(a_p), Param(b_p)) => a_p.cmp(b_p),
@@ -369,7 +410,7 @@ impl<I: Interner> Ord for TyKind<I> {
                 (Error(a_e), Error(b_e)) => a_e.cmp(b_e),
                 (Bool, Bool) | (Char, Char) | (Str, Str) | (Never, Never) => Ordering::Equal,
                 _ => {
-                    debug_assert!(false, "This branch must be unreachable, maybe the match is missing an arm? self = self = {self:?}, other = {other:?}");
+                    debug_assert!(false, "This branch must be unreachable, maybe the match is missing an arm? self = {self:?}, other = {other:?}");
                     Ordering::Equal
                 }
             }
@@ -421,6 +462,10 @@ impl<I: Interner> hash::Hash for TyKind<I> {
                 m.hash(state)
             }
             GeneratorWitness(g) => g.hash(state),
+            GeneratorWitnessMIR(d, s) => {
+                d.hash(state);
+                s.hash(state);
+            }
             Tuple(t) => t.hash(state),
             Alias(i, p) => {
                 i.hash(state);
@@ -461,6 +506,7 @@ impl<I: Interner> fmt::Debug for TyKind<I> {
             Closure(d, s) => f.debug_tuple_field2_finish("Closure", d, s),
             Generator(d, s, m) => f.debug_tuple_field3_finish("Generator", d, s, m),
             GeneratorWitness(g) => f.debug_tuple_field1_finish("GeneratorWitness", g),
+            GeneratorWitnessMIR(d, s) => f.debug_tuple_field2_finish("GeneratorWitnessMIR", d, s),
             Never => f.write_str("Never"),
             Tuple(t) => f.debug_tuple_field1_finish("Tuple", t),
             Alias(i, a) => f.debug_tuple_field2_finish("Alias", i, a),
@@ -559,6 +605,10 @@ where
             GeneratorWitness(b) => e.emit_enum_variant(disc, |e| {
                 b.encode(e);
             }),
+            GeneratorWitnessMIR(def_id, substs) => e.emit_enum_variant(disc, |e| {
+                def_id.encode(e);
+                substs.encode(e);
+            }),
             Never => e.emit_enum_variant(disc, |_| {}),
             Tuple(substs) => e.emit_enum_variant(disc, |e| {
                 substs.encode(e);
@@ -641,6 +691,7 @@ where
             23 => Placeholder(Decodable::decode(d)),
             24 => Infer(Decodable::decode(d)),
             25 => Error(Decodable::decode(d)),
+            26 => GeneratorWitnessMIR(Decodable::decode(d), Decodable::decode(d)),
             _ => panic!(
                 "{}",
                 format!(
@@ -742,6 +793,10 @@ where
             GeneratorWitness(b) => {
                 b.hash_stable(__hcx, __hasher);
             }
+            GeneratorWitnessMIR(def_id, substs) => {
+                def_id.hash_stable(__hcx, __hasher);
+                substs.hash_stable(__hcx, __hasher);
+            }
             Never => {}
             Tuple(substs) => {
                 substs.hash_stable(__hcx, __hasher);
@@ -903,6 +958,9 @@ pub enum RegionKind<I: Interner> {
 
     /// Erased region, used by trait selection, in MIR and during codegen.
     ReErased,
+
+    /// A region that resulted from some other error. Used exclusively for diagnostics.
+    ReError(I::ErrorGuaranteed),
 }
 
 // This is manually implemented for `RegionKind` because `std::mem::discriminant`
@@ -917,6 +975,7 @@ const fn regionkind_discriminant<I: Interner>(value: &RegionKind<I>) -> usize {
         ReVar(_) => 4,
         RePlaceholder(_) => 5,
         ReErased => 6,
+        ReError(_) => 7,
     }
 }
 
@@ -928,6 +987,7 @@ where
     I::FreeRegion: Copy,
     I::RegionVid: Copy,
     I::PlaceholderRegion: Copy,
+    I::ErrorGuaranteed: Copy,
 {
 }
 
@@ -942,6 +1002,7 @@ impl<I: Interner> Clone for RegionKind<I> {
             ReVar(r) => ReVar(r.clone()),
             RePlaceholder(r) => RePlaceholder(r.clone()),
             ReErased => ReErased,
+            ReError(r) => ReError(r.clone()),
         }
     }
 }
@@ -959,10 +1020,11 @@ impl<I: Interner> PartialEq for RegionKind<I> {
                 (ReVar(a_r), ReVar(b_r)) => a_r == b_r,
                 (RePlaceholder(a_r), RePlaceholder(b_r)) => a_r == b_r,
                 (ReErased, ReErased) => true,
+                (ReError(_), ReError(_)) => true,
                 _ => {
                     debug_assert!(
                         false,
-                        "This branch must be unreachable, maybe the match is missing an arm? self = self = {self:?}, other = {other:?}"
+                        "This branch must be unreachable, maybe the match is missing an arm? self = {self:?}, other = {other:?}"
                     );
                     true
                 }
@@ -1020,6 +1082,7 @@ impl<I: Interner> hash::Hash for RegionKind<I> {
             ReVar(r) => r.hash(state),
             RePlaceholder(r) => r.hash(state),
             ReErased => (),
+            ReError(_) => (),
         }
     }
 }
@@ -1043,6 +1106,8 @@ impl<I: Interner> fmt::Debug for RegionKind<I> {
             RePlaceholder(placeholder) => write!(f, "RePlaceholder({placeholder:?})"),
 
             ReErased => f.write_str("ReErased"),
+
+            ReError(_) => f.write_str("ReError"),
         }
     }
 }
@@ -1077,6 +1142,7 @@ where
                 a.encode(e);
             }),
             ReErased => e.emit_enum_variant(disc, |_| {}),
+            ReError(_) => e.emit_enum_variant(disc, |_| {}),
         }
     }
 }
@@ -1089,6 +1155,7 @@ where
     I::FreeRegion: Decodable<D>,
     I::RegionVid: Decodable<D>,
     I::PlaceholderRegion: Decodable<D>,
+    I::ErrorGuaranteed: Decodable<D>,
 {
     fn decode(d: &mut D) -> Self {
         match Decoder::read_usize(d) {
@@ -1099,6 +1166,7 @@ where
             4 => ReVar(Decodable::decode(d)),
             5 => RePlaceholder(Decodable::decode(d)),
             6 => ReErased,
+            7 => ReError(Decodable::decode(d)),
             _ => panic!(
                 "{}",
                 format!(
@@ -1127,7 +1195,7 @@ where
     ) {
         std::mem::discriminant(self).hash_stable(hcx, hasher);
         match self {
-            ReErased | ReStatic => {
+            ReErased | ReStatic | ReError(_) => {
                 // No variant fields to hash for these ...
             }
             ReLateBound(d, r) => {
diff --git a/compiler/rustc_type_ir/src/visit.rs b/compiler/rustc_type_ir/src/visit.rs
new file mode 100644
index 000000000..62239fd20
--- /dev/null
+++ b/compiler/rustc_type_ir/src/visit.rs
@@ -0,0 +1,115 @@
+//! A visiting traversal mechanism for complex data structures that contain type
+//! information.
+//!
+//! This is a read-only traversal of the data structure.
+//!
+//! This traversal has limited flexibility. Only a small number of "types of
+//! interest" within the complex data structures can receive custom
+//! visitation. These are the ones containing the most important type-related
+//! information, such as `Ty`, `Predicate`, `Region`, and `Const`.
+//!
+//! There are three groups of traits involved in each traversal.
+//! - `TypeVisitable`. This is implemented once for many types, including:
+//!   - Types of interest, for which the methods delegate to the visitor.
+//!   - All other types, including generic containers like `Vec` and `Option`.
+//!     It defines a "skeleton" of how they should be visited.
+//! - `TypeSuperVisitable`. This is implemented only for each type of interest,
+//!   and defines the visiting "skeleton" for these types.
+//! - `TypeVisitor`. This is implemented for each visitor. This defines how
+//!   types of interest are visited.
+//!
+//! This means each visit is a mixture of (a) generic visiting operations, and (b)
+//! custom visit operations that are specific to the visitor.
+//! - The `TypeVisitable` impls handle most of the traversal, and call into
+//!   `TypeVisitor` when they encounter a type of interest.
+//! - A `TypeVisitor` may call into another `TypeVisitable` impl, because some of
+//!   the types of interest are recursive and can contain other types of interest.
+//! - A `TypeVisitor` may also call into a `TypeSuperVisitable` impl, because each
+//!   visitor might provide custom handling only for some types of interest, or
+//!   only for some variants of each type of interest, and then use default
+//!   traversal for the remaining cases.
+//!
+//! For example, if you have `struct S(Ty, U)` where `S: TypeVisitable` and `U:
+//! TypeVisitable`, and an instance `s = S(ty, u)`, it would be visited like so:
+//! ```text
+//! s.visit_with(visitor) calls
+//! - ty.visit_with(visitor) calls
+//!   - visitor.visit_ty(ty) may call
+//!     - ty.super_visit_with(visitor)
+//! - u.visit_with(visitor)
+//! ```
+use crate::Interner;
+
+use std::fmt;
+use std::ops::ControlFlow;
+
+/// This trait is implemented for every type that can be visited,
+/// providing the skeleton of the traversal.
+///
+/// To implement this conveniently, use the derive macro located in
+/// `rustc_macros`.
+pub trait TypeVisitable<I: Interner>: fmt::Debug + Clone {
+    /// The entry point for visiting. To visit a value `t` with a visitor `v`
+    /// call: `t.visit_with(v)`.
+    ///
+    /// For most types, this just traverses the value, calling `visit_with` on
+    /// each field/element.
+    ///
+    /// For types of interest (such as `Ty`), the implementation of this method
+    /// that calls a visitor method specifically for that type (such as
+    /// `V::visit_ty`). This is where control transfers from `TypeFoldable` to
+    /// `TypeVisitor`.
+    fn visit_with<V: TypeVisitor<I>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy>;
+}
+
+pub trait TypeSuperVisitable<I: Interner>: TypeVisitable<I> {
+    /// Provides a default visit for a type of interest. This should only be
+    /// called within `TypeVisitor` methods, when a non-custom traversal is
+    /// desired for the value of the type of interest passed to that method.
+    /// For example, in `MyVisitor::visit_ty(ty)`, it is valid to call
+    /// `ty.super_visit_with(self)`, but any other visiting should be done
+    /// with `xyz.visit_with(self)`.
+    fn super_visit_with<V: TypeVisitor<I>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy>;
+}
+
+/// This trait is implemented for every visiting traversal. There is a visit
+/// method defined for every type of interest. Each such method has a default
+/// that recurses into the type's fields in a non-custom fashion.
+pub trait TypeVisitor<I: Interner>: Sized {
+    type BreakTy = !;
+
+    fn visit_binder<T: TypeVisitable<I>>(&mut self, t: &I::Binder<T>) -> ControlFlow<Self::BreakTy>
+    where
+        I::Binder<T>: TypeSuperVisitable<I>,
+    {
+        t.super_visit_with(self)
+    }
+
+    fn visit_ty(&mut self, t: I::Ty) -> ControlFlow<Self::BreakTy>
+    where
+        I::Ty: TypeSuperVisitable<I>,
+    {
+        t.super_visit_with(self)
+    }
+
+    fn visit_region(&mut self, r: I::Region) -> ControlFlow<Self::BreakTy>
+    where
+        I::Region: TypeSuperVisitable<I>,
+    {
+        r.super_visit_with(self)
+    }
+
+    fn visit_const(&mut self, c: I::Const) -> ControlFlow<Self::BreakTy>
+    where
+        I::Const: TypeSuperVisitable<I>,
+    {
+        c.super_visit_with(self)
+    }
+
+    fn visit_predicate(&mut self, p: I::Predicate) -> ControlFlow<Self::BreakTy>
+    where
+        I::Predicate: TypeSuperVisitable<I>,
+    {
+        p.super_visit_with(self)
+    }
+}